diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d4f066fec..e475ec9f02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,6 +255,10 @@ jobs: source .venv/bin/activate mkdir "$NLTK_DATA" make install-ci + - name: Setup docker-compose + uses: KengoTODA/actions-setup-docker-compose@v1 + with: + version: '2.22.0' - name: Test Ingest (unit) run: | source .venv/bin/activate diff --git a/.github/workflows/ingest-test-fixtures-update-pr.yml b/.github/workflows/ingest-test-fixtures-update-pr.yml index 724a893128..499a1f7593 100644 --- a/.github/workflows/ingest-test-fixtures-update-pr.yml +++ b/.github/workflows/ingest-test-fixtures-update-pr.yml @@ -9,7 +9,7 @@ env: jobs: setup: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-m if: | github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && contains(github.event.head_commit.message, 'ingest-test-fixtures-update')) @@ -56,6 +56,10 @@ jobs: source .venv/bin/activate mkdir "$NLTK_DATA" make install-ci + - name: Setup docker-compose + uses: KengoTODA/actions-setup-docker-compose@v1 + with: + version: '2.22.0' - name: Update test fixtures env: AIRTABLE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AIRTABLE_PERSONAL_ACCESS_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 507108a413..b31e35a290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,26 @@ -## 0.10.19-dev5 +## 0.10.19-dev9 ### Enhancements * **bump `unstructured-inference` to `0.6.6`** The updated version of `unstructured-inference` makes table extraction in `hi_res` mode configurable to fine tune table extraction performance; it also improves element detection by adding a deduplication post processing step in the `hi_res` partitioning of pdfs and images. +* **Detect text in HTML Heading Tags as Titles** This will increase the accuracy of hierarchies in HTML documents and provide more accurate element categorization. If text is in an HTML heading tag and is not a list item, address, or narrative text, categorize it as a title. +* **Update python-based docs** Refactor docs to use the actual unstructured code rather than using the subprocess library to run the cli command itself. +* **Adds data source properties to SharePoint, Outlook, Onedrive, Reddit, and Slack connectors** These properties (date_created, date_modified, version, source_url, record_locator) are written to element metadata during ingest, mapping elements to information about the document source from which they derive. This functionality enables downstream applications to reveal source document applications, e.g. a link to a GDrive doc, Salesforce record, etc. +* **Adds Table support for the `add_chunking_strategy` decorator to partition functions.** In addition to combining elements under Title elements, user's can now specify the `max_characters=` argument to chunk Table elements into TableChunk elements with `text` and `text_as_html` of length characters. This means partitioned Table results are ready for use in downstream applications without any post processing. +* **Expose endpoint url for s3 connectors** By allowing for the endpoint url to be explicitly overwritten, this allows for any non-AWS data providers supporting the s3 protocol to be supported (i.e. minio). + +### Features ### Features ### Fixes * **Tweak `xy-cut` ordering output to be more column friendly** While element ordering from `xy-cut` is usually mostly correct when ordering multi-column documents, sometimes elements from a RHS column will appear before elements in a LHS column. Fix: add swapped `xy-cut` ordering by sorting by X coordinate first and then Y coordinate. +* **Fixes partition_pdf is_alnum reference bug** Problem: The `partition_pdf` when attempt to get bounding box from element experienced a reference before assignment error when the first object is not text extractable. Fix: Switched to a flag when the condition is met. Importance: Crucial to be able to partition with pdf. +* **Fix various cases of HTML text missing after partition** + Problem: Under certain circumstances, text immediately after some HTML tags will be misssing from partition result. + Fix: Updated code to deal with these cases. + Importance: This will ensure the correctness when partitioning HTML and Markdown documents. ## 0.10.18 diff --git a/docs/source/source_connectors/airtable.rst b/docs/source/source_connectors/airtable.rst index 66939236d0..839ec9acff 100644 --- a/docs/source/source_connectors/airtable.rst +++ b/docs/source/source_connectors/airtable.rst @@ -29,29 +29,21 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "airtable", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--personal-access-token", "$AIRTABLE_PERSONAL_ACCESS_TOKEN", - "--output-dir", "airtable-ingest-output" - "--num-processes", "2", - "--reprocess", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.airtable import airtable + + if __name__ == "__main__": + airtable( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="airtable-ingest-output", + num_processes=2, + ), + personal_access_token=os.getenv("AIRTABLE_PERSONAL_ACCESS_TOKEN"), + ) Run via the API --------------- @@ -78,31 +70,23 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "airtable", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--personal-access-token", "$AIRTABLE_PERSONAL_ACCESS_TOKEN", - "--output-dir", "airtable-ingest-output" - "--num-processes", "2", - "--reprocess", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.airtable import airtable + + if __name__ == "__main__": + airtable( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="airtable-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + personal_access_token=os.getenv("AIRTABLE_PERSONAL_ACCESS_TOKEN"), + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/azure.rst b/docs/source/source_connectors/azure.rst index e78ad11e70..479f4e1d58 100644 --- a/docs/source/source_connectors/azure.rst +++ b/docs/source/source_connectors/azure.rst @@ -28,28 +28,20 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "azure", - "--remote-url", "abfs://container1/", - "--account-name", "azureunstructured1" - "--output-dir", "/Output/Path/To/Files", - "--num-processes", "2", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.azure import azure + + if __name__ == "__main__": + azure( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="azure-ingest-output", + num_processes=2, + ), + remote_url="abfs://container1/", + account_name="azureunstructured1", + ) Run via the API --------------- @@ -62,43 +54,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: shell - unstructured-ingest \ - azure \ - --remote-url abfs://container1/ \ - --account-name azureunstructured1 \ - --output-dir azure-ingest-output \ - --num-processes 2 \ - --partition-by-api \ - --api-key "" - - .. tab:: Python - - .. code:: python - - import subprocess - - command = [ - "unstructured-ingest", - "azure", - "--remote-url", "abfs://container1/", - "--account-name", "azureunstructured1" - "--output-dir", "/Output/Path/To/Files", - "--num-processes", "2", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.azure import azure + + if __name__ == "__main__": + azure( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="azure-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + remote_url="abfs://container1/", + account_name="azureunstructured1", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/biomed.rst b/docs/source/source_connectors/biomed.rst index 8cbd579c26..cec1deab6c 100644 --- a/docs/source/source_connectors/biomed.rst +++ b/docs/source/source_connectors/biomed.rst @@ -29,29 +29,21 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "biomed", - "--path", "oa_pdf/07/07/sbaa031.073.PMC7234218.pdf", - "--output-dir", "/Output/Path/To/Files", - "--num-processes", "2", - "--verbose", - "--preserve-downloads", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.biomed import biomed + + if __name__ == "__main__": + biomed( + verbose=True, + read_config=ReadConfig( + preserve_downloads=True, + ), + partition_config=PartitionConfig( + output_dir="biomed-ingest-output-path", + num_processes=2, + ), + path="oa_pdf/07/07/sbaa031.073.PMC7234218.pdf", + ) Run via the API --------------- @@ -78,31 +70,25 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "biomed", - "--path", "oa_pdf/07/07/sbaa031.073.PMC7234218.pdf", - "--output-dir", "/Output/Path/To/Files", - "--num-processes", "2", - "--verbose", - "--preserve-downloads", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.biomed import biomed + + if __name__ == "__main__": + biomed( + verbose=True, + read_config=ReadConfig( + preserve_downloads=True, + ), + partition_config=PartitionConfig( + output_dir="biomed-ingest-output-path", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + path="oa_pdf/07/07/sbaa031.073.PMC7234218.pdf", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/box.rst b/docs/source/source_connectors/box.rst index bf42ea512c..c075566db9 100644 --- a/docs/source/source_connectors/box.rst +++ b/docs/source/source_connectors/box.rst @@ -30,30 +30,23 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "box", - "--box_app_config", "$BOX_APP_CONFIG_PATH" - "--remote-url", "box://utic-test-ingest-fixtures" - "--output-dir", "box-output" - "--num-processes", "2" - "--recursive", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.box import box + + if __name__ == "__main__": + box( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="box-output", + num_processes=2, + ), + box_app_config=os.getenv("BOX_APP_CONFIG_PATH"), + recursive=True, + remote_url="box://utic-test-ingest-fixtures", + ) Run via the API --------------- @@ -81,32 +74,25 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "box", - "--box_app_config", "$BOX_APP_CONFIG_PATH" - "--remote-url", "box://utic-test-ingest-fixtures" - "--output-dir", "box-output" - "--num-processes", "2" - "--recursive", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.box import box + + if __name__ == "__main__": + box( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="box-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + box_app_config=os.getenv("BOX_APP_CONFIG_PATH"), + recursive=True, + remote_url="box://utic-test-ingest-fixtures", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/confluence.rst b/docs/source/source_connectors/confluence.rst index b9606d7c6c..83c3eda7e6 100644 --- a/docs/source/source_connectors/confluence.rst +++ b/docs/source/source_connectors/confluence.rst @@ -30,30 +30,22 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "confluence", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--url", "https://unstructured-ingest-test.atlassian.net", - "--user-email", "12345678@unstructured.io", - "--api-token", "ABCDE1234ABDE1234ABCDE1234", - "--output-dir", "confluence-ingest-output", - "--num-processes", "2", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.confluence import confluence + + if __name__ == "__main__": + confluence( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="confluence-ingest-output", + num_processes=2, + metadata_exclude=["filename", "file_directory", "metadata.data_source.date_processed"], + ), + url="https://unstructured-ingest-test.atlassian.net", + user_email="12345678@unstructured.io", + api_token="ABCDE1234ABDE1234ABCDE1234", + ) Run via the API --------------- @@ -81,32 +73,26 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "confluence", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--url", "https://unstructured-ingest-test.atlassian.net", - "--user-email", "12345678@unstructured.io", - "--api-token", "ABCDE1234ABDE1234ABCDE1234", - "--output-dir", "confluence-ingest-output", - "--num-processes", "2", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.confluence import confluence + + if __name__ == "__main__": + confluence( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="confluence-ingest-output", + num_processes=2, + metadata_exclude=["filename", "file_directory", "metadata.data_source.date_processed"], + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + url="https://unstructured-ingest-test.atlassian.net", + user_email="12345678@unstructured.io", + api_token="ABCDE1234ABDE1234ABCDE1234", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/delta_table.rst b/docs/source/source_connectors/delta_table.rst index 62b504206c..b8d18d94f4 100644 --- a/docs/source/source_connectors/delta_table.rst +++ b/docs/source/source_connectors/delta_table.rst @@ -29,30 +29,20 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "delta-table", - "--table-uri", "s3://utic-dev-tech-fixtures/sample-delta-lake-data/deltatable/", - "--download-dir", "delta-table-ingest-download", - "--output-dir", "delta-table-example", - "--preserve-downloads", - "--storage_options", "AWS_REGION=us-east-2,AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.delta_table import delta_table + + if __name__ == "__main__": + delta_table( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="delta-table-example", + num_processes=2, + ), + table_uri="s3://utic-dev-tech-fixtures/sample-delta-lake-data/deltatable/", + storage_options="AWS_REGION=us-east-2,AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" + ) Run via the API @@ -79,32 +69,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "delta-table", - "--table-uri", "s3://utic-dev-tech-fixtures/sample-delta-lake-data/deltatable/", - "--download-dir", "delta-table-ingest-download", - "--output-dir", "delta-table-example", - "--preserve-downloads", - "--storage_options", "AWS_REGION=us-east-2,AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.delta_table import delta_table + + if __name__ == "__main__": + delta_table( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="delta-table-example", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + table_uri="s3://utic-dev-tech-fixtures/sample-delta-lake-data/deltatable/", + storage_options="AWS_REGION=us-east-2,AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/discord.rst b/docs/source/source_connectors/discord.rst index cb2c4829a4..9455b7eeb3 100644 --- a/docs/source/source_connectors/discord.rst +++ b/docs/source/source_connectors/discord.rst @@ -30,30 +30,26 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "discord", - "--channels", "12345678", - "--token", "$DISCORD_TOKEN", - "--download-dir", "discord-ingest-download", - "--output-dir", "discord-example", - "--preserve-downloads", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.discord import discord + + if __name__ == "__main__": + discord( + verbose=True, + read_config=ReadConfig( + download_dir="discord-ingest-download", + preserve_downloads=True, + ), + partition_config=PartitionConfig( + output_dir="discord-example", + num_processes=2, + ), + channels=["12345678"], + token=os.getenv("DISCORD_TOKEN"), + period=None, + ) Run via the API --------------- @@ -81,32 +77,28 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "discord", - "--channels", "12345678", - "--token", "$DISCORD_TOKEN", - "--download-dir", "discord-ingest-download", - "--output-dir", "discord-example", - "--preserve-downloads", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.discord import discord + + if __name__ == "__main__": + discord( + verbose=True, + read_config=ReadConfig( + download_dir="discord-ingest-download", + preserve_downloads=True, + ), + partition_config=PartitionConfig( + output_dir="discord-example", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + channels=["12345678"], + token=os.getenv("DISCORD_TOKEN"), + period=None, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/dropbox.rst b/docs/source/source_connectors/dropbox.rst index 515b23912b..f8e3d9c867 100644 --- a/docs/source/source_connectors/dropbox.rst +++ b/docs/source/source_connectors/dropbox.rst @@ -30,30 +30,23 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "dropbox", - "--remote-url", "dropbox:// /", - "--output-dir", "dropbox-output", - "--token", "$DROPBOX_TOKEN", - "--num-processes", "2", - "--recursive", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.dropbox import dropbox + + if __name__ == "__main__": + dropbox( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="dropbox-output", + num_processes=2, + ), + remote_url="dropbox:// /", + token=os.getenv("DROPBOX_TOKEN"), + recursive=True, + ) Run via the API --------------- @@ -81,32 +74,25 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "dropbox", - "--remote-url", "dropbox:// /", - "--output-dir", "dropbox-output", - "--token", "$DROPBOX_TOKEN", - "--num-processes", "2", - "--recursive", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.dropbox import dropbox + + if __name__ == "__main__": + dropbox( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="dropbox-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + remote_url="dropbox:// /", + token=os.getenv("DROPBOX_TOKEN"), + recursive=True, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/elasticsearch.rst b/docs/source/source_connectors/elasticsearch.rst index fd4238ab16..f8f7391ffb 100644 --- a/docs/source/source_connectors/elasticsearch.rst +++ b/docs/source/source_connectors/elasticsearch.rst @@ -30,30 +30,22 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "elasticsearch", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--url", "http://localhost:9200", - "--index-name", "movies", - "--jq-query", "{ethnicity, director, plot}", - "--output-dir", "elasticsearch-ingest-output", - "--num-processes", "2" - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.elasticsearch import elasticsearch + + if __name__ == "__main__": + elasticsearch( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="elasticsearch-ingest-output", + num_processes=2, + metadata_exclude=["filename", "file_directory", "metadata.data_source.date_processed"], + ), + url="http://localhost:9200", + index_name="movies", + jq_query="{ethnicity, director, plot}", + ) Run via the API --------------- @@ -81,32 +73,26 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "elasticsearch", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--url", "http://localhost:9200", - "--index-name", "movies", - "--jq-query", "{ethnicity, director, plot}", - "--output-dir", "elasticsearch-ingest-output", - "--num-processes", "2", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.elasticsearch import elasticsearch + + if __name__ == "__main__": + elasticsearch( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="elasticsearch-ingest-output", + num_processes=2, + metadata_exclude=["filename", "file_directory", "metadata.data_source.date_processed"], + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + url="http://localhost:9200", + index_name="movies", + jq_query="{ethnicity, director, plot}", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/github.rst b/docs/source/source_connectors/github.rst index 0b08cac62c..a8ba7c52ca 100644 --- a/docs/source/source_connectors/github.rst +++ b/docs/source/source_connectors/github.rst @@ -29,29 +29,20 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "github", - "--url", "Unstructured-IO/unstructured", - "--git-branch", "main", - "--output-dir", "github-ingest-output", - "--num-processes", "2", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.github import github + + if __name__ == "__main__": + github( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="github-ingest-output", + num_processes=2, + ), + url="Unstructured-IO/unstructured", + git_branch="main", + ) Run via the API --------------- @@ -78,31 +69,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "github", - "--url", "Unstructured-IO/unstructured", - "--git-branch", "main", - "--output-dir", "github-ingest-output", - "--num-processes", "2", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.github import github + + if __name__ == "__main__": + github( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="github-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + url="Unstructured-IO/unstructured", + git_branch="main", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/gitlab.rst b/docs/source/source_connectors/gitlab.rst index e0f722c205..646fedb687 100644 --- a/docs/source/source_connectors/gitlab.rst +++ b/docs/source/source_connectors/gitlab.rst @@ -29,29 +29,20 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "gitlab", - "--url", "Unstructured-IO/unstructured", - "--git-branch", "v0.0.7", - "--output-dir", "gitlab-ingest-output", - "--num-processes", "2", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.gitlab import gitlab + + if __name__ == "__main__": + gitlab( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="gitlab-ingest-output", + num_processes=2, + ), + url="https://gitlab.com/gitlab-com/content-sites/docsy-gitlab", + git_branch="v0.0.7", + ) Run via the API --------------- @@ -78,31 +69,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "gitlab", - "--url", "Unstructured-IO/unstructured", - "--git-branch", "v0.0.7", - "--output-dir", "gitlab-ingest-output", - "--num-processes", "2", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.gitlab import gitlab + + if __name__ == "__main__": + gitlab( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="gitlab-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + url="https://gitlab.com/gitlab-com/content-sites/docsy-gitlab", + git_branch="v0.0.7", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/google_cloud_storage.rst b/docs/source/source_connectors/google_cloud_storage.rst index 96af2c968c..54e009fe8d 100644 --- a/docs/source/source_connectors/google_cloud_storage.rst +++ b/docs/source/source_connectors/google_cloud_storage.rst @@ -29,29 +29,20 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "gcs", - "--remote-url", "gs://utic-test-ingest-fixtures-public/", - "--output-dir", "dropbox-output", - "--num-processes", "2", - "--recursive", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.gcs import gcs + + if __name__ == "__main__": + gcs( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="gcs-output", + num_processes=2, + ), + remote_url="gs://utic-test-ingest-fixtures-public/", + recursive=True, + ) Run via the API --------------- @@ -76,29 +67,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "gcs", - "--remote-url", "gs://utic-test-ingest-fixtures-public/", - "--output-dir", "dropbox-output", - "--num-processes", "2", - "--recursive", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.gcs import gcs + + if __name__ == "__main__": + gcs( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="gcs-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + remote_url="gs://utic-test-ingest-fixtures-public/", + recursive=True, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/google_drive.rst b/docs/source/source_connectors/google_drive.rst index 9ec9724285..869f680798 100644 --- a/docs/source/source_connectors/google_drive.rst +++ b/docs/source/source_connectors/google_drive.rst @@ -30,28 +30,21 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "google-drive", - "--drive-id", "", - "--service-account-key",, "Path/To/Your/Service/Account/Key" - "--output-dir", "/Output/Path/To/Files", - "--num-processes", "2", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.google_drive import gdrive + + if __name__ == "__main__": + gdrive( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="google-drive-ingest-output", + num_processes=2, + ), + drive_id="POPULATE WITH FILE OR FOLDER ID", + service_account_key="POPULATE WITH DRIVE SERVICE ACCOUNT KEY", + recursive=True, + ) Run via the API --------------- @@ -79,30 +72,25 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "google-drive", - "--drive-id", "", - "--service-account-key",, "Path/To/Your/Service/Account/Key" - "--output-dir", "/Output/Path/To/Files", - "--num-processes", "2", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.google_drive import gdrive + + if __name__ == "__main__": + gdrive( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="google-drive-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + drive_id="POPULATE WITH FILE OR FOLDER ID", + service_account_key="POPULATE WITH DRIVE SERVICE ACCOUNT KEY", + recursive=True, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/jira.rst b/docs/source/source_connectors/jira.rst index 37b2056e62..f93bb26f40 100644 --- a/docs/source/source_connectors/jira.rst +++ b/docs/source/source_connectors/jira.rst @@ -31,30 +31,22 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "jira", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--url", "https://unstructured-jira-connector-test.atlassian.net", - "--user-email", "12345678@unstructured.io", - "--api-token", "ABCDE1234ABDE1234ABCDE1234", - "--output-dir", "jira-ingest-output", - "--num-processes", "2", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.jira import jira + + if __name__ == "__main__": + jira( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="jira-ingest-output", + num_processes=2, + metadata_exclude=["filename", "file_directory", "metadata.data_source.date_processed"], + ), + url="https://unstructured-jira-connector-test.atlassian.net", + user_email="12345678@unstructured.io", + api_token="ABCDE1234ABDE1234ABCDE1234", + ) Run via the API --------------- @@ -82,32 +74,26 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "jira", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--url", "https://unstructured-jira-connector-test.atlassian.net", - "--user-email", "12345678@unstructured.io", - "--api-token", "ABCDE1234ABDE1234ABCDE1234", - "--output-dir", "jira-ingest-output", - "--num-processes", "2", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.jira import jira + + if __name__ == "__main__": + jira( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="jira-ingest-output", + num_processes=2, + metadata_exclude=["filename", "file_directory", "metadata.data_source.date_processed"], + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + url="https://unstructured-jira-connector-test.atlassian.net", + user_email="12345678@unstructured.io", + api_token="ABCDE1234ABDE1234ABCDE1234", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/local_connector.rst b/docs/source/source_connectors/local_connector.rst index daa6645e52..b93ab589bf 100644 --- a/docs/source/source_connectors/local_connector.rst +++ b/docs/source/source_connectors/local_connector.rst @@ -23,29 +23,20 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "local", - "--input-path", "example-docs", - "--output-dir", "dropbox-output", - "--num-processes", "2", - "--recursive", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.local import local + + if __name__ == "__main__": + local( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="local-ingest-output", + num_processes=2, + ), + input_path="example-docs", + recursive=True, + ) Run via the API --------------- @@ -72,31 +63,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "local", - "--input-path", "example-docs", - "--output-dir", "dropbox-output", - "--num-processes", "2", - "--recursive", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.local import local + + if __name__ == "__main__": + local( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="local-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + input_path="example-docs", + recursive=True, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/notion.rst b/docs/source/source_connectors/notion.rst index a79bd2d251..3036a01924 100644 --- a/docs/source/source_connectors/notion.rst +++ b/docs/source/source_connectors/notion.rst @@ -30,30 +30,22 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "notion", - "--api-key", "", - "--output-dir", "notion-ingest-output", - "--page-ids", "", - "--database-ids", """", - "--num-processes", "2", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.notion import notion + + if __name__ == "__main__": + notion( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="notion-ingest-output", + num_processes=2, + ), + api_key="POPULATE API KEY", + page_ids=["LIST", "OF", "PAGE", "IDS"], + database_ids=["LIST", "OF", "DATABASE", "IDS"], + recursive=False, + ) Run via the API --------------- @@ -81,32 +73,26 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "notion", - "--api-key", "", - "--output-dir", "notion-ingest-output", - "--page-ids", "", - "--database-ids", """", - "--num-processes", "2", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.notion import notion + + if __name__ == "__main__": + notion( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="notion-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + api_key="POPULATE API KEY", + page_ids=["LIST", "OF", "PAGE", "IDS"], + database_ids=["LIST", "OF", "DATABASE", "IDS"], + recursive=False, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/onedrive.rst b/docs/source/source_connectors/onedrive.rst index 90241d0800..592a49313d 100644 --- a/docs/source/source_connectors/onedrive.rst +++ b/docs/source/source_connectors/onedrive.rst @@ -33,33 +33,25 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "onedrive", - "--client-id", "", - "--client-cred", "", - "--authority-url", "", - "--tenant", "", - "--user-pname", "", - "--path", "", - "--output-dir", "onedrive-ingest-output", - "--num-processes", "2", - "--verbose" - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.onedrive import onedrive + + if __name__ == "__main__": + onedrive( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="onedrive-ingest-output", + num_processes=2, + ), + client_id="", + client_cred="", + authority_url="", + tenant="", + user_pname="", + path="", + recursive=False, + ) Run via the API --------------- @@ -90,35 +82,29 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "onedrive", - "--client-id", "", - "--client-cred", "", - "--authority-url", "", - "--tenant", "", - "--user-pname", "", - "--path", "", - "--output-dir", "onedrive-ingest-output", - "--num-processes", "2", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.onedrive import onedrive + + if __name__ == "__main__": + onedrive( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="onedrive-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + client_id="", + client_cred="", + authority_url="", + tenant="", + user_pname="", + path="", + recursive=False, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/outlook.rst b/docs/source/source_connectors/outlook.rst index ce3c1f29f0..0e78738614 100644 --- a/docs/source/source_connectors/outlook.rst +++ b/docs/source/source_connectors/outlook.rst @@ -33,33 +33,26 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "outlook", - "--client-id", "$MS_CLIENT_ID", - "--client-cred", "$MS_CLIENT_CRED", - "--tenant", "", - "--user-email", "$MS_USER_EMAIL", - "--outlook-folders", "Inbox,Sent Items", - "--output-dir", "onedrive-ingest-output", - "--num-processes", "2", - "--recursive", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.outlook import outlook + + if __name__ == "__main__": + outlook( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="outlook-output", + num_processes=2, + ), + client_id=os.getenv("MS_CLIENT_ID"), + client_cred=os.getenv("MS_CLIENT_CRED"), + tenant=os.getenv("MS_TENANT_ID"), + user_email=os.getenv("MS_USER_EMAIL"), + outlook_folders=["Inbox", "Sent Items"], + recursive=True, + ) Run via the API --------------- @@ -86,31 +79,28 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "airtable", - "--metadata-exclude", "filename,file_directory,metadata.data_source.date_processed", - "--personal-access-token", "$AIRTABLE_PERSONAL_ACCESS_TOKEN", - "--output-dir", "airtable-ingest-output" - "--num-processes", "2", - "--reprocess", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.outlook import outlook + + if __name__ == "__main__": + outlook( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="outlook-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + client_id=os.getenv("MS_CLIENT_ID"), + client_cred=os.getenv("MS_CLIENT_CRED"), + tenant=os.getenv("MS_TENANT_ID"), + user_email=os.getenv("MS_USER_EMAIL"), + outlook_folders=["Inbox", "Sent Items"], + recursive=True, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/reddit.rst b/docs/source/source_connectors/reddit.rst index f31d0d55ba..4d7c82be82 100644 --- a/docs/source/source_connectors/reddit.rst +++ b/docs/source/source_connectors/reddit.rst @@ -33,33 +33,24 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "reddit", - "--subreddit-name", "machinelearning", - "--client-id", "", - "--client-secret", "", - "--user-agent", "Unstructured Ingest Subreddit fetcher by \\u\\...", - "--search-query", "Unstructured", - "--num-posts", "10", - "--output-dir", "reddit-ingest-output", - "--num-processes", "2", - "--verbose" - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.reddit import reddit + + if __name__ == "__main__": + reddit( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="reddit-ingest-output", + num_processes=2, + ), + subreddit_name="machinelearning", + client_id="", + client_secret="", + user_agent=r"Unstructured Ingest Subreddit fetcher by \\u\...", + search_query="Unstructured", + num_posts=10, + ) Run via the API --------------- @@ -90,35 +81,28 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "reddit", - "--subreddit-name", "machinelearning", - "--client-id", "", - "--client-secret", "", - "--user-agent", "Unstructured Ingest Subreddit fetcher by \\u\\...", - "--search-query", "Unstructured", - "--num-posts", "10", - "--output-dir", "reddit-ingest-output", - "--num-processes", "2", - "--verbose" - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.reddit import reddit + + if __name__ == "__main__": + reddit( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="reddit-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + subreddit_name="machinelearning", + client_id="", + client_secret="", + user_agent=r"Unstructured Ingest Subreddit fetcher by \\u\...", + search_query="Unstructured", + num_posts=10, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/s3.rst b/docs/source/source_connectors/s3.rst index d2f16ad77f..483937a842 100644 --- a/docs/source/source_connectors/s3.rst +++ b/docs/source/source_connectors/s3.rst @@ -28,28 +28,20 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "s3", - "--remote-url", "s3://utic-dev-tech-fixtures/small-pdf-set/", - "--anonymous", - "--output-dir", "s3-small-batch-output", - "--num-processes", "2" - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.s3 import s3 + + if __name__ == "__main__": + s3( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="s3-small-batch-output", + num_processes=2, + ), + remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/", + anonymous=True, + ) Run via the API --------------- @@ -75,30 +67,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "s3", - "--remote-url", "s3://utic-dev-tech-fixtures/small-pdf-set/", - "--anonymous", - "--output-dir", "s3-small-batch-output", - "--num-processes", "2", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.s3 import s3 + + if __name__ == "__main__": + s3( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="s3-small-batch-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/", + anonymous=True, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/salesforce.rst b/docs/source/source_connectors/salesforce.rst index 04183ed7a9..fd52ad6d6a 100644 --- a/docs/source/source_connectors/salesforce.rst +++ b/docs/source/source_connectors/salesforce.rst @@ -32,32 +32,25 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "salesforce", - "--username" "$SALESFORCE_USERNAME" - "--consumer-key" "$SALESFORCE_CONSUMER_KEY" - "--private-key-path" "$SALESFORCE_PRIVATE_KEY_PATH" - "--categories" "EmailMessage,Account,Lead,Case,Campaign" - "--output-dir" "salesforce-output" - "--num-processes", "2" - "--recursive", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.salesforce import salesforce + + if __name__ == "__main__": + salesforce( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="salesforce-output", + num_processes=2, + ), + username=os.getenv("SALESFORCE_USERNAME"), + consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"), + private_key_path=os.getenv("SALESFORCE_PRIVATE_KEY_PATH"), + categories=["EmailMessage", "Account", "Lead", "Case", "Campaign"], + recursive=True, + ) Run via the API --------------- @@ -87,34 +80,27 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "salesforce", - "--username" "$SALESFORCE_USERNAME" - "--consumer-key" "$SALESFORCE_CONSUMER_KEY" - "--private-key-path" "$SALESFORCE_PRIVATE_KEY_PATH" - "--categories" "EmailMessage,Account,Lead,Case,Campaign" - "--output-dir" "salesforce-output" - "--num-processes", "2" - "--recursive", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.salesforce import salesforce + + if __name__ == "__main__": + salesforce( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="salesforce-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + username=os.getenv("SALESFORCE_USERNAME"), + consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"), + private_key_path=os.getenv("SALESFORCE_PRIVATE_KEY_PATH"), + categories=["EmailMessage", "Account", "Lead", "Case", "Campaign"], + recursive=True, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/sharepoint.rst b/docs/source/source_connectors/sharepoint.rst index 67ffe626d9..bc0b144dd9 100644 --- a/docs/source/source_connectors/sharepoint.rst +++ b/docs/source/source_connectors/sharepoint.rst @@ -25,37 +25,32 @@ Run Locally --files-only "Flag to process only files within the site(s)" \ --output-dir sharepoint-ingest-output \ --num-processes 2 \ + --path "Shared Documents" \ --verbose .. tab:: Python .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "sharepoint", - "--client-id", "", - "--client-cred", "", - "--site", "", - "--files-only", "Flag to process only files within the site(s)", - "--output-dir", "sharepoint-ingest-output", - "--num-processes", "2", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.sharepoint import sharepoint + + if __name__ == "__main__": + sharepoint( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="sharepoint-ingest-output", + num_processes=2, + ), + client_id="", + client_cred="", + site="", + # Flag to process only files within the site(s) + files_only=True, + path="Shared Documents", + recursive=False, + ) Run via the API --------------- @@ -77,6 +72,7 @@ You can also use upstream connectors with the ``unstructured`` API. For this you --output-dir sharepoint-ingest-output \ --num-processes 2 \ --verbose \ + --path "Shared Documents" \ --partition-by-api \ --api-key "" @@ -84,33 +80,29 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "sharepoint", - "--client-id", "", - "--client-cred", "", - "--site", "", - "--files-only", "Flag to process only files within the site(s)", - "--output-dir", "sharepoint-ingest-output", - "--num-processes", "2", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.sharepoint import sharepoint + + if __name__ == "__main__": + sharepoint( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="sharepoint-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + client_id="", + client_cred="", + site="", + # Flag to process only files within the site(s) + files_only=True, + path="Shared Documents", + recursive=False, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/slack.rst b/docs/source/source_connectors/slack.rst index dcb4700e60..53da13a257 100644 --- a/docs/source/source_connectors/slack.rst +++ b/docs/source/source_connectors/slack.rst @@ -30,30 +30,22 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "slack", - "--channels", "12345678", - "--token", "12345678", - "--download-dir", "slack-ingest-download", - "--output-dir", "slack-ingest-output", - "--start-date", "2023-04-01T01:00:00-08:00", - "--end-date", "2023-04-02" - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.slack import slack + + if __name__ == "__main__": + slack( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="slack-ingest-download", + num_processes=2, + ), + channels=["12345678"], + token="12345678", + start_date="2023-04-01T01:00:00-08:00", + end_date="2023-04-02,", + ) Run via the API --------------- @@ -81,32 +73,26 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "slack", - "--channels", "12345678", - "--token", "12345678", - "--download-dir", "slack-ingest-download", - "--output-dir", "slack-ingest-output", - "--start-date", "2023-04-01T01:00:00-08:00", - "--end-date", "2023-04-02", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.slack import slack + + if __name__ == "__main__": + slack( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="slack-ingest-download", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + channels=["12345678"], + token="12345678", + start_date="2023-04-01T01:00:00-08:00", + end_date="2023-04-02,", + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/docs/source/source_connectors/wikipedia.rst b/docs/source/source_connectors/wikipedia.rst index 7d81160994..cf6a6af061 100644 --- a/docs/source/source_connectors/wikipedia.rst +++ b/docs/source/source_connectors/wikipedia.rst @@ -28,28 +28,21 @@ Run Locally .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "wikipedia", - "--page-title", "Open Source Software", - "--output-dir", "dropbox-output", - "--num-processes", "2", - "--verbose", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + from unstructured.ingest.runner.wikipedia import wikipedia + from unstructured.ingest.interfaces import ReadConfig, PartitionConfig + + + if __name__ == "__main__": + wikipedia( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="wikipedia-ingest-output", + num_processes=2 + ), + page_title="Open Source Software", + auto_suggest=False, + ) Run via the API --------------- @@ -75,30 +68,24 @@ You can also use upstream connectors with the ``unstructured`` API. For this you .. code:: python - import subprocess - - command = [ - "unstructured-ingest", - "wikipedia", - "--page-title", "Open Source Software", - "--output-dir", "dropbox-output", - "--num-processes", "2", - "--verbose", - "--partition-by-api", - "--api-key", "", - ] - - # Run the command - process = subprocess.Popen(command, stdout=subprocess.PIPE) - output, error = process.communicate() - - # Print output - if process.returncode == 0: - print('Command executed successfully. Output:') - print(output.decode()) - else: - print('Command failed. Error:') - print(error.decode()) + import os + + from unstructured.ingest.interfaces import PartitionConfig, ReadConfig + from unstructured.ingest.runner.wikipedia import wikipedia + + if __name__ == "__main__": + wikipedia( + verbose=True, + read_config=ReadConfig(), + partition_config=PartitionConfig( + output_dir="wikipedia-ingest-output", + num_processes=2, + partition_by_api=True, + api_key=os.getenv("UNSTRUCTURED_API_KEY"), + ), + page_title="Open Source Software", + auto_suggest=False, + ) Additionally, you will need to pass the ``--partition-endpoint`` if you're running the API locally. You can find more information about the ``unstructured`` API `here `_. diff --git a/example-docs/interface-config-guide-p93.pdf b/example-docs/interface-config-guide-p93.pdf new file mode 100644 index 0000000000..db41a7cae4 Binary files /dev/null and b/example-docs/interface-config-guide-p93.pdf differ diff --git a/examples/ingest/sharepoint/ingest.sh b/examples/ingest/sharepoint/ingest.sh index 4a73ca65fb..53a1218207 100644 --- a/examples/ingest/sharepoint/ingest.sh +++ b/examples/ingest/sharepoint/ingest.sh @@ -25,4 +25,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \ --files-only "Flag to process only files within the site(s)" \ --output-dir sharepoint-ingest-output \ --num-processes 2 \ + --path "Shared Documents" \ --verbose diff --git a/scripts/elasticsearch-test-helpers/create-and-check-es.sh b/scripts/elasticsearch-test-helpers/create-and-check-es.sh index 44fca2f7d3..dc06c21a16 100755 --- a/scripts/elasticsearch-test-helpers/create-and-check-es.sh +++ b/scripts/elasticsearch-test-helpers/create-and-check-es.sh @@ -1,37 +1,14 @@ #!/usr/bin/env bash -SCRIPT_DIR=$(dirname "$(realpath "$0")") - -# Create the Elasticsearch cluster and get the container id -docker run -d --rm -p 9200:9200 -p 9300:9300 -e "xpack.security.enabled=false" -e "discovery.type=single-node" --name es-test docker.elastic.co/elasticsearch/elasticsearch:8.7.0 +set -e -# Wait for Elasticsearch container to start -echo "Waiting for Elasticsearch container to start..." -sleep 1 - -url="http://localhost:9200/_cluster/health?wait_for_status=green&timeout=50s" -status_code=0 -retry_count=0 -max_retries=6 +SCRIPT_DIR=$(dirname "$(realpath "$0")") -# Check the cluster status repeatedly until it becomes live or maximum retries are reached -while [ "$status_code" -ne 200 ] && [ "$retry_count" -lt "$max_retries" ]; do - # Send a GET request to the cluster health API - response=$(curl -s -o /dev/null -w "%{http_code}" "$url") - status_code="$response" +# Create the Elasticsearch cluster +docker compose version +docker compose -f "$SCRIPT_DIR"/docker-compose.yaml up --wait +docker compose -f "$SCRIPT_DIR"/docker-compose.yaml ps - # Process the files only when the Elasticsearch cluster is live - if [ "$status_code" -eq 200 ]; then - echo "Cluster is live." - python "$SCRIPT_DIR/create_and_fill_es.py" - else - ((retry_count++)) - echo "Cluster is not available. Retrying in 5 seconds... (Attempt $retry_count)" - sleep 5 - fi -done -# If the cluster has not become live, exit after a certain number of tries -if [ "$status_code" -ne 200 ]; then - echo "Cluster took an unusually long time to create (>25 seconds). Expected time is around 10 seconds. Exiting." -fi +echo "Cluster is live." +"$SCRIPT_DIR"/create_and_fill_es.py diff --git a/scripts/elasticsearch-test-helpers/create_and_fill_es.py b/scripts/elasticsearch-test-helpers/create_and_fill_es.py old mode 100644 new mode 100755 index 796e2187a8..a761255741 --- a/scripts/elasticsearch-test-helpers/create_and_fill_es.py +++ b/scripts/elasticsearch-test-helpers/create_and_fill_es.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import pandas as pd from elasticsearch import Elasticsearch from elasticsearch.helpers import bulk diff --git a/scripts/elasticsearch-test-helpers/docker-compose.yaml b/scripts/elasticsearch-test-helpers/docker-compose.yaml new file mode 100644 index 0000000000..47cb93ae1f --- /dev/null +++ b/scripts/elasticsearch-test-helpers/docker-compose.yaml @@ -0,0 +1,15 @@ +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0 + container_name: es-test + ports: + - 9200:9200 + - 9300:9300 + environment: + - xpack.security.enabled=false + - discovery.type=single-node + healthcheck: + test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"] + interval: 30s + timeout: 30s + retries: 3 diff --git a/scripts/minio-test-helpers/create-and-check-minio.sh b/scripts/minio-test-helpers/create-and-check-minio.sh new file mode 100755 index 0000000000..09089a944a --- /dev/null +++ b/scripts/minio-test-helpers/create-and-check-minio.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(dirname "$(realpath "$0")") + +secret_key=minioadmin +access_key=minioadmin +region=us-east-2 +endpoint_url=http://localhost:9000 +bucket_name=utic-dev-tech-fixtures + +function upload(){ + echo "Uploading test content to new bucket in minio" + AWS_REGION=$region AWS_SECRET_ACCESS_KEY=$secret_key AWS_ACCESS_KEY_ID=$access_key \ + aws --output json --endpoint-url $endpoint_url s3api create-bucket --bucket $bucket_name | jq + AWS_REGION=$region AWS_SECRET_ACCESS_KEY=$secret_key AWS_ACCESS_KEY_ID=$access_key \ + aws --endpoint-url $endpoint_url s3 cp "$SCRIPT_DIR"/wiki_movie_plots_small.csv s3://$bucket_name/ +} + +# Create Minio single server +docker compose version +docker compose -f "$SCRIPT_DIR"/docker-compose.yaml up --wait +docker compose -f "$SCRIPT_DIR"/docker-compose.yaml ps + +echo "Cluster is live." +upload diff --git a/scripts/minio-test-helpers/docker-compose.yaml b/scripts/minio-test-helpers/docker-compose.yaml new file mode 100644 index 0000000000..acc3ec9b48 --- /dev/null +++ b/scripts/minio-test-helpers/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + minio: + image: quay.io/minio/minio + container_name: minio-test + ports: + - 9000:9000 + - 9001:9001 + command: server --console-address ":9001" /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 20s + retries: 3 diff --git a/scripts/minio-test-helpers/wiki_movie_plots_small.csv b/scripts/minio-test-helpers/wiki_movie_plots_small.csv new file mode 100644 index 0000000000..2fbb2b49bb --- /dev/null +++ b/scripts/minio-test-helpers/wiki_movie_plots_small.csv @@ -0,0 +1,31 @@ +Release Year,Title,Origin/Ethnicity,Director,Cast,Genre,Wiki Page,Plot +1901,Kansas Saloon Smashers,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Kansas_Saloon_Smashers,"A bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]" +1901,Love by the Light of the Moon,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Love_by_the_Light_of_the_Moon,"The moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better." +1901,The Martyred Presidents,American,Unknown,,unknown,https://en.wikipedia.org/wiki/The_Martyred_Presidents,"The film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination. +In the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice." +1901,"Terrible Teddy, the Grizzly King",American,Unknown,,unknown,"https://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_King","Lasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading ""His Photographer"" and ""His Press Agent"" respectively, follow him into the shot; the photographer sets up his camera. ""Teddy"" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. ""Teddy"" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. ""Teddy"" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs." +1902,Jack and the Beanstalk,American,"George S. Fleming, Edwin S. Porter",,unknown,https://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film),"The earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince." +1903,Alice in Wonderland,American,Cecil Hepworth,May Clark,unknown,https://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film),"Alice follows a large white rabbit down a ""Rabbit-hole"". She finds a tiny door. When she finds a bottle labeled ""Drink me"", she does, and shrinks, but not enough to pass through the door. She then eats something labeled ""Eat me"" and grows larger. She finds a fan when enables her to shrink enough to get into the ""Garden"" and try to get a ""Dog"" to play with her. She enters the ""White Rabbit's tiny House,"" but suddenly resumes her normal size. In order to get out, she has to use the ""magic fan."" +She enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. ""The Duchess's Cheshire Cat"" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's ""Mad Tea-Party."" After a while, she leaves. +The Queen invites Alice to join the ""ROYAL PROCESSION"": a parade of marching playing cards and others headed by the White Rabbit. When Alice ""unintentionally offends the Queen"", the latter summons the ""Executioner"". Alice ""boxes the ears"", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream." +1903,The Great Train Robbery,American,Edwin S. Porter,,western,https://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film),"The film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left. +Meanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water. +There is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail." +1904,The Suburbanite,American,Wallace McCutcheon,,comedy,https://en.wikipedia.org/wiki/The_Suburbanite,"The film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest." +1905,The Little Train Robbery,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Little_Train_Robbery,"The opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The ""Bandit Queen,"" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The ""Bandit Queen"" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin. +The next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the ""valuables,"" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies. +In the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the ""plunder."" The police, however, have struck the right trail and are in close pursuit. While the ""plunder"" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the ""Bandit Queen.""" +1905,The Night Before Christmas,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film),"Scenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents." +1906,Dream of a Rarebit Fiend,American,Wallace McCutcheon and Edwin S. Porter,,short,https://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film),"The Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed." +1906,From Leadville to Aspen: A Hold-Up in the Rockies,American,Francis J. Marion and Wallace McCutcheon,,short action/crime western,https://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_Rockies,The film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart. +1906,Kathleen Mavourneen,American,Edwin S. Porter,,short film,https://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film),"Irish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration. +Film historian Charles Musser writes of Porter's adaptation, ""O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.""[1]" +1907,Daniel Boone,American,Wallace McCutcheon and Ediwin S. Porter,"William Craven, Florence Lawrence",biographical,https://en.wikipedia.org/wiki/Daniel_Boone_(1907_film),"Boone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]" +1907,How Brown Saw the Baseball Game,American,Unknown,Unknown,comedy,https://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_Game,"Before heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]" +1907,Laughing Gas,American,Edwin Stanton Porter,"Bertha Regustus, Edward Boulden",comedy,https://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_Film,"The plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets ""catches"" the laughter from her, including a vendor and police officers." +1908,The Adventures of Dollie,American,D. W. Griffith,"Arthur V. Johnson, Linda Arvidson",drama,https://en.wikipedia.org/wiki/The_Adventures_of_Dollie,"On a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents." +1908,The Black Viper,American,D. W. Griffith,D. W. Griffith,drama,https://en.wikipedia.org/wiki/The_Black_Viper,"A thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house." +1908,A Calamitous Elopement,American,D.W. Griffith,"Harry Solter, Linda Arvidson",comedy,https://en.wikipedia.org/wiki/A_Calamitous_Elopement,"A young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings." +1908,The Call of the Wild,American,D. W. Griffith,Charles Inslee,adventure,https://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film),"A white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as ""The Biograph Girl.""" +1908,A Christmas Carol,American,Unknown,Tom Ricketts,drama,https://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film),"No prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life." +1908,The Fight for Freedom,American,D. W. Griffith,"Florence Auer, John G. Adolfi",western,https://en.wikipedia.org/wiki/The_Fight_for_Freedom,"The film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town." diff --git a/test_unstructured/chunking/test_title.py b/test_unstructured/chunking/test_title.py index 8ccfde5af8..bc8bdcc6b0 100644 --- a/test_unstructured/chunking/test_title.py +++ b/test_unstructured/chunking/test_title.py @@ -31,7 +31,7 @@ def test_split_elements_by_title_and_table(): Text("It is storming outside."), CheckBox(), ] - sections = _split_elements_by_title_and_table(elements, combine_under_n_chars=0) + sections = _split_elements_by_title_and_table(elements, combine_text_under_n_chars=0) assert sections == [ [ @@ -75,7 +75,7 @@ def test_chunk_by_title(): Text("It is storming outside."), CheckBox(), ] - chunks = chunk_by_title(elements, combine_under_n_chars=0) + chunks = chunk_by_title(elements, combine_text_under_n_chars=0) assert chunks == [ CompositeElement( @@ -112,7 +112,7 @@ def test_chunk_by_title_respects_section_change(): Text("It is storming outside."), CheckBox(), ] - chunks = chunk_by_title(elements, combine_under_n_chars=0) + chunks = chunk_by_title(elements, combine_text_under_n_chars=0) assert chunks == [ CompositeElement( @@ -147,7 +147,7 @@ def test_chunk_by_title_separates_by_page_number(): Text("It is storming outside."), CheckBox(), ] - chunks = chunk_by_title(elements, multipage_sections=False, combine_under_n_chars=0) + chunks = chunk_by_title(elements, multipage_sections=False, combine_text_under_n_chars=0) assert chunks == [ CompositeElement( @@ -182,7 +182,7 @@ def test_chunk_by_title_groups_across_pages(): Text("It is storming outside."), CheckBox(), ] - chunks = chunk_by_title(elements, multipage_sections=True, combine_under_n_chars=0) + chunks = chunk_by_title(elements, multipage_sections=True, combine_text_under_n_chars=0) assert chunks == [ CompositeElement( @@ -212,24 +212,32 @@ def test_add_chunking_strategy_on_partition_html_respects_multipage(): filename, chunking_strategy="by_title", multipage_sections=False, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) partitioned_elements_multipage_true_combine_chars_0 = partition_html( filename, chunking_strategy="by_title", multipage_sections=True, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) elements = partition_html(filename) cleaned_elements_multipage_false_combine_chars_0 = chunk_by_title( elements, multipage_sections=False, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) cleaned_elements_multipage_true_combine_chars_0 = chunk_by_title( elements, multipage_sections=True, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) assert ( partitioned_elements_multipage_false_combine_chars_0 @@ -244,7 +252,21 @@ def test_add_chunking_strategy_on_partition_html_respects_multipage(): ) -def test_add_chunking_strategy_raises_error_for_invalid_n_chars(): +@pytest.mark.parametrize( + ("combine_text_under_n_chars", "new_after_n_chars", "max_characters"), + [ + (-1, -1, -1), + (0, 0, 0), + (-5666, -6777, -8999), + (-5, 40, 50), + (50, 100, 20), + ], +) +def test_add_chunking_strategy_raises_error_for_invalid_n_chars( + combine_text_under_n_chars, + new_after_n_chars, + max_characters, +): elements = [ Title("A Great Day"), Text("Today is a great day."), @@ -258,7 +280,12 @@ def test_add_chunking_strategy_raises_error_for_invalid_n_chars(): CheckBox(), ] with pytest.raises(ValueError): - chunk_by_title(elements, combine_under_n_chars=1, new_after_n_chars=0) + chunk_by_title( + elements, + combine_text_under_n_chars=combine_text_under_n_chars, + new_after_n_chars=new_after_n_chars, + max_characters=max_characters, + ) def test_chunk_by_title_drops_extra_metadata(): @@ -335,7 +362,7 @@ def test_chunk_by_title_drops_extra_metadata(): ), ] - chunks = chunk_by_title(elements, combine_under_n_chars=0) + chunks = chunk_by_title(elements, combine_text_under_n_chars=0) assert str(chunks[0]) == str( CompositeElement("A Great Day\n\nToday is a great day.\n\nIt is sunny outside."), diff --git a/test_unstructured/documents/test_html.py b/test_unstructured/documents/test_html.py index d6d236f08f..02f6d6bc72 100644 --- a/test_unstructured/documents/test_html.py +++ b/test_unstructured/documents/test_html.py @@ -17,6 +17,7 @@ from unstructured.documents.html import ( HEADING_TAGS, LIST_ITEM_TAGS, + SECTION_TAGS, TABLE_TAGS, TEXT_TAGS, HTMLDocument, @@ -41,8 +42,15 @@ TAGS = TAGS.replace(">", "").split("<")[1:] -INCLUDED_TAGS = TEXT_TAGS + HEADING_TAGS + LIST_ITEM_TAGS + ["div"] -EXCLUDED_TAGS = "tag", [tag for tag in TAGS if tag not in INCLUDED_TAGS] +VOID_TAGS = "
" +VOID_TAGS = VOID_TAGS.replace(">", "").split("<")[1:] + +INCLUDED_TAGS = TEXT_TAGS + HEADING_TAGS + LIST_ITEM_TAGS + SECTION_TAGS +EXCLUDED_TAGS = [ + tag + for tag in TAGS + if tag not in (INCLUDED_TAGS + TABLE_TAGS + VOID_TAGS + ["html", "head", "body"]) +] @pytest.fixture() @@ -685,3 +693,31 @@ def test_sample_doc_with_emoji(): # NOTE(robinson) - unclear why right now, but the output is the emoji on the test runners # and the byte string representation when running locally on mac assert doc.elements[0].text in ["Hello again ð\x9f\x98\x80", "Hello again 😀"] + + +def test_only_plain_text_in_body(): + raw_html = "Hello" + doc = HTMLDocument.from_string(raw_html) + assert doc.elements[0].text == "Hello" + + +def test_plain_text_before_anything_in_body(): + raw_html = "Hello

World

" + doc = HTMLDocument.from_string(raw_html) + assert doc.elements[0].text == "Hello" + assert doc.elements[1].text == "World" + + +def test_line_break_in_container(): + raw_html = "
Hello
World
" + doc = HTMLDocument.from_string(raw_html) + assert doc.elements[0].text == "Hello" + assert doc.elements[1].text == "World" + + +@pytest.mark.parametrize("tag", TEXT_TAGS) +def test_line_break_in_text_tag(tag): + raw_html = f"<{tag}>Hello
World" + doc = HTMLDocument.from_string(raw_html) + assert doc.elements[0].text == "Hello" + assert doc.elements[1].text == "World" diff --git a/test_unstructured/partition/csv/test_csv.py b/test_unstructured/partition/csv/test_csv.py index 050c2c2567..3f3d5e4ae0 100644 --- a/test_unstructured/partition/csv/test_csv.py +++ b/test_unstructured/partition/csv/test_csv.py @@ -8,6 +8,7 @@ EXPECTED_TEXT, EXPECTED_TEXT_WITH_EMOJI, ) +from unstructured.chunking.title import chunk_by_title from unstructured.cleaners.core import clean_extra_whitespace from unstructured.documents.elements import Table from unstructured.partition.csv import partition_csv @@ -189,3 +190,18 @@ def test_partition_csv_with_json(filename, expected_text, expected_table): assert elements[0].metadata.filename == test_elements[0].metadata.filename for i in range(len(elements)): assert elements[i] == test_elements[i] + + +def test_add_chunking_strategy_to_partition_csv_non_default(): + filename = "example-docs/stanley-cups.csv" + + elements = partition_csv(filename=filename) + chunk_elements = partition_csv( + filename, + chunking_strategy="by_title", + max_characters=9, + combine_text_under_n_chars=0, + ) + chunks = chunk_by_title(elements, max_characters=9, combine_text_under_n_chars=0) + assert chunk_elements != elements + assert chunk_elements == chunks diff --git a/test_unstructured/partition/docx/test_docx.py b/test_unstructured/partition/docx/test_docx.py index 9c82c9d471..c622c390b4 100644 --- a/test_unstructured/partition/docx/test_docx.py +++ b/test_unstructured/partition/docx/test_docx.py @@ -18,6 +18,7 @@ ListItem, NarrativeText, Table, + TableChunk, Text, Title, ) @@ -422,14 +423,6 @@ def test_partition_docx_with_json(mock_document, expected_elements, tmpdir): assert elements[i] == test_elements[i] -def test_add_chunking_strategy_on_partition_docx(filename="example-docs/handbook-1p.docx"): - chunk_elements = partition_docx(filename, chunking_strategy="by_title") - elements = partition_docx(filename) - chunks = chunk_by_title(elements) - assert chunk_elements != elements - assert chunk_elements == chunks - - def test_parse_category_depth_by_style(): partitioner = _DocxPartitioner("example-docs/category-level.docx", None, None, False, None) @@ -489,3 +482,37 @@ def test_parse_category_depth_by_style_name(): def test_parse_category_depth_by_style_ilvl(): partitioner = _DocxPartitioner(None, None, None, False, None) assert partitioner._parse_category_depth_by_style_ilvl() == 0 + + +def test_add_chunking_strategy_on_partition_docx_default_args( + filename="example-docs/handbook-1p.docx", +): + chunk_elements = partition_docx(filename, chunking_strategy="by_title") + elements = partition_docx(filename) + chunks = chunk_by_title(elements) + + assert chunk_elements != elements + assert chunk_elements == chunks + + +def test_add_chunking_strategy_on_partition_docx( + filename="example-docs/fake-doc-emphasized-text.docx", +): + chunk_elements = partition_docx( + filename, + chunking_strategy="by_title", + max_characters=9, + combine_text_under_n_chars=5, + ) + elements = partition_docx(filename) + chunks = chunk_by_title(elements, max_characters=9, combine_text_under_n_chars=5) + # remove the last element of the TableChunk list because it will be the leftover slice + # and not necessarily the max_characters len + table_chunks = [chunk for chunk in chunks if isinstance(chunk, TableChunk)][:-1] + other_chunks = [chunk for chunk in chunks if not isinstance(chunk, TableChunk)] + for table_chunk in table_chunks: + assert len(table_chunk.text) == 9 + for chunk in other_chunks: + assert len(chunk.text) >= 5 + assert chunk_elements != elements + assert chunk_elements == chunks diff --git a/test_unstructured/partition/epub/test_epub.py b/test_unstructured/partition/epub/test_epub.py index 7d0e741899..991ec1991f 100644 --- a/test_unstructured/partition/epub/test_epub.py +++ b/test_unstructured/partition/epub/test_epub.py @@ -193,3 +193,24 @@ def test_add_chunking_strategy_on_partition_epub( chunks = chunk_by_title(elements) assert chunk_elements != elements assert chunk_elements == chunks + + +def test_add_chunking_strategy_on_partition_epub_non_default( + filename=os.path.join(DIRECTORY, "..", "..", "..", "example-docs", "winter-sports.epub"), +): + elements = partition_epub(filename=filename) + chunk_elements = partition_epub( + filename, + chunking_strategy="by_title", + max_characters=5, + new_after_n_chars=5, + combine_text_under_n_chars=0, + ) + chunks = chunk_by_title( + elements, + max_characters=5, + new_after_n_chars=5, + combine_text_under_n_chars=0, + ) + assert chunk_elements != elements + assert chunk_elements == chunks diff --git a/test_unstructured/partition/markdown/test_md.py b/test_unstructured/partition/markdown/test_md.py index 33d131b7a3..c73247998c 100644 --- a/test_unstructured/partition/markdown/test_md.py +++ b/test_unstructured/partition/markdown/test_md.py @@ -276,7 +276,7 @@ def test_partition_md_with_json( assert elements[i] == test_elements[i] -def test_add_chunking_strategy_on_partition_md( +def test_add_chunking_strategy_by_title_on_partition_md( filename="example-docs/README.md", ): elements = partition_md(filename=filename) diff --git a/test_unstructured/partition/msg/test_msg.py b/test_unstructured/partition/msg/test_msg.py index 6e179987a2..7678a6cda5 100644 --- a/test_unstructured/partition/msg/test_msg.py +++ b/test_unstructured/partition/msg/test_msg.py @@ -285,7 +285,7 @@ def test_partition_msg_with_pgp_encrypted_message( assert "Encrypted email detected" in caplog.text -def test_add_chunking_strategy_on_partition_msg( +def test_add_chunking_strategy_by_title_on_partition_msg( filename=os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake-email.msg"), ): elements = partition_msg(filename=filename) diff --git a/test_unstructured/partition/odt/test_odt.py b/test_unstructured/partition/odt/test_odt.py index 9fe9b4b99d..982a11f9b4 100644 --- a/test_unstructured/partition/odt/test_odt.py +++ b/test_unstructured/partition/odt/test_odt.py @@ -2,7 +2,7 @@ import pathlib from unstructured.chunking.title import chunk_by_title -from unstructured.documents.elements import Table, Title +from unstructured.documents.elements import Table, TableChunk, Title from unstructured.partition.json import partition_json from unstructured.partition.odt import partition_odt from unstructured.staging.base import elements_to_json @@ -169,3 +169,24 @@ def test_add_chunking_strategy_on_partition_odt( chunks = chunk_by_title(elements) assert chunk_elements != elements assert chunk_elements == chunks + + +def test_add_chunking_strategy_on_partition_odt_non_default(): + filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake.odt") + elements = partition_odt(filename=filename) + chunk_elements = partition_odt( + filename, + chunking_strategy="by_title", + max_characters=7, + combine_text_under_n_chars=5, + ) + chunks = chunk_by_title( + elements, + max_characters=7, + combine_text_under_n_chars=5, + ) + for chunk in chunk_elements: + if isinstance(chunk, TableChunk): + assert len(chunk.text) <= 7 + assert chunk_elements != elements + assert chunk_elements == chunks diff --git a/test_unstructured/partition/pdf-image/test_image.py b/test_unstructured/partition/pdf-image/test_image.py index 9752f61887..4d8f262f06 100644 --- a/test_unstructured/partition/pdf-image/test_image.py +++ b/test_unstructured/partition/pdf-image/test_image.py @@ -462,6 +462,25 @@ def test_add_chunking_strategy_on_partition_image( assert chunk_elements == chunks +def test_add_chunking_strategy_on_partition_image_hi_res( + filename="example-docs/layout-parser-paper-with-table.jpg", +): + elements = image.partition_image( + filename=filename, + strategy="hi_res", + infer_table_structure=True, + ) + chunk_elements = image.partition_image( + filename, + strategy="hi_res", + infer_table_structure=True, + chunking_strategy="by_title", + ) + chunks = chunk_by_title(elements) + assert chunk_elements != elements + assert chunk_elements == chunks + + def test_partition_image_uses_model_name(): with mock.patch.object( pdf, diff --git a/test_unstructured/partition/pdf-image/test_pdf.py b/test_unstructured/partition/pdf-image/test_pdf.py index 212b4ff28b..8dfb9df5e9 100644 --- a/test_unstructured/partition/pdf-image/test_pdf.py +++ b/test_unstructured/partition/pdf-image/test_pdf.py @@ -837,7 +837,7 @@ def test_partition_pdf_with_ocr_coordinates_are_not_nan_from_file( assert point[1] is not math.nan -def test_add_chunking_strategy_on_partition_pdf( +def test_add_chunking_strategy_by_title_on_partition_pdf( filename="example-docs/layout-parser-paper-fast.pdf", ): elements = pdf.partition_pdf(filename=filename) @@ -906,7 +906,7 @@ def test_combine_numbered_list(filename): "filename", ["example-docs/layout-parser-paper-fast.pdf"], ) -def test_hyperlinks(filename): +def test_partition_pdf_hyperlinks(filename): elements = pdf.partition_pdf(filename=filename, strategy="auto") links = [ { @@ -932,7 +932,7 @@ def test_hyperlinks(filename): "filename", ["example-docs/embedded-link.pdf"], ) -def test_hyperlinks_multiple_lines(filename): +def test_partition_pdf_hyperlinks_multiple_lines(filename): elements = pdf.partition_pdf(filename=filename, strategy="auto") assert elements[-1].metadata.links[-1]["text"] == "capturing" assert len(elements[-1].metadata.links) == 2 @@ -952,3 +952,13 @@ def test_partition_pdf_uses_model_name(): mockpartition.assert_called_once() assert "model_name" in mockpartition.call_args.kwargs assert mockpartition.call_args.kwargs["model_name"] + + +def test_partition_pdf_word_bbox_not_char( + filename="example-docs/interface-config-guide-p93.pdf", +): + try: + elements = pdf.partition_pdf(filename=filename) + except Exception as e: + raise ("Partitioning fail: %s" % e) + assert len(elements) == 17 diff --git a/test_unstructured/partition/pptx/test_ppt.py b/test_unstructured/partition/pptx/test_ppt.py index 1662002ddd..3750e0e9c6 100644 --- a/test_unstructured/partition/pptx/test_ppt.py +++ b/test_unstructured/partition/pptx/test_ppt.py @@ -174,7 +174,7 @@ def test_partition_ppt_with_json( assert elements[i] == test_elements[i] -def test_add_chunking_strategy_on_partition_ppt( +def test_add_chunking_strategy_by_title_on_partition_ppt( filename=os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake-power-point.ppt"), ): elements = partition_ppt(filename=filename) diff --git a/test_unstructured/partition/pptx/test_pptx.py b/test_unstructured/partition/pptx/test_pptx.py index 3540c020e7..37e9b7ce3e 100644 --- a/test_unstructured/partition/pptx/test_pptx.py +++ b/test_unstructured/partition/pptx/test_pptx.py @@ -371,8 +371,8 @@ def test_partition_pptx_with_json(): assert elements[i] == test_elements[i] -def test_add_chunking_strategy_on_partition_pptx(): - filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake-power-point.pptx") +def test_add_chunking_strategy_by_title_on_partition_pptx(): + filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "science-exploration-1p.pptx") elements = partition_pptx(filename=filename) chunk_elements = partition_pptx(filename, chunking_strategy="by_title") chunks = chunk_by_title(elements) diff --git a/test_unstructured/partition/pypandoc/test_org.py b/test_unstructured/partition/pypandoc/test_org.py index 9017c5e86f..81ad6d4ed2 100644 --- a/test_unstructured/partition/pypandoc/test_org.py +++ b/test_unstructured/partition/pypandoc/test_org.py @@ -136,7 +136,7 @@ def test_partition_org_with_json(filename="example-docs/README.org"): assert elements[i] == test_elements[i] -def test_add_chunking_strategy_on_partition_org( +def test_add_chunking_strategy_by_title_on_partition_org( filename="example-docs/README.org", ): elements = partition_org(filename=filename) diff --git a/test_unstructured/partition/test_auto.py b/test_unstructured/partition/test_auto.py index 1e8156c3b4..438c761185 100644 --- a/test_unstructured/partition/test_auto.py +++ b/test_unstructured/partition/test_auto.py @@ -17,6 +17,7 @@ ListItem, NarrativeText, Table, + TableChunk, Text, Title, ) @@ -942,37 +943,45 @@ def test_get_partition_with_extras_prompts_for_install_if_missing(): def test_add_chunking_strategy_on_partition_auto(): filename = "example-docs/example-10k-1p.html" - chunk_elements = partition(filename, chunking_strategy="by_title") elements = partition(filename) + chunk_elements = partition(filename, chunking_strategy="by_title") chunks = chunk_by_title(elements) assert chunk_elements != elements assert chunk_elements == chunks -def test_add_chunking_strategy_on_partition_auto_respects_multipage(): +def test_add_chunking_strategy_title_on_partition_auto_respects_multipage(): filename = "example-docs/example-10k-1p.html" partitioned_elements_multipage_false_combine_chars_0 = partition( filename, chunking_strategy="by_title", multipage_sections=False, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) partitioned_elements_multipage_true_combine_chars_0 = partition( filename, chunking_strategy="by_title", multipage_sections=True, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) elements = partition(filename) cleaned_elements_multipage_false_combine_chars_0 = chunk_by_title( elements, multipage_sections=False, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) cleaned_elements_multipage_true_combine_chars_0 = chunk_by_title( elements, multipage_sections=True, - combine_under_n_chars=0, + combine_text_under_n_chars=0, + new_after_n_chars=300, + max_characters=400, ) assert ( partitioned_elements_multipage_false_combine_chars_0 @@ -985,3 +994,69 @@ def test_add_chunking_strategy_on_partition_auto_respects_multipage(): assert len(partitioned_elements_multipage_true_combine_chars_0) != len( partitioned_elements_multipage_false_combine_chars_0, ) + + +def test_add_chunking_strategy_on_partition_auto_respects_max_chars(): + filename = "example-docs/example-10k-1p.html" + + # default chunk size in chars is 200 + partitioned_table_elements_200_chars = [ + e + for e in partition( + filename, + chunking_strategy="by_title", + max_characters=200, + combine_text_under_n_chars=5, + ) + if isinstance(e, (Table, TableChunk)) + ] + + partitioned_table_elements_5_chars = [ + e + for e in partition( + filename, + chunking_strategy="by_title", + max_characters=5, + combine_text_under_n_chars=5, + ) + if isinstance(e, (Table, TableChunk)) + ] + + elements = partition(filename) + + table_elements = [e for e in elements if isinstance(e, Table)] + + assert len(partitioned_table_elements_5_chars) != len(table_elements) + assert len(partitioned_table_elements_200_chars) != len(table_elements) + + assert len(partitioned_table_elements_5_chars[0].text) == 5 + assert len(partitioned_table_elements_5_chars[0].metadata.text_as_html) == 5 + + # the first table element is under 200 chars so doesn't get chunked! + assert table_elements[0] == partitioned_table_elements_200_chars[0] + assert len(partitioned_table_elements_200_chars[0].text) < 200 + assert len(partitioned_table_elements_200_chars[1].text) == 200 + assert len(partitioned_table_elements_200_chars[1].metadata.text_as_html) == 200 + + +def test_add_chunking_strategy_chars_on_partition_auto_adds_is_continuation(): + filename = "example-docs/example-10k-1p.html" + + # default chunk size in chars is 200 + partitioned_table_elements_200_chars = [ + e + for e in partition( + filename, + chunking_strategy="by_num_characters", + ) + if isinstance(e, Table) + ] + + i = 0 + for table in partitioned_table_elements_200_chars: + # have to reset the counter to 0 here when we encounter a Table element + if isinstance(table, Table): + i = 0 + if i > 0 and isinstance(table, TableChunk): + assert table.metadata.is_continuation is True + i += 1 diff --git a/test_unstructured/partition/test_html_partition.py b/test_unstructured/partition/test_html_partition.py index 48934f73f4..82976a2621 100644 --- a/test_unstructured/partition/test_html_partition.py +++ b/test_unstructured/partition/test_html_partition.py @@ -8,7 +8,7 @@ from unstructured.chunking.title import chunk_by_title from unstructured.cleaners.core import clean_extra_whitespace -from unstructured.documents.elements import ListItem, NarrativeText, Table, Title +from unstructured.documents.elements import EmailAddress, ListItem, NarrativeText, Table, Title from unstructured.documents.html import HTMLTitle from unstructured.partition.html import partition_html from unstructured.partition.json import partition_json @@ -645,3 +645,25 @@ def test_add_chunking_strategy_on_partition_html( chunks = chunk_by_title(elements) assert chunk_elements != elements assert chunk_elements == chunks + + +def test_html_heading_title_detection(): + html_text = """ +

This is a section of narrative text, it's long, flows and has meaning

+

This is a section of narrative text, it's long, flows and has meaning

+

A heading that is at the second level

+

Finally, the third heading

+

December 1-17, 2017

+

email@example.com

+

  • - bulleted item

    • + """ + elements = partition_html(text=html_text) + assert elements == [ + NarrativeText("This is a section of narrative text, it's long, flows and has meaning"), + Title("This is a section of narrative text, it's long, flows and has meaning"), + Title("A heading that is at the second level"), + Title("Finally, the third heading"), + Title("December 1-17, 2017"), + EmailAddress("email@example.com"), + ListItem("- bulleted item"), + ] diff --git a/test_unstructured_ingest/expected-structured-output/s3-minio/wiki_movie_plots_small.csv.json b/test_unstructured_ingest/expected-structured-output/s3-minio/wiki_movie_plots_small.csv.json new file mode 100644 index 0000000000..0a44c84aba --- /dev/null +++ b/test_unstructured_ingest/expected-structured-output/s3-minio/wiki_movie_plots_small.csv.json @@ -0,0 +1,19 @@ +[ + { + "type": "Table", + "element_id": "f078b58f281b4e231430e34a3ece07f3", + "metadata": { + "data_source": { + "url": "s3://utic-dev-tech-fixtures/wiki_movie_plots_small.csv", + "version": 103589111396252091980300895568390462924, + "record_locator": { + "protocol": "s3", + "remote_file_path": "utic-dev-tech-fixtures/wiki_movie_plots_small.csv" + } + }, + "filetype": "text/csv", + "text_as_html": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
    1901Kansas Saloon SmashersAmericanUnknownunknownhttps://en.wikipedia.org/wiki/Kansas_Saloon_SmashersA bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]
    1901Love by the Light of the MoonAmericanUnknownunknownhttps://en.wikipedia.org/wiki/Love_by_the_Light_of_the_MoonThe moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better.
    1901The Martyred PresidentsAmericanUnknownunknownhttps://en.wikipedia.org/wiki/The_Martyred_PresidentsThe film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination.\\r\\nIn the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice.
    1901Terrible Teddy, the Grizzly KingAmericanUnknownunknownhttps://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_KingLasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading \"His Photographer\" and \"His Press Agent\" respectively, follow him into the shot; the photographer sets up his camera. \"Teddy\" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. \"Teddy\" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. \"Teddy\" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs.
    1902Jack and the BeanstalkAmericanGeorge S. Fleming, Edwin S. Porterunknownhttps://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film)The earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince.
    1903Alice in WonderlandAmericanCecil HepworthMay Clarkunknownhttps://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film)Alice follows a large white rabbit down a \"Rabbit-hole\". She finds a tiny door. When she finds a bottle labeled \"Drink me\", she does, and shrinks, but not enough to pass through the door. She then eats something labeled \"Eat me\" and grows larger. She finds a fan when enables her to shrink enough to get into the \"Garden\" and try to get a \"Dog\" to play with her. She enters the \"White Rabbit's tiny House,\" but suddenly resumes her normal size. In order to get out, she has to use the \"magic fan.\"\\r\\nShe enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. \"The Duchess's Cheshire Cat\" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's \"Mad Tea-Party.\" After a while, she leaves.\\r\\nThe Queen invites Alice to join the \"ROYAL PROCESSION\": a parade of marching playing cards and others headed by the White Rabbit. When Alice \"unintentionally offends the Queen\", the latter summons the \"Executioner\". Alice \"boxes the ears\", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream.
    1903The Great Train RobberyAmericanEdwin S. Porterwesternhttps://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film)The film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left.\\r\\nMeanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water.\\r\\nThere is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail.
    1904The SuburbaniteAmericanWallace McCutcheoncomedyhttps://en.wikipedia.org/wiki/The_SuburbaniteThe film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest.
    1905The Little Train RobberyAmericanEdwin Stanton Porterunknownhttps://en.wikipedia.org/wiki/The_Little_Train_RobberyThe opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The \"Bandit Queen,\" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The \"Bandit Queen\" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin.\\r\\nThe next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the \"valuables,\" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies.\\r\\nIn the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the \"plunder.\" The police, however, have struck the right trail and are in close pursuit. While the \"plunder\" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the \"Bandit Queen.\"
    1905The Night Before ChristmasAmericanEdwin Stanton Porterunknownhttps://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film)Scenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents.
    1906Dream of a Rarebit FiendAmericanWallace McCutcheon and Edwin S. Portershorthttps://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film)The Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed.
    1906From Leadville to Aspen: A Hold-Up in the RockiesAmericanFrancis J. Marion and Wallace McCutcheonshort action/crime westernhttps://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_RockiesThe film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart.
    1906Kathleen MavourneenAmericanEdwin S. Portershort filmhttps://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film)Irish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration.\\r\\nFilm historian Charles Musser writes of Porter's adaptation, \"O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.\"[1]
    1907Daniel BooneAmericanWallace McCutcheon and Ediwin S. PorterWilliam Craven, Florence Lawrencebiographicalhttps://en.wikipedia.org/wiki/Daniel_Boone_(1907_film)Boone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]
    1907How Brown Saw the Baseball GameAmericanUnknownUnknowncomedyhttps://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_GameBefore heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]
    1907Laughing GasAmericanEdwin Stanton PorterBertha Regustus, Edward Bouldencomedyhttps://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_FilmThe plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets \"catches\" the laughter from her, including a vendor and police officers.
    1908The Adventures of DollieAmericanD. W. GriffithArthur V. Johnson, Linda Arvidsondramahttps://en.wikipedia.org/wiki/The_Adventures_of_DollieOn a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents.
    1908The Black ViperAmericanD. W. GriffithD. W. Griffithdramahttps://en.wikipedia.org/wiki/The_Black_ViperA thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house.
    1908A Calamitous ElopementAmericanD.W. GriffithHarry Solter, Linda Arvidsoncomedyhttps://en.wikipedia.org/wiki/A_Calamitous_ElopementA young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings.
    1908The Call of the WildAmericanD. W. GriffithCharles Insleeadventurehttps://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film)A white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as \"The Biograph Girl.\"
    1908A Christmas CarolAmericanUnknownTom Rickettsdramahttps://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film)No prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life.
    1908The Fight for FreedomAmericanD. W. GriffithFlorence Auer, John G. Adolfiwesternhttps://en.wikipedia.org/wiki/The_Fight_for_FreedomThe film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town.
    " + }, + "text": "\n\n\n1901\nKansas Saloon Smashers\nAmerican\nUnknown\n\nunknown\nhttps://en.wikipedia.org/wiki/Kansas_Saloon_Smashers\nA bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]\n\n\n1901\nLove by the Light of the Moon\nAmerican\nUnknown\n\nunknown\nhttps://en.wikipedia.org/wiki/Love_by_the_Light_of_the_Moon\nThe moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better.\n\n\n1901\nThe Martyred Presidents\nAmerican\nUnknown\n\nunknown\nhttps://en.wikipedia.org/wiki/The_Martyred_Presidents\nThe film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination.\\r\\nIn the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice.\n\n\n1901\nTerrible Teddy, the Grizzly King\nAmerican\nUnknown\n\nunknown\nhttps://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_King\nLasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading \"His Photographer\" and \"His Press Agent\" respectively, follow him into the shot; the photographer sets up his camera. \"Teddy\" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. \"Teddy\" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. \"Teddy\" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs.\n\n\n1902\nJack and the Beanstalk\nAmerican\nGeorge S. Fleming, Edwin S. Porter\n\nunknown\nhttps://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film)\nThe earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince.\n\n\n1903\nAlice in Wonderland\nAmerican\nCecil Hepworth\nMay Clark\nunknown\nhttps://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film)\nAlice follows a large white rabbit down a \"Rabbit-hole\". She finds a tiny door. When she finds a bottle labeled \"Drink me\", she does, and shrinks, but not enough to pass through the door. She then eats something labeled \"Eat me\" and grows larger. She finds a fan when enables her to shrink enough to get into the \"Garden\" and try to get a \"Dog\" to play with her. She enters the \"White Rabbit's tiny House,\" but suddenly resumes her normal size. In order to get out, she has to use the \"magic fan.\"\\r\\nShe enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. \"The Duchess's Cheshire Cat\" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's \"Mad Tea-Party.\" After a while, she leaves.\\r\\nThe Queen invites Alice to join the \"ROYAL PROCESSION\": a parade of marching playing cards and others headed by the White Rabbit. When Alice \"unintentionally offends the Queen\", the latter summons the \"Executioner\". Alice \"boxes the ears\", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream.\n\n\n1903\nThe Great Train Robbery\nAmerican\nEdwin S. Porter\n\nwestern\nhttps://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film)\nThe film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left.\\r\\nMeanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water.\\r\\nThere is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail.\n\n\n1904\nThe Suburbanite\nAmerican\nWallace McCutcheon\n\ncomedy\nhttps://en.wikipedia.org/wiki/The_Suburbanite\nThe film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest.\n\n\n1905\nThe Little Train Robbery\nAmerican\nEdwin Stanton Porter\n\nunknown\nhttps://en.wikipedia.org/wiki/The_Little_Train_Robbery\nThe opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The \"Bandit Queen,\" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The \"Bandit Queen\" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin.\\r\\nThe next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the \"valuables,\" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies.\\r\\nIn the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the \"plunder.\" The police, however, have struck the right trail and are in close pursuit. While the \"plunder\" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the \"Bandit Queen.\"\n\n\n1905\nThe Night Before Christmas\nAmerican\nEdwin Stanton Porter\n\nunknown\nhttps://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film)\nScenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents.\n\n\n1906\nDream of a Rarebit Fiend\nAmerican\nWallace McCutcheon and Edwin S. Porter\n\nshort\nhttps://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film)\nThe Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed.\n\n\n1906\nFrom Leadville to Aspen: A Hold-Up in the Rockies\nAmerican\nFrancis J. Marion and Wallace McCutcheon\n\nshort action/crime western\nhttps://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_Rockies\nThe film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart.\n\n\n1906\nKathleen Mavourneen\nAmerican\nEdwin S. Porter\n\nshort film\nhttps://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film)\nIrish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration.\\r\\nFilm historian Charles Musser writes of Porter's adaptation, \"O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.\"[1]\n\n\n1907\nDaniel Boone\nAmerican\nWallace McCutcheon and Ediwin S. Porter\nWilliam Craven, Florence Lawrence\nbiographical\nhttps://en.wikipedia.org/wiki/Daniel_Boone_(1907_film)\nBoone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]\n\n\n1907\nHow Brown Saw the Baseball Game\nAmerican\nUnknown\nUnknown\ncomedy\nhttps://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_Game\nBefore heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]\n\n\n1907\nLaughing Gas\nAmerican\nEdwin Stanton Porter\nBertha Regustus, Edward Boulden\ncomedy\nhttps://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_Film\nThe plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets \"catches\" the laughter from her, including a vendor and police officers.\n\n\n1908\nThe Adventures of Dollie\nAmerican\nD. W. Griffith\nArthur V. Johnson, Linda Arvidson\ndrama\nhttps://en.wikipedia.org/wiki/The_Adventures_of_Dollie\nOn a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents.\n\n\n1908\nThe Black Viper\nAmerican\nD. W. Griffith\nD. W. Griffith\ndrama\nhttps://en.wikipedia.org/wiki/The_Black_Viper\nA thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house.\n\n\n1908\nA Calamitous Elopement\nAmerican\nD.W. Griffith\nHarry Solter, Linda Arvidson\ncomedy\nhttps://en.wikipedia.org/wiki/A_Calamitous_Elopement\nA young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings.\n\n\n1908\nThe Call of the Wild\nAmerican\nD. W. Griffith\nCharles Inslee\nadventure\nhttps://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film)\nA white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as \"The Biograph Girl.\"\n\n\n1908\nA Christmas Carol\nAmerican\nUnknown\nTom Ricketts\ndrama\nhttps://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film)\nNo prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life.\n\n\n1908\nThe Fight for Freedom\nAmerican\nD. W. Griffith\nFlorence Auer, John G. Adolfi\nwestern\nhttps://en.wikipedia.org/wiki/The_Fight_for_Freedom\nThe film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town.\n\n\n" + } +] \ No newline at end of file diff --git a/test_unstructured_ingest/test-ingest-elasticsearch.sh b/test_unstructured_ingest/test-ingest-elasticsearch.sh index 530ddf1bed..7b181f90ba 100755 --- a/test_unstructured_ingest/test-ingest-elasticsearch.sh +++ b/test_unstructured_ingest/test-ingest-elasticsearch.sh @@ -16,10 +16,8 @@ source "$SCRIPT_DIR"/cleanup.sh function cleanup() { # Kill the container so the script can be repeatedly run using the same ports - if docker ps --filter "name=es-test"; then - echo "Stopping Elasticsearch Docker container" - docker stop es-test - fi + echo "Stopping Elasticsearch Docker container" + docker-compose -f scripts/elasticsearch-test-helpers/docker-compose.yaml down --remove-orphans -v cleanup_dir "$OUTPUT_DIR" if [ "$CI" == "true" ]; then diff --git a/test_unstructured_ingest/test-ingest-s3-minio.sh b/test_unstructured_ingest/test-ingest-s3-minio.sh new file mode 100755 index 0000000000..000c28e28b --- /dev/null +++ b/test_unstructured_ingest/test-ingest-s3-minio.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e + + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +cd "$SCRIPT_DIR"/.. || exit 1 +OUTPUT_FOLDER_NAME=s3-minio +OUTPUT_DIR=$SCRIPT_DIR/structured-output/$OUTPUT_FOLDER_NAME +DOWNLOAD_DIR=$SCRIPT_DIR/download/$OUTPUT_FOLDER_NAME +max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")} +secret_key=minioadmin +access_key=minioadmin + +# shellcheck disable=SC1091 +source "$SCRIPT_DIR"/cleanup.sh + +function cleanup() { + # Kill the container so the script can be repeatedly run using the same ports + echo "Stopping Minio Docker container" + docker-compose -f scripts/minio-test-helpers/docker-compose.yaml down --remove-orphans -v + + cleanup_dir "$OUTPUT_DIR" +} + +trap cleanup EXIT + +# shellcheck source=/dev/null +scripts/minio-test-helpers/create-and-check-minio.sh +wait + +AWS_SECRET_ACCESS_KEY=$secret_key AWS_ACCESS_KEY_ID=$access_key PYTHONPATH=. ./unstructured/ingest/main.py \ + s3 \ + --num-processes "$max_processes" \ + --download-dir "$DOWNLOAD_DIR" \ + --metadata-exclude coordinates,filename,file_directory,metadata.data_source.date_processed,metadata.data_source.date_modified,metadata.last_modified,metadata.detection_class_prob,metadata.parent_id,metadata.category_depth \ + --strategy hi_res \ + --preserve-downloads \ + --reprocess \ + --output-dir "$OUTPUT_DIR" \ + --verbose \ + --remote-url s3://utic-dev-tech-fixtures/ \ + --endpoint-url http://localhost:9000 + + +"$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME diff --git a/test_unstructured_ingest/test-ingest.sh b/test_unstructured_ingest/test-ingest.sh index 56568b37f0..8c2dffc977 100755 --- a/test_unstructured_ingest/test-ingest.sh +++ b/test_unstructured_ingest/test-ingest.sh @@ -10,6 +10,7 @@ export OMP_THREAD_LIMIT=1 scripts=( 'test-ingest-s3.sh' +'test-ingest-s3-minio.sh' 'test-ingest-azure.sh' 'test-ingest-biomed-api.sh' 'test-ingest-biomed-path.sh' diff --git a/unstructured/__version__.py b/unstructured/__version__.py index ea97a53bc8..d71d465e92 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.10.19-dev5" # pragma: no cover +__version__ = "0.10.19-dev9" # pragma: no cover diff --git a/unstructured/chunking/title.py b/unstructured/chunking/title.py index 8fd38d62f3..0c5bde799c 100644 --- a/unstructured/chunking/title.py +++ b/unstructured/chunking/title.py @@ -1,6 +1,7 @@ +import copy import functools import inspect -from typing import Any, Callable, Dict, List, TypeVar +from typing import Any, Callable, Dict, List, Optional, TypeVar, Union from typing_extensions import ParamSpec @@ -9,96 +10,130 @@ Element, ElementMetadata, Table, + TableChunk, Text, Title, ) +def chunk_table_element( + element: Table, + max_characters: Optional[int] = 500, +) -> List[Union[Table, TableChunk]]: + text = element.text + html = getattr(element, "text_as_html", None) + + if len(text) <= max_characters and ( # type: ignore + html is None or len(html) <= max_characters # type: ignore + ): + return [element] + + chunks: List[Union[Table, TableChunk]] = [] + metadata = copy.copy(element.metadata) + is_continuation = False + + while text or html: + text_chunk, text = text[:max_characters], text[max_characters:] + table_chunk = TableChunk(text=text_chunk, metadata=copy.copy(metadata)) + + if html: + html_chunk, html = html[:max_characters], html[max_characters:] + table_chunk.metadata.text_as_html = html_chunk + + if is_continuation: + table_chunk.metadata.is_continuation = True + + chunks.append(table_chunk) + is_continuation = True + + return chunks + + def chunk_by_title( elements: List[Element], multipage_sections: bool = True, - combine_under_n_chars: int = 500, - new_after_n_chars: int = 1500, + combine_text_under_n_chars: int = 500, + new_after_n_chars: int = 500, + max_characters: int = 500, ) -> List[Element]: """Uses title elements to identify sections within the document for chunking. Splits off into a new section when a title is detected or if metadata changes, which happens when page numbers or sections change. Cuts off sections once they have exceeded - a character length of new_after_n_chars. + a character length of max_characters. Parameters ---------- elements - A list of unstructured elements. Usually the ouput of a partition functions. + A list of unstructured elements. Usually the output of a partition functions. multipage_sections If True, sections can span multiple pages. Defaults to True. - combine_under_n_chars + combine_text_under_n_chars Combines elements (for example a series of titles) until a section reaches a length of n characters. new_after_n_chars - Cuts off new sections once they reach a length of n characters + Cuts off new sections once they reach a length of n characters (soft max) + max_characters + Chunks table elements text and text_as_html into chunks of length n characters (hard max) + TODO: (amanda) extend to other elements """ if ( - combine_under_n_chars is not None + combine_text_under_n_chars is not None and new_after_n_chars is not None + and max_characters is not None and ( - combine_under_n_chars > new_after_n_chars - or combine_under_n_chars < 0 + combine_text_under_n_chars > new_after_n_chars + or combine_text_under_n_chars < 0 or new_after_n_chars < 0 + or max_characters <= 0 + or combine_text_under_n_chars > max_characters ) ): raise ValueError( - "Invalid values for combine_under_n_chars and/or new_after_n_chars.", + "Invalid values for combine_text_under_n_chars and/or max_characters.", ) chunked_elements: List[Element] = [] sections = _split_elements_by_title_and_table( elements, multipage_sections=multipage_sections, - combine_under_n_chars=combine_under_n_chars, + combine_text_under_n_chars=combine_text_under_n_chars, new_after_n_chars=new_after_n_chars, ) - for section in sections: if not section: continue - if not isinstance(section[0], Text) or isinstance(section[0], Table): - chunked_elements.extend(section) - elif isinstance(section[0], Text): - text = "" - metadata = section[0].metadata + first_element = section[0] - for i, element in enumerate(section): - if isinstance(element, Text): - text += "\n\n" if text else "" - start_char = len(text) - text += element.text + if not isinstance(first_element, Text): + chunked_elements.extend(section) + continue - for attr, value in vars(element.metadata).items(): - if not isinstance(value, list): - continue + elif isinstance(first_element, Table): + chunked_elements.extend(chunk_table_element(first_element, max_characters)) + continue - _value = getattr(metadata, attr, []) - if _value is None: - _value = [] + text = "" + metadata = first_element.metadata + start_char = 0 + for element in section: + if isinstance(element, Text): + text += "\n\n" if text else "" + start_char = len(text) + text += element.text + for attr, value in vars(element.metadata).items(): + if isinstance(value, list): + _value = getattr(metadata, attr, []) or [] if attr == "regex_metadata": for item in value: item["start"] += start_char item["end"] += start_char - if i > 0: - # NOTE(newelh): Previously, _value was extended with value. - # This caused a memory error if the content was a list of strings - # with a large number of elements -- doubling the list size each time. - # This now instead ensures that the _value list is unique and updated. - for item in value: - if item not in _value: - _value.append(item) - - setattr(metadata, attr, _value) + _value.extend(item for item in value if item not in _value) + setattr(metadata, attr, _value) - chunked_elements.append(CompositeElement(text=text, metadata=metadata)) + chunked_elements.append(CompositeElement(text=text, metadata=metadata)) return chunked_elements @@ -106,8 +141,8 @@ def chunk_by_title( def _split_elements_by_title_and_table( elements: List[Element], multipage_sections: bool = True, - combine_under_n_chars: int = 500, - new_after_n_chars: int = 1500, + combine_text_under_n_chars: int = 500, + new_after_n_chars: int = 500, ) -> List[List[Element]]: sections: List[List[Element]] = [] section: List[Element] = [] @@ -123,11 +158,11 @@ def _split_elements_by_title_and_table( ) section_length = sum([len(str(element)) for element in section]) - new_section = (isinstance(element, Title) and section_length > combine_under_n_chars) or ( - not metadata_matches or section_length > new_after_n_chars - ) + new_section = ( + isinstance(element, Title) and section_length > combine_text_under_n_chars + ) or (not metadata_matches or section_length > new_after_n_chars) - if isinstance(element, Table) or not isinstance(element, Text): + if not isinstance(element, Text) or isinstance(element, Table): sections.append(section) sections.append([element]) section = [] @@ -185,7 +220,7 @@ def add_chunking_strategy() -> Callable[[Callable[_P, List[Element]]], Callable[ """Decorator for chuncking text. Uses title elements to identify sections within the document for chunking. Splits off a new section when a title is detected or if metadata changes, which happens when page numbers or sections change. Cuts off sections once they have exceeded - a character length of new_after_n_chars.""" + a character length of max_characters.""" def decorator(func: Callable[_P, List[Element]]) -> Callable[_P, List[Element]]: if func.__doc__ and ( @@ -199,11 +234,15 @@ def decorator(func: Callable[_P, List[Element]]) -> Callable[_P, List[Element]]: + "\n\tAdditional Parameters:" + "\n\t\tmultipage_sections" + "\n\t\t\tIf True, sections can span multiple pages. Defaults to True." - + "\n\t\tcombine_under_n_chars" + + "\n\t\tcombine_text_under_n_chars" + "\n\t\t\tCombines elements (for example a series of titles) until a section" + "\n\t\t\treaches a length of n characters." + "\n\t\tnew_after_n_chars" - + "\n\t\t\tCuts off new sections once they reach a length of n characters" + + "\n\t\t\t Cuts off new sections once they reach a length of n characters" + + "\n\t\t\t a soft max." + + "\n\t\tmax_characters" + + "\n\t\t\tChunks table elements text and text_as_html into chunks" + + "\n\t\t\tof length n characters, a hard max." ) @functools.wraps(func) @@ -218,8 +257,9 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> List[Element]: elements = chunk_by_title( elements, multipage_sections=params.get("multipage_sections", True), - combine_under_n_chars=params.get("combine_under_n_chars", 500), - new_after_n_chars=params.get("new_after_n_chars", 1500), + combine_text_under_n_chars=params.get("combine_text_under_n_chars", 500), + new_after_n_chars=params.get("new_after_n_chars", 500), + max_characters=params.get("max_characters", 500), ) return elements diff --git a/unstructured/documents/elements.py b/unstructured/documents/elements.py index f051e1b4f6..75c15a3e36 100644 --- a/unstructured/documents/elements.py +++ b/unstructured/documents/elements.py @@ -185,6 +185,10 @@ class ElementMetadata: # Metadata extracted via regex regex_metadata: Optional[Dict[str, List[RegexMetadata]]] = None + # Chunking metadata fields + num_characters: Optional[int] = None + is_continuation: Optional[bool] = None + # Detection Model Class Probabilities from Unstructured-Inference Hi-Res detection_class_prob: Optional[float] = None @@ -566,6 +570,14 @@ class Table(Text): pass +class TableChunk(Table): + """An element for capturing chunks of tables.""" + + category = "Table" + + pass + + class Header(Text): """An element for capturing document headers.""" diff --git a/unstructured/documents/html.py b/unstructured/documents/html.py index 3bfdb1e680..77afae1e4a 100644 --- a/unstructured/documents/html.py +++ b/unstructured/documents/html.py @@ -389,7 +389,7 @@ def _text_to_element( links=links, emphasized_texts=emphasized_texts, ) - elif is_possible_title(text): + elif is_heading_tag(tag) or is_possible_title(text): return HTMLTitle( text, tag=tag, @@ -417,7 +417,7 @@ def _is_container_with_text(tag_elem: etree.Element) -> bool:
    Please read my message!
    """ - if tag_elem.tag not in SECTION_TAGS or len(tag_elem) == 0: + if tag_elem.tag not in SECTION_TAGS + ["body"] or len(tag_elem) == 0: return False if tag_elem.text is None or tag_elem.text.strip() == "": @@ -431,6 +431,11 @@ def is_narrative_tag(text: str, tag: str) -> bool: return tag not in HEADING_TAGS and is_possible_narrative_text(text) +def is_heading_tag(tag: str) -> bool: + """Uses tag information to infer whether text is a heading.""" + return tag in HEADING_TAGS + + def _construct_text(tag_elem: etree.Element, include_tail_text: bool = True) -> str: """Extracts text from a text tag element.""" text = "" @@ -451,6 +456,12 @@ def _has_break_tags(tag_elem: etree._Element) -> bool: # pyright: ignore[report def _unfurl_break_tags(tag_elem: etree.Element) -> List[etree.Element]: unfurled = [] + + if tag_elem.text: + _tag_elem = etree.Element(tag_elem.tag) + _tag_elem.text = tag_elem.text + unfurled.append(_tag_elem) + children = tag_elem.getchildren() for child in children: if not _has_break_tags(child): @@ -474,13 +485,13 @@ def _is_text_tag(tag_elem: etree.Element, max_predecessor_len: int = 5) -> bool: if len(tag_elem) > max_predecessor_len + empty_elems_len: return False - if tag_elem.tag in TEXT_TAGS + HEADING_TAGS: + if tag_elem.tag in TEXT_TAGS + HEADING_TAGS + TEXTBREAK_TAGS: return True # NOTE(robinson) - This indicates that a div tag has no children. If that's the # case and the tag has text, its potential a text tag children = tag_elem.getchildren() - if tag_elem.tag in SECTION_TAGS and len(children) == 0: + if tag_elem.tag in SECTION_TAGS + ["body"] and len(children) == 0: return True if _has_adjacent_bulleted_spans(tag_elem, children): diff --git a/unstructured/ingest/cli/cmds/s3.py b/unstructured/ingest/cli/cmds/s3.py index 88c46fdb25..34a7845f1b 100644 --- a/unstructured/ingest/cli/cmds/s3.py +++ b/unstructured/ingest/cli/cmds/s3.py @@ -1,4 +1,5 @@ import logging +import typing as t from dataclasses import dataclass import click @@ -22,6 +23,7 @@ @dataclass class S3CliConfig(BaseConfig, CliMixin): anonymous: bool = False + endpoint_url: t.Optional[str] = None @staticmethod def add_cli_options(cmd: click.Command) -> None: @@ -32,6 +34,13 @@ def add_cli_options(cmd: click.Command) -> None: default=False, help="Connect to s3 without local AWS credentials.", ), + click.Option( + ["--endpoint-url"], + type=str, + default=None, + help="Use this endpoint_url, if specified. Needed for " + "connecting to non-AWS S3 buckets.", + ), ] cmd.params.extend(options) diff --git a/unstructured/ingest/interfaces.py b/unstructured/ingest/interfaces.py index c76fdfb783..caefa50afd 100644 --- a/unstructured/ingest/interfaces.py +++ b/unstructured/ingest/interfaces.py @@ -83,16 +83,16 @@ def get_embedder(self) -> BaseEmbeddingEncoder: class ChunkingConfig(BaseConfig): chunk_elements: bool = False multipage_sections: bool = True - combine_under_n_chars: int = 500 - new_after_n_chars: int = 1500 + combine_text_under_n_chars: int = 500 + max_characters: int = 1500 def chunk(self, elements: t.List[Element]) -> t.List[Element]: if self.chunk_elements: return chunk_by_title( elements=elements, multipage_sections=self.multipage_sections, - combine_under_n_chars=self.combine_under_n_chars, - new_after_n_chars=self.new_after_n_chars, + combine_text_under_n_chars=self.combine_text_under_n_chars, + max_characters=self.max_characters, ) else: return elements diff --git a/unstructured/ingest/runner/airtable.py b/unstructured/ingest/runner/airtable.py index 92f5bd735d..48fc109b6a 100644 --- a/unstructured/ingest/runner/airtable.py +++ b/unstructured/ingest/runner/airtable.py @@ -10,11 +10,11 @@ def airtable( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, personal_access_token: str, - list_of_paths: t.Optional[str], + verbose: bool = False, + list_of_paths: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/azure.py b/unstructured/ingest/runner/azure.py index 90b08e0654..58e2594b4d 100644 --- a/unstructured/ingest/runner/azure.py +++ b/unstructured/ingest/runner/azure.py @@ -9,14 +9,14 @@ def azure( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, account_name: t.Optional[str], account_key: t.Optional[str], connection_string: t.Optional[str], remote_url: str, - recursive: bool, + verbose: bool = False, + recursive: bool = False, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/biomed.py b/unstructured/ingest/runner/biomed.py index 62e6bb1671..fe23aa34ca 100644 --- a/unstructured/ingest/runner/biomed.py +++ b/unstructured/ingest/runner/biomed.py @@ -13,13 +13,13 @@ def biomed( verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, - path: t.Optional[str], - api_id: t.Optional[str], - api_from: t.Optional[str], - api_until: t.Optional[str], max_retries: int, max_request_time: int, decay: float, + path: t.Optional[str] = None, + api_id: t.Optional[str] = None, + api_from: t.Optional[str] = None, + api_until: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/box.py b/unstructured/ingest/runner/box.py index 1856f075ca..20f066dfa3 100644 --- a/unstructured/ingest/runner/box.py +++ b/unstructured/ingest/runner/box.py @@ -9,12 +9,12 @@ def box( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, remote_url: str, - recursive: bool, - box_app_config: t.Optional[str], + verbose: bool = False, + recursive: bool = False, + box_app_config: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/confluence.py b/unstructured/ingest/runner/confluence.py index 64db4233c2..5192d07dc2 100644 --- a/unstructured/ingest/runner/confluence.py +++ b/unstructured/ingest/runner/confluence.py @@ -10,7 +10,6 @@ def confluence( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, url: str, @@ -18,6 +17,7 @@ def confluence( api_token: str, max_num_of_spaces: int, max_num_of_docs_from_each_space: int, + verbose: bool = False, spaces: t.Optional[t.List[str]] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, diff --git a/unstructured/ingest/runner/delta_table.py b/unstructured/ingest/runner/delta_table.py index a547831dbd..f19a4d9c4a 100644 --- a/unstructured/ingest/runner/delta_table.py +++ b/unstructured/ingest/runner/delta_table.py @@ -11,12 +11,12 @@ def delta_table( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, table_uri: t.Union[str, Path], version: t.Optional[int] = None, storage_options: t.Optional[str] = None, + verbose: bool = False, without_files: bool = False, columns: t.Optional[t.List[str]] = None, writer_type: t.Optional[str] = None, diff --git a/unstructured/ingest/runner/discord.py b/unstructured/ingest/runner/discord.py index d5a44a5086..de1a7d4cbb 100644 --- a/unstructured/ingest/runner/discord.py +++ b/unstructured/ingest/runner/discord.py @@ -10,12 +10,12 @@ def discord( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, channels: t.List[str], token: str, - period: t.Optional[int], + verbose: bool = False, + period: t.Optional[int] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/dropbox.py b/unstructured/ingest/runner/dropbox.py index bacb3b8127..e30ab36af3 100644 --- a/unstructured/ingest/runner/dropbox.py +++ b/unstructured/ingest/runner/dropbox.py @@ -9,12 +9,12 @@ def dropbox( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, remote_url: str, - recursive: bool, - token: t.Optional[str], + verbose: bool = False, + recursive: bool = False, + token: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/elasticsearch.py b/unstructured/ingest/runner/elasticsearch.py index cd02a2f638..8c5a511576 100644 --- a/unstructured/ingest/runner/elasticsearch.py +++ b/unstructured/ingest/runner/elasticsearch.py @@ -10,12 +10,12 @@ def elasticsearch( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, url: str, index_name: str, - jq_query: t.Optional[str], + verbose: bool = False, + jq_query: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/fsspec.py b/unstructured/ingest/runner/fsspec.py index f0260af409..7822b30140 100644 --- a/unstructured/ingest/runner/fsspec.py +++ b/unstructured/ingest/runner/fsspec.py @@ -11,11 +11,11 @@ def fsspec( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, remote_url: str, - recursive: bool, + verbose: bool = False, + recursive: bool = False, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/gcs.py b/unstructured/ingest/runner/gcs.py index eab4fb4bc6..a442a28916 100644 --- a/unstructured/ingest/runner/gcs.py +++ b/unstructured/ingest/runner/gcs.py @@ -9,12 +9,12 @@ def gcs( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, remote_url: str, - recursive: bool, - token: t.Optional[str], + verbose: bool = False, + recursive: bool = False, + token: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/github.py b/unstructured/ingest/runner/github.py index 4bbf09e5aa..ff726da597 100644 --- a/unstructured/ingest/runner/github.py +++ b/unstructured/ingest/runner/github.py @@ -10,13 +10,13 @@ def github( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, url: str, git_branch: str, - git_access_token: t.Optional[str], - git_file_glob: t.Optional[str], + verbose: bool = False, + git_access_token: t.Optional[str] = None, + git_file_glob: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/gitlab.py b/unstructured/ingest/runner/gitlab.py index 4d15385a98..a4e6d9b947 100644 --- a/unstructured/ingest/runner/gitlab.py +++ b/unstructured/ingest/runner/gitlab.py @@ -10,13 +10,13 @@ def gitlab( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, url: str, git_branch: str, - git_access_token: t.Optional[str], - git_file_glob: t.Optional[str], + verbose: bool = False, + git_access_token: t.Optional[str] = None, + git_file_glob: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/google_drive.py b/unstructured/ingest/runner/google_drive.py index 2f6f437086..27ad5979bd 100644 --- a/unstructured/ingest/runner/google_drive.py +++ b/unstructured/ingest/runner/google_drive.py @@ -10,13 +10,13 @@ def gdrive( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, service_account_key: str, - recursive: bool, drive_id: str, - extension: t.Optional[str], + verbose: bool = False, + recursive: bool = False, + extension: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/jira.py b/unstructured/ingest/runner/jira.py index bcecda323b..e9875e51ee 100644 --- a/unstructured/ingest/runner/jira.py +++ b/unstructured/ingest/runner/jira.py @@ -10,15 +10,15 @@ def jira( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, url: str, user_email: str, api_token: str, - projects: t.Optional[t.List[str]], - boards: t.Optional[t.List[str]], - issues: t.Optional[t.List[str]], + verbose: bool = False, + projects: t.Optional[t.List[str]] = None, + boards: t.Optional[t.List[str]] = None, + issues: t.Optional[t.List[str]] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/local.py b/unstructured/ingest/runner/local.py index 6278079324..a52ee598ec 100644 --- a/unstructured/ingest/runner/local.py +++ b/unstructured/ingest/runner/local.py @@ -8,12 +8,12 @@ def local( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, input_path: str, - recursive: bool, - file_glob: t.Optional[str], + verbose: bool = False, + recursive: bool = False, + file_glob: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/notion.py b/unstructured/ingest/runner/notion.py index 9bd10e9b03..7aa22e9c4e 100644 --- a/unstructured/ingest/runner/notion.py +++ b/unstructured/ingest/runner/notion.py @@ -11,11 +11,11 @@ def notion( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, api_key: str, - recursive: bool, + verbose: bool = False, + recursive: bool = False, page_ids: t.Optional[t.List[str]] = None, database_ids: t.Optional[t.List[str]] = None, writer_type: t.Optional[str] = None, diff --git a/unstructured/ingest/runner/onedrive.py b/unstructured/ingest/runner/onedrive.py index 3cee6b9467..abf3d18938 100644 --- a/unstructured/ingest/runner/onedrive.py +++ b/unstructured/ingest/runner/onedrive.py @@ -10,16 +10,16 @@ def onedrive( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, tenant: str, user_pname: str, client_id: str, client_cred: str, - authority_url: t.Optional[str], - path: t.Optional[str], - recursive: bool, + verbose: bool = False, + authority_url: t.Optional[str] = None, + path: t.Optional[str] = None, + recursive: bool = False, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/outlook.py b/unstructured/ingest/runner/outlook.py index 3592634bd0..d0613ce340 100644 --- a/unstructured/ingest/runner/outlook.py +++ b/unstructured/ingest/runner/outlook.py @@ -10,15 +10,15 @@ def outlook( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, user_email: str, - client_id: t.Optional[str], - client_cred: t.Optional[str], - tenant: t.Optional[str], - authority_url: t.Optional[str], - recursive: bool, + verbose: bool = False, + recursive: bool = False, + client_id: t.Optional[str] = None, + client_cred: t.Optional[str] = None, + tenant: t.Optional[str] = None, + authority_url: t.Optional[str] = None, outlook_folders: t.Optional[t.List[str]] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, diff --git a/unstructured/ingest/runner/reddit.py b/unstructured/ingest/runner/reddit.py index 2003723789..fea56f1f12 100644 --- a/unstructured/ingest/runner/reddit.py +++ b/unstructured/ingest/runner/reddit.py @@ -10,15 +10,15 @@ def reddit( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, subreddit_name: str, - client_id: t.Optional[str], - client_secret: t.Optional[str], user_agent: str, - search_query: t.Optional[str], num_posts: int, + verbose: bool = False, + client_id: t.Optional[str] = None, + client_secret: t.Optional[str] = None, + search_query: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/s3.py b/unstructured/ingest/runner/s3.py index 45f27ce43d..e3646305fa 100644 --- a/unstructured/ingest/runner/s3.py +++ b/unstructured/ingest/runner/s3.py @@ -9,12 +9,13 @@ def s3( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, remote_url: str, - recursive: bool, - anonymous: bool, + verbose: bool = False, + recursive: bool = False, + anonymous: bool = False, + endpoint_url: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, @@ -31,11 +32,14 @@ def s3( from unstructured.ingest.connector.s3 import S3SourceConnector, SimpleS3Config + access_kwargs: t.Dict[str, t.Any] = {"anon": anonymous} + if endpoint_url: + access_kwargs["endpoint_url"] = endpoint_url source_doc_connector = S3SourceConnector( # type: ignore connector_config=SimpleS3Config( path=remote_url, recursive=recursive, - access_kwargs={"anon": anonymous}, + access_kwargs=access_kwargs, ), read_config=read_config, partition_config=partition_config, diff --git a/unstructured/ingest/runner/salesforce.py b/unstructured/ingest/runner/salesforce.py index ad0f050ed5..415d9be79b 100644 --- a/unstructured/ingest/runner/salesforce.py +++ b/unstructured/ingest/runner/salesforce.py @@ -10,14 +10,14 @@ def salesforce( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, - recursive: bool, username: str, consumer_key: str, private_key_path: str, categories: t.List[str], + verbose: bool = False, + recursive: bool = False, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/sharepoint.py b/unstructured/ingest/runner/sharepoint.py index d5ab2ec940..1781e2cbd9 100644 --- a/unstructured/ingest/runner/sharepoint.py +++ b/unstructured/ingest/runner/sharepoint.py @@ -14,9 +14,9 @@ def run( site: str, client_id: str, client_cred: str, - files_only: bool, path: str, - recursive: bool, + files_only: bool = False, + recursive: bool = False, **kwargs, ): writer_kwargs = self.writer_kwargs if self.writer_kwargs else {} diff --git a/unstructured/ingest/runner/slack.py b/unstructured/ingest/runner/slack.py index d2c61e9faf..0b9919c216 100644 --- a/unstructured/ingest/runner/slack.py +++ b/unstructured/ingest/runner/slack.py @@ -10,13 +10,13 @@ def slack( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, channels: t.List[str], token: str, - start_date: t.Optional[str], - end_date: t.Optional[str], + verbose: bool = False, + start_date: t.Optional[str] = None, + end_date: t.Optional[str] = None, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/wikipedia.py b/unstructured/ingest/runner/wikipedia.py index 44a031cd60..8914042cad 100644 --- a/unstructured/ingest/runner/wikipedia.py +++ b/unstructured/ingest/runner/wikipedia.py @@ -10,11 +10,11 @@ def wikipedia( - verbose: bool, read_config: ReadConfig, partition_config: PartitionConfig, page_title: str, - auto_suggest: bool, + verbose: bool = False, + auto_suggest: bool = False, writer_type: t.Optional[str] = None, writer_kwargs: t.Optional[dict] = None, **kwargs, diff --git a/unstructured/ingest/runner/writers.py b/unstructured/ingest/runner/writers.py index 46a875035e..7be5073c0f 100644 --- a/unstructured/ingest/runner/writers.py +++ b/unstructured/ingest/runner/writers.py @@ -9,6 +9,7 @@ def s3_writer( remote_url: str, anonymous: bool, + endpoint_url: t.Optional[str] = None, verbose: bool = False, **kwargs, ): @@ -17,11 +18,15 @@ def s3_writer( SimpleS3Config, ) + access_kwargs: t.Dict[str, t.Any] = {"anon": anonymous} + if endpoint_url: + access_kwargs["endpoint_url"] = endpoint_url + return S3DestinationConnector( write_config=WriteConfig(), connector_config=SimpleS3Config( path=remote_url, - access_kwargs={"anon": anonymous}, + access_kwargs=access_kwargs, ), ) diff --git a/unstructured/partition/csv.py b/unstructured/partition/csv.py index 6a7314de03..2528f321cd 100644 --- a/unstructured/partition/csv.py +++ b/unstructured/partition/csv.py @@ -4,6 +4,7 @@ import pandas as pd from lxml.html.soupparser import fromstring as soupparser_fromstring +from unstructured.chunking.title import add_chunking_strategy from unstructured.documents.elements import ( Element, ElementMetadata, @@ -21,6 +22,7 @@ @process_metadata() @add_metadata_with_filetype(FileType.CSV) +@add_chunking_strategy() def partition_csv( filename: Optional[str] = None, file: Optional[Union[IO[bytes], SpooledTemporaryFile]] = None, diff --git a/unstructured/partition/pdf.py b/unstructured/partition/pdf.py index 4cfa6b044a..5c1c3cbfd4 100644 --- a/unstructured/partition/pdf.py +++ b/unstructured/partition/pdf.py @@ -868,6 +868,23 @@ def get_uris( coordinate_system: Union[PixelSpace, PointSpace], page_number: int, ) -> List[dict]: + """ + Extracts URI annotations from a single or a list of PDF object references on a specific page. + The type of annots (list or not) depends on the pdf formatting. The function detectes the type + of annots and then pass on to get_uris_from_annots function as a List. + + Args: + annots (Union[PDFObjRef, List[PDFObjRef]]): A single or a list of PDF object references + representing annotations on the page. + height (float): The height of the page in the specified coordinate system. + coordinate_system (Union[PixelSpace, PointSpace]): The coordinate system used to represent + the annotations' coordinates. + page_number (int): The page number from which to extract annotations. + + Returns: + List[dict]: A list of dictionaries, each containing information about a URI annotation, + including its coordinates, bounding box, type, URI link, and page number. + """ if isinstance(annots, List): return get_uris_from_annots(annots, height, coordinate_system, page_number) return get_uris_from_annots(annots.resolve(), height, coordinate_system, page_number) @@ -879,6 +896,21 @@ def get_uris_from_annots( coordinate_system: Union[PixelSpace, PointSpace], page_number: int, ) -> List[dict]: + """ + Extracts URI annotations from a list of PDF object references. + + Args: + annots (List[PDFObjRef]): A list of PDF object references representing annotations on + a page. + height (Union[int, float]): The height of the page in the specified coordinate system. + coordinate_system (Union[PixelSpace, PointSpace]): The coordinate system used to represent + the annotations' coordinates. + page_number (int): The page number from which to extract annotations. + + Returns: + List[dict]: A list of dictionaries, each containing information about a URI annotation, + including its coordinates, bounding box, type, URI link, and page number. + """ annotation_list = [] for annotation in annots: annotation_dict = try_resolve(annotation) @@ -916,6 +948,10 @@ def get_uris_from_annots( def try_resolve(annot: PDFObjRef): + """ + Attempt to resolve a PDF object reference. If successful, returns the resolved object; + otherwise, returns the original reference. + """ try: return annot.resolve() except Exception: @@ -926,6 +962,19 @@ def rect_to_bbox( rect: Tuple[float, float, float, float], height: float, ) -> Tuple[float, float, float, float]: + """ + Converts a PDF rectangle coordinates (x1, y1, x2, y2) to a bounding box in the specified + coordinate system where the vertical axis is measured from the top of the page. + + Args: + rect (Tuple[float, float, float, float]): A tuple representing a PDF rectangle + coordinates (x1, y1, x2, y2). + height (float): The height of the page in the specified coordinate system. + + Returns: + Tuple[float, float, float, float]: A tuple representing the bounding box coordinates + (x1, y1, x2, y2) with the y-coordinates adjusted to be measured from the top of the page. + """ x1, y2, x2, y1 = rect y1 = height - y1 y2 = height - y2 @@ -936,6 +985,19 @@ def calculate_intersection_area( bbox1: Tuple[float, float, float, float], bbox2: Tuple[float, float, float, float], ) -> float: + """ + Calculate the area of intersection between two bounding boxes. + + Args: + bbox1 (Tuple[float, float, float, float]): The coordinates of the first bounding box + in the format (x1, y1, x2, y2). + bbox2 (Tuple[float, float, float, float]): The coordinates of the second bounding box + in the format (x1, y1, x2, y2). + + Returns: + float: The area of intersection between the two bounding boxes. If there is no + intersection, the function returns 0.0. + """ x1_1, y1_1, x2_1, y2_1 = bbox1 x1_2, y1_2, x2_2, y2_2 = bbox2 @@ -954,6 +1016,16 @@ def calculate_intersection_area( def calculate_bbox_area(bbox: Tuple[float, float, float, float]) -> float: + """ + Calculate the area of a bounding box. + + Args: + bbox (Tuple[float, float, float, float]): The coordinates of the bounding box + in the format (x1, y1, x2, y2). + + Returns: + float: The area of the bounding box, computed as the product of its width and height. + """ x1, y1, x2, y2 = bbox area = (x2 - x1) * (y2 - y1) return area @@ -965,6 +1037,24 @@ def check_annotations_within_element( page_number: int, threshold: float = 0.9, ) -> List[dict]: + """ + Filter annotations that are within or highly overlap with a specified element on a page. + + Args: + annotation_list (List[dict]): A list of dictionaries, each containing information + about an annotation. + element_bbox (Tuple[float, float, float, float]): The bounding box coordinates of the + specified element in the bbox format (x1, y1, x2, y2). + page_number (int): The page number to which the annotations and element belong. + threshold (float, optional): The threshold value (between 0.0 and 1.0) that determines + the minimum overlap required for an annotation to be considered within the element. + Default is 0.9. + + Returns: + List[dict]: A list of dictionaries containing information about annotations that are + within or highly overlap with the specified element on the given page, based on the + specified threshold. + """ annotations_within_element = [] for annotation in annotation_list: if annotation["page_number"] == page_number and ( @@ -980,6 +1070,19 @@ def get_word_bounding_box_from_element( obj: LTTextBox, height: float, ) -> Tuple[List[LTChar], List[dict]]: + """ + Extracts characters and word bounding boxes from a PDF text element. + + Args: + obj (LTTextBox): The PDF text element from which to extract characters and words. + height (float): The height of the page in the specified coordinate system. + + Returns: + Tuple[List[LTChar], List[dict]]: A tuple containing two lists: + - List[LTChar]: A list of LTChar objects representing individual characters. + - List[dict]: A list of dictionaries, each containing information about a word, + including its text, bounding box, and start index in the element's text. + """ characters = [] words = [] text_len = 0 @@ -1002,10 +1105,9 @@ def get_word_bounding_box_from_element( # TODO(klaijan) - isalnum() only works with A-Z, a-z and 0-9 # will need to switch to some pattern matching once we support more languages - if index == 0: + if not word: isalnum = char.isalnum() - - if char.isalnum() != isalnum: + if word and char.isalnum() != isalnum: isalnum = char.isalnum() words.append( {"text": word, "bbox": (x1, y1, x2, y2), "start_index": start_index}, @@ -1028,6 +1130,19 @@ def get_word_bounding_box_from_element( def map_bbox_and_index(words: List[dict], annot: dict): + """ + Maps a bounding box annotation to the corresponding text and start index within a list of words. + + Args: + words (List[dict]): A list of dictionaries, each containing information about a word, + including its text, bounding box, and start index. + annot (dict): The annotation dictionary to be mapped, which will be updated with "text" and + "start_index" fields. + + Returns: + dict: The updated annotation dictionary with "text" representing the mapped text and + "start_index" representing the start index of the mapped text in the list of words. + """ if len(words) == 0: annot["text"] = "" annot["start_index"] = -1 @@ -1059,6 +1174,16 @@ def map_bbox_and_index(words: List[dict], annot: dict): def try_argmin(array: np.ndarray) -> int: + """ + Attempt to find the index of the minimum value in a NumPy array. + + Args: + array (np.ndarray): The NumPy array in which to find the minimum value's index. + + Returns: + int: The index of the minimum value in the array. If the array is empty or an + IndexError occurs, it returns -1. + """ try: return int(np.argmin(array)) except IndexError: diff --git a/unstructured/partition/xlsx.py b/unstructured/partition/xlsx.py index 2f4538210f..ebffd6cdf9 100644 --- a/unstructured/partition/xlsx.py +++ b/unstructured/partition/xlsx.py @@ -4,6 +4,7 @@ import pandas as pd from lxml.html.soupparser import fromstring as soupparser_fromstring +from unstructured.chunking.title import add_chunking_strategy from unstructured.documents.elements import ( Element, ElementMetadata, @@ -21,6 +22,7 @@ @process_metadata() @add_metadata_with_filetype(FileType.XLSX) +@add_chunking_strategy() def partition_xlsx( filename: Optional[str] = None, file: Optional[Union[IO[bytes], SpooledTemporaryFile]] = None, diff --git a/unstructured/staging/weaviate.py b/unstructured/staging/weaviate.py index c6efc80bd4..4a4e15276c 100644 --- a/unstructured/staging/weaviate.py +++ b/unstructured/staging/weaviate.py @@ -15,6 +15,7 @@ class Properties(TypedDict): "regex_metadata", "emphasized_texts", "detection_class_prob", + "is_continuation", )