improve docstrings and remove detect_per_element kwarg from partition…

…ers that only return one element (xlsx, csv, tsv)
Unstructured-IO · Oct 4, 2023 · b1c4048 · b1c4048
1 parent 2879f21
commit b1c4048
Show file tree

Hide file tree

Showing 18 changed files with 60 additions and 60 deletions.
diff --git a/unstructured/partition/csv.py b/unstructured/partition/csv.py
@@ -31,7 +31,6 @@ def partition_csv(
     metadata_last_modified: Optional[str] = None,
     include_metadata: bool = True,
     languages: List[str] = ["auto"],
-    detect_language_per_element: bool = False,
     **kwargs,
 ) -> List[Element]:
     """Partitions Microsoft Excel Documents in .csv format into its document elements.
@@ -49,11 +48,9 @@ def partition_csv(
     include_metadata
         Determines whether or not metadata is included in the output.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
-        Additional Parameters:
-            detect_language_per_element
-                Detect language per element instead of at the document level.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
     """
     exactly_one(filename=filename, file=file)
 
@@ -75,11 +72,6 @@ def partition_csv(
 
     html_text = table.to_html(index=False, header=False, na_rep="")
     text = soupparser_fromstring(html_text).text_content()
-    # languages = detect_languages(
-    #     text=text,
-    #     languages=languages,
-    #     detect_language_per_element=detect_language_per_element,
-    # )
 
     if include_metadata:
         metadata = ElementMetadata(
@@ -95,6 +87,5 @@ def partition_csv(
         apply_lang_metadata(
             [Table(text=text, metadata=metadata)],
             languages=languages,
-            detect_language_per_element=detect_language_per_element,
         ),
     )
diff --git a/unstructured/partition/doc.py b/unstructured/partition/doc.py
@@ -45,8 +45,9 @@ def partition_doc(
         filter that is required when using LibreOffice7. Pass in None
         if you do not want to apply any filter.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/docx.py b/unstructured/partition/docx.py
@@ -107,8 +107,9 @@ def convert_and_partition_docx(
         Determines whether or not metadata is included in the metadata attribute on the elements in
         the output.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.
@@ -187,8 +188,9 @@ def partition_docx(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/email.py b/unstructured/partition/email.py
@@ -294,8 +294,9 @@ def partition_email(
         The minimum number of characters to include in a partition. Only applies if
         processing the text/plain content.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/epub.py b/unstructured/partition/epub.py
@@ -49,8 +49,9 @@ def partition_epub(
     encoding
         The encoding method used to decode the text input. If None, utf-8 will be used.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/html.py b/unstructured/partition/html.py
@@ -82,8 +82,9 @@ def partition_html(
     skip_headers_and_footers
         If True, ignores any content that is within <header> or <footer> tags
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.
@@ -183,8 +184,9 @@ def convert_and_partition_html(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/md.py b/unstructured/partition/md.py
@@ -60,8 +60,9 @@ def partition_md(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/msg.py b/unstructured/partition/msg.py
@@ -57,8 +57,9 @@ def partition_msg(
         The minimum number of characters to include in a partition. Only applies if
         processing text/plain content.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/odt.py b/unstructured/partition/odt.py
@@ -35,8 +35,9 @@ def partition_odt(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/org.py b/unstructured/partition/org.py
@@ -33,8 +33,9 @@ def partition_org(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/ppt.py b/unstructured/partition/ppt.py
@@ -42,8 +42,9 @@ def partition_ppt(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/pptx.py b/unstructured/partition/pptx.py
@@ -78,8 +78,9 @@ def partition_pptx(
     include_slide_notes
         If True, includes the slide notes as element
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/rst.py b/unstructured/partition/rst.py
@@ -35,8 +35,9 @@ def partition_rst(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/rtf.py b/unstructured/partition/rtf.py
@@ -35,8 +35,9 @@ def partition_rtf(
     metadata_last_modified
         The last modified date for the document.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/text.py b/unstructured/partition/text.py
@@ -195,8 +195,9 @@ def partition_text(
     include_metadata
         Determines whether or not metadata is included in the output.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.

diff --git a/unstructured/partition/tsv.py b/unstructured/partition/tsv.py
@@ -29,7 +29,6 @@ def partition_tsv(
     metadata_last_modified: Optional[str] = None,
     include_metadata: bool = True,
     languages: List[str] = ["auto"],
-    detect_language_per_element: bool = False,
     **kwargs,
 ) -> List[Element]:
     """Partitions TSV files into document elements.
@@ -45,11 +44,9 @@ def partition_tsv(
     metadata_last_modified
         The day of the last modification.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
-        Additional Parameters:
-            detect_language_per_element
-                Detect language per element instead of at the document level.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
     """
     exactly_one(filename=filename, file=file)
 
@@ -86,6 +83,5 @@ def partition_tsv(
         apply_lang_metadata(
             [Table(text=text, metadata=metadata)],
             languages=languages,
-            detect_language_per_element=detect_language_per_element,
         ),
     )
diff --git a/unstructured/partition/xlsx.py b/unstructured/partition/xlsx.py
@@ -32,7 +32,6 @@ def partition_xlsx(
     metadata_last_modified: Optional[str] = None,
     include_header: bool = True,
     languages: List[str] = ["auto"],
-    detect_language_per_element: bool = False,
     **kwargs,
 ) -> List[Element]:
     """Partitions Microsoft Excel Documents in .xlsx format into its document elements.
@@ -50,11 +49,9 @@ def partition_xlsx(
     include_header
         Determines whether or not header info info is included in text and medatada.text_as_html
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
-        Additional Parameters:
-            detect_language_per_element
-                Detect language per element instead of at the document level.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
     """
     exactly_one(filename=filename, file=file)
 
@@ -100,6 +97,5 @@ def partition_xlsx(
         apply_lang_metadata(
             elements=elements,
             languages=languages,
-            detect_language_per_element=detect_language_per_element,
         ),
     )
diff --git a/unstructured/partition/xml.py b/unstructured/partition/xml.py
@@ -117,8 +117,9 @@ def partition_xml(
     metadata_last_modified
         The day of the last modification.
     languages
-        Detected language of a text using naive Bayesian filter. Multiple languages indicates text
-        could be in either language.
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
         Additional Parameters:
             detect_language_per_element
                 Detect language per element instead of at the document level.