add additional docs and tests for multilanguage detection in docx, ep…

…ub, odt, and rtf partitioners

test_unstructured/partition/docx/test_docx.py

@@ -517,9 +517,15 @@ def test_add_chunking_strategy_on_partition_docx(
     assert chunk_elements != elements
     assert chunk_elements == chunks
 
-    
+
 def test_partition_docx_element_metadata_has_languages():
     filename = "example-docs/handbook-1p.docx"
     elements = partition_docx(filename=filename)
     assert elements[0].metadata.languages == ["eng"]
-
+
+
+def test_partition_docx_detects_multiple_elements_in_other_language():
+    filename = "example-docs/language-docs/eng_spa_mult.docx"
+    elements = partition_docx(filename=filename, detect_language_per_element=True)
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]

test_unstructured/partition/epub/test_epub.py

@@ -215,8 +215,15 @@ def test_add_chunking_strategy_on_partition_epub_non_default(
     assert chunk_elements != elements
     assert chunk_elements == chunks
 
-    
+
 def test_partition_epub_element_metadata_has_languages():
     filename = os.path.join(DIRECTORY, "..", "..", "..", "example-docs", "winter-sports.epub")
     elements = partition_epub(filename=filename)
-    assert elements[0].metadata.languages == ["eng"]    
+    assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_epub_detects_multiple_elements_in_other_language():
+    filename = "example-docs/language-docs/eng_spa_mult.epub"
+    elements = partition_epub(filename=filename, detect_language_per_element=True)
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng", "ron"], ["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]

test_unstructured/partition/odt/test_odt.py

@@ -191,8 +191,15 @@ def test_add_chunking_strategy_on_partition_odt_non_default():
     assert chunk_elements != elements
     assert chunk_elements == chunks
 
-    
+
 def test_partition_odt_element_metadata_has_languages():
     filename = "example-docs/fake.odt"
     elements = partition_odt(filename=filename)
-    assert elements[0].metadata.languages == ["eng"]    
+    assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_odt_detects_multiple_elements_in_other_language():
+    filename = "example-docs/language-docs/eng_spa_mult.odt"
+    elements = partition_odt(filename=filename, detect_language_per_element=True)
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]

test_unstructured/partition/pypandoc/test_rtf.py

@@ -167,3 +167,10 @@ def test_partition_rtf_element_metadata_has_languages():
     filename = "example-docs/fake-doc.rtf"
     elements = partition_rtf(filename=filename)
     assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_rtf_detects_multiple_elements_in_other_language():
+    filename = "example-docs/language-docs/eng_spa_mult.rtf"
+    elements = partition_rtf(filename=filename, detect_language_per_element=True)
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]

test_unstructured/partition/test_text.py

@@ -539,8 +539,8 @@ def test_partition_text_element_metadata_has_languages():
 def test_partition_text_detects_multiple_elements_in_other_language():
     filename = "example-docs/language-docs/eng_spa_mult.txt"
     elements = partition_text(filename=filename, detect_language_per_element=True)
-    langs = {element.metadata.languages[0] for element in elements}
-    assert langs == {"eng", "spa"}
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]
 
 
 def test_partition_text_detects_more_than_3_languages():