fix html2text

guo-yong-zhi · Aug 22, 2024 · 364a2ba · 364a2ba · guo-yong-zhi · Aug 22, 2024
1 parent b185354
commit 364a2ba
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 2 deletions.
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "WordCloud"
 uuid = "6385f0a0-cb03-45b6-9089-4e0acc74b26b"
 authors = ["guoyongzhi <[email protected]>"]
-version = "1.1.0"
+version = "1.1.1"
 
 [deps]
 ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4"

diff --git a/src/textprocessing.jl b/src/textprocessing.jl
@@ -294,9 +294,10 @@ function processtext(counter::AbstractVector{<:Union{Pair,Tuple,AbstractVector}}
 end
 function html2text(content::AbstractString)
     patterns = [
-        r"\"[\s\S]*?\"" => " ",
         r"<[\s]*?script[^>]*?>[\s\S]*?<[\s]*?/[\s]*?script[\s]*?>" => " ",
         r"<[\s]*?style[^>]*?>[\s\S]*?<[\s]*?/[\s]*?style[\s]*?>" => " ",
+        r"<!--[\s\S]*?-->" => " ",
+        r"<[\s\S]*?=\s*?\"[\s\S]*?\"\s*?>" => " ",
         "<br>" => "\n",
         r"<[\s\S]*?>" => " ",
     ]

diff --git a/test/test_textprocessing.jl b/test/test_textprocessing.jl
@@ -78,4 +78,6 @@
 
     htstr = """&pound;abcd<div x-component-name="DisasterSokuho" x-component-data="{&quot;earthquake&quot;:&quot;&lt;!-- 地震速報のメッセージを消しました （2024-04-25 12:00:08）-->\n&quot;,&quot;tsunami&quot;:&quot;&lt;!-- 津波速報のメッセージを消しました （2024-04-25 12:05:35）-->\n&quot;}"><div class="tYQVs"><div>"""
     @test strip(html2text(htstr)) == "abcd"
+    htstr= """<span class="reference-text">"<a rel="nofollow" class="external text" href="不应该出现">something</a>." <i>"""
+    @test replace(html2text(htstr), r"\s"=>"") == "\"something.\""
 end