Merge pull request #121 from aquatiko/add-regex

changed remove_patterns

src/preprocessing.jl

@@ -254,11 +254,14 @@ function remove_patterns(s::AbstractString, rex::Regex)
     v=codeunits(s)
     for m in eachmatch(rex, s)
         len = m.match.offset-ibegin+1
+	next = nextind(s, lastindex(m.match)+m.match.offset)
         if len > 0
             Base.write_sub(iob, v, ibegin, len)
-            write(iob, ' ')
+	    if  next != length(s)+1
+            	write(iob, ' ')
+	    end
         end
-        ibegin = nextind(s, lastindex(m.match)+m.match.offset)
+        ibegin = next
     end
     len = length(v) - ibegin + 1
     (len > 0) && Base.write_sub(iob, v, ibegin, len)
@@ -272,17 +275,21 @@ function remove_patterns(s::SubString{T}, rex::Regex) where T <: String
     ibegin = 1
     for m in eachmatch(rex, s)
         len = m.match.offset-ibegin+1
+	next = nextind(s, lastindex(m.match)+m.match.offset)
         if len > 0
             Base.write_sub(iob, data, ibegin+ioffset, len)
-            write(iob, ' ')
+            if  next != length(s)+1
+            	write(iob, ' ')
+	    end
         end
-        ibegin = nextind(s, lastindex(m.match)+m.match.offset)
+        ibegin = next
     end
     len = lastindex(s) - ibegin + 1
     (len > 0) && Base.write_sub(iob, data, ibegin+ioffset, len)
     String(take!(iob))
 end
 
+
 remove_patterns!(d::FileDocument, rex::Regex) = error("FileDocument cannot be modified")
 
 function remove_patterns!(d::StringDocument, rex::Regex)

test/preprocessing.jl

@@ -93,12 +93,17 @@
 
     #Tests strip_punctuation regex conditions
     str = Document("These punctuations should be removed [-.,:;,!?'\"[](){}|\`#\$%@^&*_+<>")
-    answer = Document("These punctuations should be removed  ")
+    answer = Document("These punctuations should be removed ")
     prepare!(str, strip_punctuation)
     @test isequal(str.text, answer.text)
 
     str = Document("Intel(tm) Core i5-3300k, is a geat CPU! ")
     answer = Document("Intel tm  Core i5 3300k  is a geat CPU  ")   #tests old implementation   
     prepare!(str, strip_punctuation)
     @test isequal(str.text, answer.text)
+
+    #Tests no whitespace at end or begining
+    doc = Document("   this is sample text   ")
+    prepare!(doc, strip_whitespace)
+    @test isequal(doc.text, "this is sample text")
 end