cleanup

Querent-ai · Sep 2, 2023 · 00d1b86 · 00d1b86
1 parent afb83e3
commit 00d1b86
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/querent/collectors/webscaper/web_scraper_collector.py b/querent/collectors/webscaper/web_scraper_collector.py
@@ -32,7 +32,7 @@ async def scrape_website(self, website_url: str):
             async with ClientSession(connector=TCPConnector(ssl=False)) as session:
                 async with session.get(website_url) as response:
                     content = await response.text()
-                    max_length = len(" ".join(content.split(" ")[:600]))
+                    max_length = len(content)
                     return CollectedBytes(
                         data=content[:max_length], file=None, error=None
                     )