Minor improvements

hardikp · Apr 27, 2017 · 6afecba · 6afecba
1 parent 3673861
commit 6afecba
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/data/textdata/textdata/spiders/qplum.py b/data/textdata/textdata/spiders/qplum.py
@@ -1,4 +1,5 @@
 import json
+import re
 
 from scrapy.spiders import CrawlSpider
 from w3lib.html import remove_tags, remove_tags_with_content
@@ -29,8 +30,11 @@ def parse(self, response):
         # Replace &#39; with '
         paragraph = paragraph.replace('&#39;', "'")
         paragraph = paragraph.replace('&rsquo;', "'")
+        paragraph = paragraph.replace('&ldquo;', "'")
+        paragraph = paragraph.replace('&rdquo;', "'")
         # Replace &nbsp; with a space
-        paragraph = paragraph.replace('&nbsp;', ' ')
+        paragraph = re.sub("&.....;", ' ', paragraph)
+        paragraph = re.sub("&....;", ' ', paragraph)
 
         # Some more replacements to improve the default tokenization
         for c in ['\n', '\r', '\t']: