diff --git a/zipline/sources/requests_csv.py b/zipline/sources/requests_csv.py index 1ea36f691e..d2acdc472a 100644 --- a/zipline/sources/requests_csv.py +++ b/zipline/sources/requests_csv.py @@ -563,6 +563,19 @@ def fetch_url(self, url): if content_length > self.MAX_DOCUMENT_SIZE: raise Exception('Document size too big.') if chunk: + if isinstance(chunk, bytes): + if content_length == 0: + import codecs + if chunk[:3] == codecs.BOM_UTF8: + encoding = 'utf-8-sig' + elif chunk[:2] == codecs.BOM_UTF16_LE or chunk[:2] == codecs.BOM_UTF16_BE: + encoding = 'utf-16' + elif chunk[:4] == codecs.BOM_UTF32_LE or chunk[:4] == codecs.BOM_UTF32_BE: + encoding = 'utf-32' + else: + encoding = 'utf-8' + decoder = codecs.getincrementaldecoder(encoding)(errors='replace') + chunk = decoder.decode(chunk) content_length += len(chunk) yield chunk