Should fix unknown encoding errors + removed https warnings

crawl_server/remote_http.py

@@ -1,5 +1,4 @@
 from urllib.parse import unquote, urljoin
-import warnings
 import os
 from html.parser import HTMLParser
 from itertools import repeat
@@ -11,6 +10,9 @@ import config
 from dateutil.parser import parse as parse_date
 import hashlib
 
+import urllib3
+urllib3.disable_warnings()
+
 
 class Anchor:
     def __init__(self):
@@ -134,8 +136,6 @@ class HttpDirectory(RemoteDirectory):
 
     def _request_file(self, url):
 
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
         retries = HttpDirectory.MAX_RETRIES
         while retries > 0:
             try:
@@ -159,14 +159,16 @@ class HttpDirectory(RemoteDirectory):
         return None
 
     def _stream_body(self, url: str):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
         retries = HttpDirectory.MAX_RETRIES
         while retries > 0:
             try:
                 r = self.session.get(url, stream=True, timeout=40)
                 for chunk in r.iter_content(chunk_size=4096):
+                    try:
                         yield chunk.decode(r.encoding if r.encoding else "utf-8", errors="ignore")
+                    except LookupError:
+                        # Unsupported encoding
+                        yield chunk.decode("utf-8", errors="ignore")
                 r.close()
                 del r
                 break