diff --git a/chan/chan.py b/chan/chan.py index e38765e..b81d8fd 100644 --- a/chan/chan.py +++ b/chan/chan.py @@ -163,7 +163,7 @@ CHANS = { ( "jp", "drive" ), - rps=1 / 60 + rps=1 / 120 ), "synch": SynchJsonChanHelper( 12, diff --git a/chan/fchan_html.py b/chan/fchan_html.py index c6444fd..33a424b 100644 --- a/chan/fchan_html.py +++ b/chan/fchan_html.py @@ -1,5 +1,4 @@ import datetime -import _strptime import re from urllib.parse import urljoin @@ -11,7 +10,7 @@ from chan.desuchan_html import DesuChanHtmlChanHelper class FChanHtmlChanHelper(DesuChanHtmlChanHelper): def parse_threads_list(self, r): - soup = BeautifulSoup(r.text, "html.parser") + soup = BeautifulSoup(r.content.decode('utf-8', 'ignore'), "html.parser") threads = [] diff --git a/chan/helper.py b/chan/helper.py index e89fc29..c1740d3 100644 --- a/chan/helper.py +++ b/chan/helper.py @@ -58,7 +58,7 @@ class ChanHelper: @staticmethod def parse_thread(r): - soup = BeautifulSoup(r.text, "html.parser") + soup = BeautifulSoup(r.content.decode('utf-8', 'ignore'), "html.parser") op_el = soup.find("div", attrs={"class": "innerOP"}) yield { diff --git a/chan/json.py b/chan/json.py index 7e80104..431d202 100644 --- a/chan/json.py +++ b/chan/json.py @@ -39,7 +39,7 @@ class JsonChanHelper(ChanHelper): @staticmethod def parse_threads_list(r): try: - j = json.loads(r.text) + j = json.loads(r.content.decode('utf-8', 'ignore')) if len(j) == 0 or "threads" not in j[0]: logger.warning("No threads in response for %s: %s" % (r.url, r.text,)) return [], None @@ -56,5 +56,10 @@ class JsonChanHelper(ChanHelper): @staticmethod def parse_thread(r): - j = json.loads(r.text) + try: + j = json.loads(r.content.decode('utf-8', 'ignore')) + except JSONDecodeError: + logger.warning("JSONDecodeError for %s:" % (r.url,)) + logger.warning(r.text) + return [] return j["posts"] diff --git a/chan/lolnada_html.py b/chan/lolnada_html.py index f229aaa..b85a767 100644 --- a/chan/lolnada_html.py +++ b/chan/lolnada_html.py @@ -39,7 +39,7 @@ class LolNadaHtmlChanHelper(ChanHelper): return item["time"] def parse_threads_list(self, r): - soup = BeautifulSoup(r.text, "html.parser") + soup = BeautifulSoup(r.content.decode('utf-8', 'ignore'), "html.parser") threads = [] @@ -59,7 +59,7 @@ class LolNadaHtmlChanHelper(ChanHelper): @staticmethod def parse_thread(r): - soup = BeautifulSoup(r.text, "html.parser") + soup = BeautifulSoup(r.content.decode('utf-8', 'ignore'), "html.parser") op_el = soup.find("div", class_="hilo") for post_el in op_el.find_all("div", class_="post reply"): diff --git a/chan/mayuri.py b/chan/mayuri.py index 6c57256..3a0930f 100644 --- a/chan/mayuri.py +++ b/chan/mayuri.py @@ -42,7 +42,7 @@ class MayuriChanHelper(ChanHelper): def parse_threads_list(self, r): try: - j = json.loads(r.text) + j = json.loads(r.content.decode('utf-8', 'ignore')) except JSONDecodeError: logger.warning("JSONDecodeError for %s:" % (r.url,)) logger.warning(r.text) @@ -54,7 +54,7 @@ class MayuriChanHelper(ChanHelper): @staticmethod def parse_thread(r): try: - j = json.loads(r.text) + j = json.loads(r.content.decode('utf-8', 'ignore')) except JSONDecodeError: logger.warning("JSONDecodeError for %s:" % (r.url,)) logger.warning(r.text) diff --git a/chan/russian_json.py b/chan/russian_json.py index df40a56..fa4f09a 100644 --- a/chan/russian_json.py +++ b/chan/russian_json.py @@ -19,7 +19,7 @@ class RussianJsonChanHelper(ChanHelper): @staticmethod def parse_threads_list(r): try: - j = json.loads(r.text) + j = json.loads(r.content.decode('utf-8', 'ignore')) except JSONDecodeError: logger.warning("JSONDecodeError for %s:" % (r.url,)) logger.warning(r.text) @@ -28,7 +28,12 @@ class RussianJsonChanHelper(ChanHelper): @staticmethod def parse_thread(r): - j = json.loads(r.text) + try: + j = json.loads(r.content.decode('utf-8', 'ignore')) + except JSONDecodeError: + logger.warning("JSONDecodeError for %s:" % (r.url,)) + logger.warning(r.text) + return [] for thread in j["threads"]: for post in thread["posts"]: yield post