simon987/chan_feed
1
0
Fork 0
mirror of https://github.com/simon987/chan_feed.git synced 2025-04-24 12:15:50 +00:00
Code Issues Projects Releases Wiki Activity

handle json decode error

Browse Source
This commit is contained in:
simon 2019-09-07 20:48:08 -04:00
parent f25d091da8
commit 8cc4d60541
1 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
chan.py
View File

@ -1,9 +1,11 @@
import json import json
from json import JSONDecodeError
from urllib.parse import urljoin from urllib.parse import urljoin
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from post_process import get_links_from_body from post_process import get_links_from_body
from util import logger
class ChanHelper: class ChanHelper:
@ -155,7 +157,13 @@ class JsonChanHelper(ChanHelper):
@staticmethod @staticmethod
def parse_threads_list(r): def parse_threads_list(r):
try:
j = json.loads(r.text) j = json.loads(r.text)
except JSONDecodeError:
logger.warning("JSONDecodeError for %s:" % (r.url, ))
logger.warning(r.text)
return [], None
threads = [] threads = []
for page in j: for page in j:
for thread in page["threads"]: for thread in page["threads"]:
@ -180,7 +188,12 @@ class RussianJsonChanHelper(ChanHelper):
@staticmethod @staticmethod
def parse_threads_list(r): def parse_threads_list(r):
try:
j = json.loads(r.text) j = json.loads(r.text)
except JSONDecodeError:
logger.warning("JSONDecodeError for %s:" % (r.url, ))
logger.warning(r.text)
return [], None
return j["threads"], None return j["threads"], None
@staticmethod @staticmethod
@ -207,7 +220,7 @@ class RussianJsonChanHelper(ChanHelper):
urls.update(get_links_from_body(item["subject"])) urls.update(get_links_from_body(item["subject"]))
for file in item["files"]: for file in item["files"]:
urls.add(self._base_url + file["path"]) urls.add(self._base_url.rstrip("/") + file["path"])
return list(urls) return list(urls)
@ -309,7 +322,7 @@ CHANS = {
"a", "fd", "ja", "ma", "vn", "fg", "fur", "gg", "ga", "a", "fd", "ja", "ma", "vn", "fg", "fur", "gg", "ga",
"vape", "h", "ho", "hc", "e", "fet", "sex", "fag" "vape", "h", "ho", "hc", "e", "fet", "sex", "fag"
), ),
rps=1 rps=10
), ),
"endchan": HtmlChanHelper( "endchan": HtmlChanHelper(
8, 8,