diff --git a/run.py b/run.py index d3149be..66f358e 100644 --- a/run.py +++ b/run.py @@ -25,7 +25,7 @@ class ChanScanner: def _threads(self, board): r = self.web.get(self.helper.threads_url(board)) - if r.status_code != 200: + if not r or r.status_code != 200: return [] while True: @@ -35,12 +35,12 @@ class ChanScanner: if not next_url: break r = self.web.get(next_url) - if r.status_code != 200: + if not r or r.status_code != 200: break def _fetch_posts(self, board, thread): r = self.web.get(self.helper.posts_url(board, thread)) - if r.status_code == 200: + if r and r.status_code == 200: return self.helper.parse_thread(r) return [] diff --git a/util.py b/util.py index d55bbb8..b578bd3 100644 --- a/util.py +++ b/util.py @@ -1,6 +1,7 @@ import logging import sys import time +import traceback from datetime import datetime from logging import FileHandler, StreamHandler @@ -47,17 +48,32 @@ class Web: @rate_limit(1 / 2) # TODO: per chan rate limit? def get(self, url, **kwargs): - r = self.session.get(url, **kwargs) - logger.debug("GET %s <%d>" % (url, r.status_code)) - if self.monitoring: - self.monitoring.log([{ - "measurement": "web", - "time": str(datetime.utcnow()), - "fields": { - "status_code": r.status_code - }, - "tags": { - "ok": r.status_code == 200 - }, - }]) - return r + try: + r = self.session.get(url, **kwargs) + logger.debug("GET %s <%d>" % (url, r.status_code)) + if self.monitoring: + self.monitoring.log([{ + "measurement": "web", + "time": str(datetime.utcnow()), + "fields": { + "status_code": r.status_code + }, + "tags": { + "ok": r.status_code == 200 + }, + }]) + return r + except Exception as e: + logger.error(str(e) + traceback.format_exc()) + if self.monitoring: + self.monitoring.log([{ + "measurement": "web", + "time": str(datetime.utcnow()), + "fields": { + "status_code": 0 + }, + "tags": { + "ok": False + }, + }]) + return None