simon987/chan_feed
1
0
Fork 0
mirror of https://github.com/simon987/chan_feed.git synced 2025-04-19 18:16:44 +00:00
Code Issues Projects Releases Wiki Activity

More error handling

Browse Source
This commit is contained in:
simon 2019-09-06 17:08:45 -04:00
parent 7a4c814408
commit c1c4f8fb8f
2 changed files with 33 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
run.py
View File

@ -25,7 +25,7 @@ class ChanScanner:
def _threads(self, board): def _threads(self, board):
r = self.web.get(self.helper.threads_url(board)) r = self.web.get(self.helper.threads_url(board))
if r.status_code != 200: if not r or r.status_code != 200:
return [] return []
while True: while True:
@ -35,12 +35,12 @@ class ChanScanner:
if not next_url: if not next_url:
break break
r = self.web.get(next_url) r = self.web.get(next_url)
if r.status_code != 200: if not r or r.status_code != 200:
break break
def _fetch_posts(self, board, thread): def _fetch_posts(self, board, thread):
r = self.web.get(self.helper.posts_url(board, thread)) r = self.web.get(self.helper.posts_url(board, thread))
if r.status_code == 200: if r and r.status_code == 200:
return self.helper.parse_thread(r) return self.helper.parse_thread(r)
return [] return []

16
util.py
View File

@ -1,6 +1,7 @@
import logging import logging
import sys import sys
import time import time
import traceback
from datetime import datetime from datetime import datetime
from logging import FileHandler, StreamHandler from logging import FileHandler, StreamHandler
@ -47,6 +48,7 @@ class Web:
@rate_limit(1 / 2) # TODO: per chan rate limit? @rate_limit(1 / 2) # TODO: per chan rate limit?
def get(self, url, **kwargs): def get(self, url, **kwargs):
try:
r = self.session.get(url, **kwargs) r = self.session.get(url, **kwargs)
logger.debug("GET %s <%d>" % (url, r.status_code)) logger.debug("GET %s <%d>" % (url, r.status_code))
if self.monitoring: if self.monitoring:
@ -61,3 +63,17 @@ class Web:
}, },
}]) }])
return r return r
except Exception as e:
logger.error(str(e) + traceback.format_exc())
if self.monitoring:
self.monitoring.log([{
"measurement": "web",
"time": str(datetime.utcnow()),
"fields": {
"status_code": 0
},
"tags": {
"ok": False
},
}])
return None