simon987/chan_feed
1
0
Fork 0
mirror of https://github.com/simon987/chan_feed.git synced 2025-04-20 02:26:44 +00:00
Code Issues Projects Releases Wiki Activity

More error handling

Browse Source
This commit is contained in:
simon 2019-09-06 17:08:45 -04:00
parent 7a4c814408
commit c1c4f8fb8f
2 changed files with 33 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
run.py
View File

@ -25,7 +25,7 @@ class ChanScanner:
def _threads(self, board):
r = self.web.get(self.helper.threads_url(board))
if r.status_code != 200:
if not r or r.status_code != 200:
return []
while True:
@ -35,12 +35,12 @@ class ChanScanner:
if not next_url:
break
r = self.web.get(next_url)
if r.status_code != 200:
if not r or r.status_code != 200:
break
def _fetch_posts(self, board, thread):
r = self.web.get(self.helper.posts_url(board, thread))
if r.status_code == 200:
if r and r.status_code == 200:
return self.helper.parse_thread(r)
return []

44
util.py
View File

@ -1,6 +1,7 @@
import logging
import sys
import time
import traceback
from datetime import datetime
from logging import FileHandler, StreamHandler
@ -47,17 +48,32 @@ class Web:
@rate_limit(1 / 2) # TODO: per chan rate limit?
def get(self, url, **kwargs):
r = self.session.get(url, **kwargs)
logger.debug("GET %s <%d>" % (url, r.status_code))
if self.monitoring:
self.monitoring.log([{
"measurement": "web",
"time": str(datetime.utcnow()),
"fields": {
"status_code": r.status_code
},
"tags": {
"ok": r.status_code == 200
},
}])
return r
try:
r = self.session.get(url, **kwargs)
logger.debug("GET %s <%d>" % (url, r.status_code))
if self.monitoring:
self.monitoring.log([{
"measurement": "web",
"time": str(datetime.utcnow()),
"fields": {
"status_code": r.status_code
},
"tags": {
"ok": r.status_code == 200
},
}])
return r
except Exception as e:
logger.error(str(e) + traceback.format_exc())
if self.monitoring:
self.monitoring.log([{
"measurement": "web",
"time": str(datetime.utcnow()),
"fields": {
"status_code": 0
},
"tags": {
"ok": False
},
}])
return None