simon987/chan_feed
1
0
Fork 0
mirror of https://github.com/simon987/chan_feed.git synced 2025-04-10 14:06:42 +00:00
Code Issues Projects Releases Wiki Activity

bug fix, change caching

Browse Source
This commit is contained in:
simon987 2021-03-27 15:02:28 -04:00
parent 0133c42d62
commit 4e0f837cec
2 changed files with 3 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
chan/chan_json.py
View File

@ -35,7 +35,7 @@ class JsonChanHelper(ChanHelper):
@staticmethod @staticmethod
def thread_mtime(thread): def thread_mtime(thread):
return thread["last_modified"] return thread["last_modified"] if "last_modified" in thread else -1
@staticmethod @staticmethod
def parse_threads_list(r): def parse_threads_list(r):

17
run.py
View File

@ -56,15 +56,6 @@ class ChanScanner:
yield post, board yield post, board
def once(func):
def wrapper(item, board, helper):
if not state.has_visited(helper.item_unique_id(item, board)):
func(item, board, helper)
state.mark_visited(helper.item_unique_id(item, board))
return wrapper
class ChanState: class ChanState:
def __init__(self, prefix): def __init__(self, prefix):
self._posts = VolatileBooleanState(prefix) self._posts = VolatileBooleanState(prefix)
@ -83,13 +74,10 @@ class ChanState:
t = self._threads["threads"][helper.item_unique_id(thread, board)] t = self._threads["threads"][helper.item_unique_id(thread, board)]
return not t or helper.thread_mtime(thread) != t["m"] or t["t"] + 86400 < int(time.time()) return not t or helper.thread_mtime(thread) != t
def mark_thread_as_visited(self, thread, helper, board): def mark_thread_as_visited(self, thread, helper, board):
self._threads["threads"][helper.item_unique_id(thread, board)] = { self._threads["threads"][helper.item_unique_id(thread, board)] = helper.thread_mtime(thread)
"t": int(time.time()),
"m": helper.thread_mtime(thread)
}
def publish_worker(queue: Queue, helper): def publish_worker(queue: Queue, helper):
@ -100,7 +88,6 @@ def publish_worker(queue: Queue, helper):
logger.error(str(e) + ": " + traceback.format_exc()) logger.error(str(e) + ": " + traceback.format_exc())
@once
def publish(item, board, helper): def publish(item, board, helper):
post_process(item, board, helper) post_process(item, board, helper)