From 4e0f837cec4d7c3f8826d8742f9b3365d07873a9 Mon Sep 17 00:00:00 2001
From: simon987 <me@simon987.net>
Date: Sat, 27 Mar 2021 15:02:28 -0400
Subject: [PATCH] bug fix, change caching
---
chan/chan_json.py | 2 +-
run.py | 17 ++---------------
2 files changed, 3 insertions(+), 16 deletions(-)
diff --git a/chan/chan_json.py b/chan/chan_json.py
index a8179f5..e8cb01e 100644
--- a/chan/chan_json.py
+++ b/chan/chan_json.py
@@ -35,7 +35,7 @@ class JsonChanHelper(ChanHelper):
@staticmethod
def thread_mtime(thread):
- return thread["last_modified"]
+ return thread["last_modified"] if "last_modified" in thread else -1
@staticmethod
def parse_threads_list(r):
diff --git a/run.py b/run.py
index 238fd20..f61c3f6 100644
--- a/run.py
+++ b/run.py
@@ -56,15 +56,6 @@ class ChanScanner:
yield post, board
-def once(func):
- def wrapper(item, board, helper):
- if not state.has_visited(helper.item_unique_id(item, board)):
- func(item, board, helper)
- state.mark_visited(helper.item_unique_id(item, board))
-
- return wrapper
-
-
class ChanState:
def __init__(self, prefix):
self._posts = VolatileBooleanState(prefix)
@@ -83,13 +74,10 @@ class ChanState:
t = self._threads["threads"][helper.item_unique_id(thread, board)]
- return not t or helper.thread_mtime(thread) != t["m"] or t["t"] + 86400 < int(time.time())
+ return not t or helper.thread_mtime(thread) != t
def mark_thread_as_visited(self, thread, helper, board):
- self._threads["threads"][helper.item_unique_id(thread, board)] = {
- "t": int(time.time()),
- "m": helper.thread_mtime(thread)
- }
+ self._threads["threads"][helper.item_unique_id(thread, board)] = helper.thread_mtime(thread)
def publish_worker(queue: Queue, helper):
@@ -100,7 +88,6 @@ def publish_worker(queue: Queue, helper):
logger.error(str(e) + ": " + traceback.format_exc())
-@once
def publish(item, board, helper):
post_process(item, board, helper)