chan/plus4chan_html.py

@@ -45,7 +45,7 @@ class Plus4ChanHelper(ChanHelper):
 
             threads.append({
                 "id": int(threadEl.get("id")[1:]),
-                "omit": int(omit.text.strip().split(" ")[1]) if omit else 0
+                "omit": int(omit.text.split(" ")[1]) if omit else 0
             })
 
         for link in soup.find_all("a", href=lambda x: x):

chan/zerochan_html.py

@@ -78,5 +78,5 @@ class ZerochanHtmlChanHelper(DoushioHtmlChanHelper):
             "type": "thread",
             "html": str(op_el),
             "time": int(datetime.datetime.strptime(_ru_datefmt(op_el.find("time").text),
-                                                   "%d %b %Y %H:%M:%S").timestamp())
+                                                   "%d %b %Y %H:%M").timestamp())
         }

run.py

@@ -8,7 +8,7 @@ from queue import Queue
 from threading import Thread
 import redis
 
-from hexlib.db import VolatileBooleanState, VolatileState
+from hexlib.db import VolatileState
 from hexlib.monitoring import Monitoring
 
 from chan.chan import CHANS
@@ -27,9 +27,10 @@ else:
 REDIS_HOST = os.environ.get("CF_REDIS_HOST", "localhost")
 REDIS_PORT = os.environ.get("CF_REDIS_PORT", 6379)
 CHAN = os.environ.get("CF_CHAN", None)
-CF_PUBLISH = os.environ.get("CF_PUBLISH", False)
 
-ARC_LISTS = os.environ.get("CF_ARC_LISTS", "arc").split(",")
+ARC_LISTS = os.environ.get("CF_ARC_LISTS", "arc,imhash").split(",")
+
+PUB_CHANNEL = os.environ.get("CF_PUB_CHANNEL", "chan_feed")
 
 
 class ChanScanner:
@@ -83,29 +84,30 @@ def once(func):
 
 class ChanState:
     def __init__(self, prefix):
-        self._posts = VolatileBooleanState(prefix, host=REDIS_HOST, port=REDIS_PORT)
+        self._state = VolatileState(prefix, 86400 * 7, host=REDIS_HOST, port=REDIS_PORT)
-        self._threads = VolatileState(prefix, host=REDIS_HOST, port=REDIS_PORT)
         print("redis host=" + REDIS_HOST)
 
     def mark_visited(self, item: int):
-        self._posts["posts"][item] = True
+        self._state["posts"][item] = 1
 
     def has_visited(self, item: int):
-        return self._posts["posts"][item]
+        return self._state["posts"][item] is not None
 
     def has_new_posts(self, thread, helper, board):
         mtime = helper.thread_mtime(thread)
         if mtime == -1:
             return True
 
-        t = self._threads["threads"][helper.item_unique_id(thread, board)]
+        t = self._state["threads"][helper.item_unique_id(thread, board)]
 
-        return not t or helper.thread_mtime(thread) != t["m"] or t["t"] + 86400 < int(time.time())
+        if not t or helper.thread_mtime(thread) != t["last_modified"] or t["ts"] + 86400 < int(time.time()):
+            return True
+        return False
 
     def mark_thread_as_visited(self, thread, helper, board):
-        self._threads["threads"][helper.item_unique_id(thread, board)] = {
+        self._state["threads"][helper.item_unique_id(thread, board)] = {
-            "t": int(time.time()),
+            "ts": time.time(),
-            "m": helper.thread_mtime(thread)
+            "last_modified": helper.thread_mtime(thread)
         }
 
 
@@ -131,8 +133,7 @@ def publish(item, board, helper):
     routing_key = "%s.%s.%s" % (CHAN, item_type, board)
 
     message = json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True)
-    if CF_PUBLISH:
+    rdb.publish("chan." + routing_key, message)
-        rdb.publish("chan." + routing_key, message)
     for arc in ARC_LISTS:
         rdb.lpush(arc + ".chan." + routing_key, message)