simon987/chan_feed
1
0
Fork 0
mirror of https://github.com/simon987/chan_feed.git synced 2025-10-30 22:36:52 +00:00
Code Issues Projects Releases Wiki Activity

Performance improvements

Browse Source
This commit is contained in:
simon987 2020-12-20 20:27:24 -05:00
parent 75f5d79a75
commit 6da2cd54c3
1 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
run.py
View File

@ -8,7 +8,7 @@ from queue import Queue
from threading import Thread from threading import Thread
import redis import redis
from hexlib.db import VolatileBooleanState from hexlib.db import VolatileBooleanState, VolatileState
from hexlib.monitoring import Monitoring from hexlib.monitoring import Monitoring
from chan.chan import CHANS from chan.chan import CHANS
@ -27,6 +27,7 @@ else:
REDIS_HOST = os.environ.get("CF_REDIS_HOST", "localhost") REDIS_HOST = os.environ.get("CF_REDIS_HOST", "localhost")
REDIS_PORT = os.environ.get("CF_REDIS_PORT", 6379) REDIS_PORT = os.environ.get("CF_REDIS_PORT", 6379)
CHAN = os.environ.get("CF_CHAN", None) CHAN = os.environ.get("CF_CHAN", None)
CF_PUBLISH = os.environ.get("CF_PUBLISH", False)
ARC_LISTS = os.environ.get("CF_ARC_LISTS", "arc").split(",") ARC_LISTS = os.environ.get("CF_ARC_LISTS", "arc").split(",")
@ -82,30 +83,29 @@ def once(func):
class ChanState: class ChanState:
def __init__(self, prefix): def __init__(self, prefix):
self._state = VolatileBooleanState(prefix, host=REDIS_HOST, port=REDIS_PORT) self._posts = VolatileBooleanState(prefix, host=REDIS_HOST, port=REDIS_PORT)
self._threads = VolatileState(prefix, host=REDIS_HOST, port=REDIS_PORT)
print("redis host=" + REDIS_HOST) print("redis host=" + REDIS_HOST)
def mark_visited(self, item: int): def mark_visited(self, item: int):
self._state["posts"][item] = True self._posts["posts"][item] = True
def has_visited(self, item: int): def has_visited(self, item: int):
return self._state["posts"][item] is not None return self._posts["posts"][item]
def has_new_posts(self, thread, helper, board): def has_new_posts(self, thread, helper, board):
mtime = helper.thread_mtime(thread) mtime = helper.thread_mtime(thread)
if mtime == -1: if mtime == -1:
return True return True
t = self._state["threads"][helper.item_unique_id(thread, board)] t = self._threads["threads"][helper.item_unique_id(thread, board)]
if not t or helper.thread_mtime(thread) != t["last_modified"] or t["ts"] + 86400 < int(time.time()): return not t or helper.thread_mtime(thread) != t["m"] or t["t"] + 86400 < int(time.time())
return True
return False
def mark_thread_as_visited(self, thread, helper, board): def mark_thread_as_visited(self, thread, helper, board):
self._state["threads"][helper.item_unique_id(thread, board)] = { self._threads["threads"][helper.item_unique_id(thread, board)] = {
"ts": time.time(), "t": int(time.time()),
"last_modified": helper.thread_mtime(thread) "m": helper.thread_mtime(thread)
} }
@ -131,7 +131,8 @@ def publish(item, board, helper):
routing_key = "%s.%s.%s" % (CHAN, item_type, board) routing_key = "%s.%s.%s" % (CHAN, item_type, board)
message = json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True) message = json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True)
rdb.publish("chan." + routing_key, message) if CF_PUBLISH:
rdb.publish("chan." + routing_key, message)
for arc in ARC_LISTS: for arc in ARC_LISTS:
rdb.lpush(arc + ".chan." + routing_key, message) rdb.lpush(arc + ".chan." + routing_key, message)