From c09c6c3098ae49f39aab33790919107d16bb2d59 Mon Sep 17 00:00:00 2001 From: simon987 Date: Thu, 3 Jun 2021 16:52:17 -0400 Subject: [PATCH] Use redis_publish function, rename project to 'chan' --- requirements.txt | 2 +- run.py | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/requirements.txt b/requirements.txt index 56040dd..51b8051 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ stem influxdb bs4 urllib3 -git+git://github.com/simon987/hexlib.git git+git://github.com/simon987/vanwanet_scrape.git +git+git://github.com/simon987/hexlib.git cloudscraper redis \ No newline at end of file diff --git a/run.py b/run.py index f61c3f6..f13cbc4 100644 --- a/run.py +++ b/run.py @@ -1,13 +1,12 @@ import json import os -import time import traceback from queue import Queue from threading import Thread from hexlib.concurrency import queue_iter from hexlib.db import VolatileBooleanState, VolatileState -from hexlib.env import get_web, get_redis +from hexlib.env import get_web, get_redis, redis_publish from hexlib.log import logger from chan.chan import CHANS @@ -83,21 +82,20 @@ class ChanState: def publish_worker(queue: Queue, helper): for item, board in queue_iter(queue): try: - publish(item, board, helper) + post_process(item, board, helper) + + redis_publish( + rdb, + item=json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True), + item_project="chan", + item_subproject=CHAN, + item_type=helper.item_type(item), + item_category=board + ) except Exception as e: logger.error(str(e) + ": " + traceback.format_exc()) -def publish(item, board, helper): - post_process(item, board, helper) - - item_type = helper.item_type(item) - routing_key = "%s.%s.%s" % (CHAN, item_type, board) - - message = json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True) - rdb.lpush("arc.chan2." + routing_key, message) - - if __name__ == "__main__": chan_helper = CHANS[CHAN] save_folder = os.environ.get("CF_SAVE_FOLDER", "")