diff --git a/requirements.txt b/requirements.txt index 56040dd..51b8051 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ stem influxdb bs4 urllib3 -git+git://github.com/simon987/hexlib.git git+git://github.com/simon987/vanwanet_scrape.git +git+git://github.com/simon987/hexlib.git cloudscraper redis \ No newline at end of file diff --git a/run.py b/run.py index f61c3f6..f13cbc4 100644 --- a/run.py +++ b/run.py @@ -1,13 +1,12 @@ import json import os -import time import traceback from queue import Queue from threading import Thread from hexlib.concurrency import queue_iter from hexlib.db import VolatileBooleanState, VolatileState -from hexlib.env import get_web, get_redis +from hexlib.env import get_web, get_redis, redis_publish from hexlib.log import logger from chan.chan import CHANS @@ -83,21 +82,20 @@ class ChanState: def publish_worker(queue: Queue, helper): for item, board in queue_iter(queue): try: - publish(item, board, helper) + post_process(item, board, helper) + + redis_publish( + rdb, + item=json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True), + item_project="chan", + item_subproject=CHAN, + item_type=helper.item_type(item), + item_category=board + ) except Exception as e: logger.error(str(e) + ": " + traceback.format_exc()) -def publish(item, board, helper): - post_process(item, board, helper) - - item_type = helper.item_type(item) - routing_key = "%s.%s.%s" % (CHAN, item_type, board) - - message = json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True) - rdb.lpush("arc.chan2." + routing_key, message) - - if __name__ == "__main__": chan_helper = CHANS[CHAN] save_folder = os.environ.get("CF_SAVE_FOLDER", "")