Use redis_publish function, rename project to 'chan'

This commit is contained in:
simon987 2021-06-03 16:52:17 -04:00
parent 4e0f837cec
commit c09c6c3098
2 changed files with 12 additions and 14 deletions

View File

@ -4,7 +4,7 @@ stem
influxdb
bs4
urllib3
git+git://github.com/simon987/hexlib.git
git+git://github.com/simon987/vanwanet_scrape.git
git+git://github.com/simon987/hexlib.git
cloudscraper
redis

24
run.py
View File

@ -1,13 +1,12 @@
import json
import os
import time
import traceback
from queue import Queue
from threading import Thread
from hexlib.concurrency import queue_iter
from hexlib.db import VolatileBooleanState, VolatileState
from hexlib.env import get_web, get_redis
from hexlib.env import get_web, get_redis, redis_publish
from hexlib.log import logger
from chan.chan import CHANS
@ -83,21 +82,20 @@ class ChanState:
def publish_worker(queue: Queue, helper):
for item, board in queue_iter(queue):
try:
publish(item, board, helper)
post_process(item, board, helper)
redis_publish(
rdb,
item=json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True),
item_project="chan",
item_subproject=CHAN,
item_type=helper.item_type(item),
item_category=board
)
except Exception as e:
logger.error(str(e) + ": " + traceback.format_exc())
def publish(item, board, helper):
post_process(item, board, helper)
item_type = helper.item_type(item)
routing_key = "%s.%s.%s" % (CHAN, item_type, board)
message = json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True)
rdb.lpush("arc.chan2." + routing_key, message)
if __name__ == "__main__":
chan_helper = CHANS[CHAN]
save_folder = os.environ.get("CF_SAVE_FOLDER", "")