simon987/chan_feed
1
0
Fork 0
mirror of https://github.com/simon987/chan_feed.git synced 2025-04-10 14:06:42 +00:00
Code Issues Projects Releases Wiki Activity

Use redis_publish function, rename project to 'chan'

Browse Source
This commit is contained in:
simon987 2021-06-03 16:52:17 -04:00
parent 4e0f837cec
commit c09c6c3098
2 changed files with 12 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -4,7 +4,7 @@ stem
influxdb influxdb
bs4 bs4
urllib3 urllib3
git+git://github.com/simon987/hexlib.git
git+git://github.com/simon987/vanwanet_scrape.git git+git://github.com/simon987/vanwanet_scrape.git
git+git://github.com/simon987/hexlib.git
cloudscraper cloudscraper
redis redis

24
run.py
View File

@ -1,13 +1,12 @@
import json import json
import os import os
import time
import traceback import traceback
from queue import Queue from queue import Queue
from threading import Thread from threading import Thread
from hexlib.concurrency import queue_iter from hexlib.concurrency import queue_iter
from hexlib.db import VolatileBooleanState, VolatileState from hexlib.db import VolatileBooleanState, VolatileState
from hexlib.env import get_web, get_redis from hexlib.env import get_web, get_redis, redis_publish
from hexlib.log import logger from hexlib.log import logger
from chan.chan import CHANS from chan.chan import CHANS
@ -83,21 +82,20 @@ class ChanState:
def publish_worker(queue: Queue, helper): def publish_worker(queue: Queue, helper):
for item, board in queue_iter(queue): for item, board in queue_iter(queue):
try: try:
publish(item, board, helper) post_process(item, board, helper)
redis_publish(
rdb,
item=json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True),
item_project="chan",
item_subproject=CHAN,
item_type=helper.item_type(item),
item_category=board
)
except Exception as e: except Exception as e:
logger.error(str(e) + ": " + traceback.format_exc()) logger.error(str(e) + ": " + traceback.format_exc())
def publish(item, board, helper):
post_process(item, board, helper)
item_type = helper.item_type(item)
routing_key = "%s.%s.%s" % (CHAN, item_type, board)
message = json.dumps(item, separators=(',', ':'), ensure_ascii=False, sort_keys=True)
rdb.lpush("arc.chan2." + routing_key, message)
if __name__ == "__main__": if __name__ == "__main__":
chan_helper = CHANS[CHAN] chan_helper = CHANS[CHAN]
save_folder = os.environ.get("CF_SAVE_FOLDER", "") save_folder = os.environ.get("CF_SAVE_FOLDER", "")