simon987/reddit_feed
1
0
Fork 0
mirror of https://github.com/simon987/reddit_feed.git synced 2025-04-24 12:15:51 +00:00
Code Issues Projects Releases Wiki Activity

prefix underscore in post_process step

Browse Source
This commit is contained in:
simon 2019-09-10 15:56:58 -04:00
parent 6f4b19538b
commit cc4408dbbf
2 changed files with 19 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
post_process.py
View File

@ -6,7 +6,7 @@ INTERNAL_RE = re.compile(r"^https?://(reddit.com|redd.it|old.reddit.com|www.redd
def post_process(thing): def post_process(thing):
thing["v"] = 1.2 thing["_v"] = 1.2
urls = set() urls = set()
@ -23,7 +23,7 @@ def post_process(thing):
if "url" in thing and thing["url"] and is_external(thing["url"]): if "url" in thing and thing["url"] and is_external(thing["url"]):
urls.add(thing["url"]) urls.add(thing["url"])
thing["urls"] = list(urls) thing["_urls"] = list(urls)
return thing return thing
@ -40,15 +40,11 @@ def get_links_from_body_html(body):
def get_links_from_body(body): def get_links_from_body(body):
result = set()
body = body.replace("\\)", "(") body = body.replace("\\)", "(")
for match in LINK_RE.finditer(body): for match in LINK_RE.finditer(body):
url = match.group(1) url = match.group(1)
if is_external(url): if is_external(url):
result.add(url) yield url
return list(result)
def is_external(url): def is_external(url):

36
run.py
View File

@ -198,25 +198,21 @@ if __name__ == "__main__":
if MONITORING: if MONITORING:
monitoring.init() monitoring.init()
pub_queue = Queue() pub_queue = Queue()
while True: try:
try: publish_thread = threading.Thread(target=publish_worker, args=(pub_queue,))
publish_thread = threading.Thread(target=publish_worker, args=(pub_queue,)) if MONITORING:
if MONITORING: monitoring_queue = Queue()
monitoring_queue = Queue() log_thread = threading.Thread(target=mon_worker, args=(monitoring_queue,))
log_thread = threading.Thread(target=mon_worker, args=(monitoring_queue,)) log_thread.start()
log_thread.start()
comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue, monitoring_queue)) comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue, monitoring_queue))
post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue, monitoring_queue)) post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue, monitoring_queue))
else: else:
comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue)) comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue))
post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue)) post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue))
comment_thread.start() comment_thread.start()
post_thread.start() post_thread.start()
publish_thread.start() publish_thread.start()
except Exception as e: except Exception as e:
logger.error(str(e) + ": " + traceback.format_exc()) logger.error(str(e) + ": " + traceback.format_exc())
while True:
time.sleep(10)