From cc4408dbbf11c461f22157b8b4c1a22037802e84 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 10 Sep 2019 15:56:58 -0400 Subject: [PATCH] prefix underscore in post_process step --- post_process.py | 10 +++------- run.py | 36 ++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/post_process.py b/post_process.py index 43bbe07..f42afe6 100644 --- a/post_process.py +++ b/post_process.py @@ -6,7 +6,7 @@ INTERNAL_RE = re.compile(r"^https?://(reddit.com|redd.it|old.reddit.com|www.redd def post_process(thing): - thing["v"] = 1.2 + thing["_v"] = 1.2 urls = set() @@ -23,7 +23,7 @@ def post_process(thing): if "url" in thing and thing["url"] and is_external(thing["url"]): urls.add(thing["url"]) - thing["urls"] = list(urls) + thing["_urls"] = list(urls) return thing @@ -40,15 +40,11 @@ def get_links_from_body_html(body): def get_links_from_body(body): - result = set() - body = body.replace("\\)", "(") for match in LINK_RE.finditer(body): url = match.group(1) if is_external(url): - result.add(url) - - return list(result) + yield url def is_external(url): diff --git a/run.py b/run.py index 1c2f091..5216dce 100755 --- a/run.py +++ b/run.py @@ -198,25 +198,21 @@ if __name__ == "__main__": if MONITORING: monitoring.init() pub_queue = Queue() - while True: - try: - publish_thread = threading.Thread(target=publish_worker, args=(pub_queue,)) - if MONITORING: - monitoring_queue = Queue() - log_thread = threading.Thread(target=mon_worker, args=(monitoring_queue,)) - log_thread.start() + try: + publish_thread = threading.Thread(target=publish_worker, args=(pub_queue,)) + if MONITORING: + monitoring_queue = Queue() + log_thread = threading.Thread(target=mon_worker, args=(monitoring_queue,)) + log_thread.start() - comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue, monitoring_queue)) - post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue, monitoring_queue)) - else: - comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue)) - post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue)) + comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue, monitoring_queue)) + post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue, monitoring_queue)) + else: + comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue)) + post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue)) - comment_thread.start() - post_thread.start() - publish_thread.start() - except Exception as e: - logger.error(str(e) + ": " + traceback.format_exc()) - - while True: - time.sleep(10) + comment_thread.start() + post_thread.start() + publish_thread.start() + except Exception as e: + logger.error(str(e) + ": " + traceback.format_exc())