simon987/reddit_feed
1
0
Fork 0
mirror of https://github.com/simon987/reddit_feed.git synced 2025-04-24 12:15:51 +00:00
Code Issues Projects Releases Wiki Activity

prefix underscore in post_process step

Browse Source
This commit is contained in:
simon 2019-09-10 15:56:58 -04:00
parent 6f4b19538b
commit cc4408dbbf
2 changed files with 19 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
post_process.py
View File

@ -6,7 +6,7 @@ INTERNAL_RE = re.compile(r"^https?://(reddit.com|redd.it|old.reddit.com|www.redd
def post_process(thing):
thing["v"] = 1.2
thing["_v"] = 1.2
urls = set()
@ -23,7 +23,7 @@ def post_process(thing):
if "url" in thing and thing["url"] and is_external(thing["url"]):
urls.add(thing["url"])
thing["urls"] = list(urls)
thing["_urls"] = list(urls)
return thing
@ -40,15 +40,11 @@ def get_links_from_body_html(body):
def get_links_from_body(body):
result = set()
body = body.replace("\\)", "(")
for match in LINK_RE.finditer(body):
url = match.group(1)
if is_external(url):
result.add(url)
return list(result)
yield url
def is_external(url):

36
run.py
View File

@ -198,25 +198,21 @@ if __name__ == "__main__":
if MONITORING:
monitoring.init()
pub_queue = Queue()
while True:
try:
publish_thread = threading.Thread(target=publish_worker, args=(pub_queue,))
if MONITORING:
monitoring_queue = Queue()
log_thread = threading.Thread(target=mon_worker, args=(monitoring_queue,))
log_thread.start()
try:
publish_thread = threading.Thread(target=publish_worker, args=(pub_queue,))
if MONITORING:
monitoring_queue = Queue()
log_thread = threading.Thread(target=mon_worker, args=(monitoring_queue,))
log_thread.start()
comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue, monitoring_queue))
post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue, monitoring_queue))
else:
comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue))
post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue))
comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue, monitoring_queue))
post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue, monitoring_queue))
else:
comment_thread = threading.Thread(target=stream_thing, args=("t1_", pub_queue))
post_thread = threading.Thread(target=stream_thing, args=("t3_", pub_queue))
comment_thread.start()
post_thread.start()
publish_thread.start()
except Exception as e:
logger.error(str(e) + ": " + traceback.format_exc())
while True:
time.sleep(10)
comment_thread.start()
post_thread.start()
publish_thread.start()
except Exception as e:
logger.error(str(e) + ": " + traceback.format_exc())