simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 10:56:47 +00:00
Code Issues Projects Releases Wiki Activity

Multi threading for bulk enqueue

Browse Source
This commit is contained in:
Simon 2018-07-15 12:50:26 -04:00
parent 08c3e119f0
commit c35491cb15
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app.py
View File

@ -1,4 +1,5 @@
from flask import Flask, render_template, redirect, request, flash, abort, Response, send_from_directory, session from flask import Flask, render_template, redirect, request, flash, abort, Response, send_from_directory, session
from multiprocessing import Pool
import json import json
from urllib.parse import urlparse from urllib.parse import urlparse
import os import os
@ -386,6 +387,12 @@ def enqueue():
return redirect("/submit") return redirect("/submit")
def check_url(url):
url = os.path.join(url, "")
try_enqueue(url)
return None
@app.route("/enqueue_bulk", methods=["POST"]) @app.route("/enqueue_bulk", methods=["POST"])
def enqueue_bulk(): def enqueue_bulk():
if not config.CAPTCHA_SUBMIT or recaptcha.verify(): if not config.CAPTCHA_SUBMIT or recaptcha.verify():
@ -396,11 +403,12 @@ def enqueue_bulk():
if 0 < len(urls) <= 1000000000000: if 0 < len(urls) <= 1000000000000:
for url in urls: pool = Pool(processes=4)
url = os.path.join(url, "") pool.map(func=check_url, iterable=urls)
message, msg_type = try_enqueue(url) pool.close()
message += ' <span class="badge badge-' + msg_type + '">' + url + '</span>'
flash(message, msg_type) flash("Submitted websites to the queue", "success")
return redirect("/submit") return redirect("/submit")
else: else: