simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 18:36:44 +00:00
Code Issues Projects Releases Wiki Activity

Updated README and catpchas are now toggled from the config

Browse Source
This commit is contained in:
Simon 2018-07-10 22:57:52 -04:00
parent 123f38e65d
commit 1b743e7aba
4 changed files with 58 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -11,16 +11,27 @@ sudo pip3 install -r requirements.txt
``` ```
Create `/config.py` and fill out the parameters. Sample config: Create `/config.py` and fill out the parameters. Sample config:
```python ```python
# Leave default values for no CAPTCHAs
CAPTCHA_LOGIN = False
CAPTCHA_SUBMIT = False
CAPTCHA_SITE_KEY = "" CAPTCHA_SITE_KEY = ""
CAPTCHA_SECRET_KEY = "" CAPTCHA_SECRET_KEY = ""
# Flask secret key for sessions
FLASK_SECRET = "" FLASK_SECRET = ""
RESULTS_PER_PAGE = (25, 50, 100, 250, 500, 1000) RESULTS_PER_PAGE = (25, 50, 100, 250, 500, 1000)
# Headers for http crawler
HEADERS = {} HEADERS = {}
# Token for the crawl server, used by the server to communicate to the crawl server
CRAWL_SERVER_TOKEN = "" CRAWL_SERVER_TOKEN = ""
CRAWL_SERVER_PORT = 5001 CRAWL_SERVER_PORT = 5001
# Number of crawler instances (one per task)
CRAWL_SERVER_PROCESSES = 3 CRAWL_SERVER_PROCESSES = 3
# Number of threads per crawler instance
CRAWL_SERVER_THREADS = 20 CRAWL_SERVER_THREADS = 20
# Allow ftp websites in /submit
SUBMIT_FTP = False SUBMIT_FTP = False
# Allow http(s) websites in /submit
SUBMIT_HTTP = True SUBMIT_HTTP = True
``` ```

68
app.py
View File

@ -13,9 +13,12 @@ from task import TaskDispatcher, Task, CrawlServer
from search.search import ElasticSearchEngine from search.search import ElasticSearchEngine
app = Flask(__name__) app = Flask(__name__)
recaptcha = ReCaptcha(app=app, if config.CAPTCHA_SUBMIT or config.CAPTCHA_LOGIN:
site_key=config.CAPTCHA_SITE_KEY, recaptcha = ReCaptcha(app=app,
secret_key=config.CAPTCHA_SECRET_KEY) site_key=config.CAPTCHA_SITE_KEY,
secret_key=config.CAPTCHA_SECRET_KEY)
else:
recaptcha = None
app.secret_key = config.FLASK_SECRET app.secret_key = config.FLASK_SECRET
db = Database("db.sqlite3") db = Database("db.sqlite3")
cache = Cache(app, config={'CACHE_TYPE': 'simple'}) cache = Cache(app, config={'CACHE_TYPE': 'simple'})
@ -317,7 +320,7 @@ def home():
@app.route("/submit") @app.route("/submit")
def submit(): def submit():
queued_websites = taskDispatcher.get_queued_tasks() queued_websites = taskDispatcher.get_queued_tasks()
return render_template("submit.html", queue=queued_websites, recaptcha=recaptcha) return render_template("submit.html", queue=queued_websites, recaptcha=recaptcha, show_captcha=config.CAPTCHA_SUBMIT)
def try_enqueue(url): def try_enqueue(url):
@ -356,59 +359,56 @@ def try_enqueue(url):
@app.route("/enqueue", methods=["POST"]) @app.route("/enqueue", methods=["POST"])
def enqueue(): def enqueue():
# if recaptcha.verify(): if not config.CAPTCHA_SUBMIT or recaptcha.verify():
url = os.path.join(request.form.get("url"), "") url = os.path.join(request.form.get("url"), "")
message, msg_type = try_enqueue(url) message, msg_type = try_enqueue(url)
flash(message, msg_type) flash(message, msg_type)
return redirect("/submit") return redirect("/submit")
else:
# else: flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
# flash("<strong>Error:</strong> Invalid captcha please try again", "danger") return redirect("/submit")
# return redirect("/submit")
@app.route("/enqueue_bulk", methods=["POST"]) @app.route("/enqueue_bulk", methods=["POST"])
def enqueue_bulk(): def enqueue_bulk():
# if recaptcha.verify(): if not config.CAPTCHA_SUBMIT or recaptcha.verify():
urls = request.form.get("urls") urls = request.form.get("urls")
if urls: if urls:
urls = urls.split() urls = urls.split()
if 0 < len(urls) <= 1000000000000: if 0 < len(urls) <= 1000000000000:
for url in urls: for url in urls:
url = os.path.join(url, "") url = os.path.join(url, "")
message, msg_type = try_enqueue(url) message, msg_type = try_enqueue(url)
message += ' <span class="badge badge-' + msg_type + '">' + url + '</span>' message += ' <span class="badge badge-' + msg_type + '">' + url + '</span>'
flash(message, msg_type) flash(message, msg_type)
return redirect("/submit") return redirect("/submit")
else:
flash("Too few or too many urls, please submit 1-10 urls", "danger")
return redirect("/submit")
else: else:
flash("Too few or too many urls, please submit 1-10 urls", "danger") return abort(500)
return redirect("/submit")
else: else:
return abort(500) flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
return redirect("/submit")
# else:
# flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
# return redirect("/submit")
@app.route("/admin") @app.route("/admin")
def admin_login_form(): def admin_login_form():
if "username" in session: if "username" in session:
return redirect("/dashboard") return redirect("/dashboard")
return render_template("admin.html", recaptcha=recaptcha) return render_template("admin.html", recaptcha=recaptcha, show_captcha=config.CAPTCHA_LOGIN)
@app.route("/login", methods=["POST"]) @app.route("/login", methods=["POST"])
def admin_login(): def admin_login():
if recaptcha.verify(): if not config.CAPTCHA_LOGIN or recaptcha.verify():
username = request.form.get("username") username = request.form.get("username")
password = request.form.get("password") password = request.form.get("password")

4
templates/admin.html
View File

@ -15,7 +15,9 @@
<input class="form-control" name="password" placeholder="Password" type="password"> <input class="form-control" name="password" placeholder="Password" type="password">
</div> </div>
{{ recaptcha.get_code()|safe }} {% if show_captcha %}
{{ recaptcha.get_code()|safe }}
{% endif %}
<input type="submit" value="Login"> <input type="submit" value="Login">

16
templates/submit.html
View File

@ -25,9 +25,11 @@
<input class="form-control" name="url" id="url" placeholder="URL"> <input class="form-control" name="url" id="url" placeholder="URL">
</div> </div>
<div class="row"> <div class="row">
<div class="col"> {% if show_captcha %}
{# {{ recaptcha.get_code()|safe }}#} <div class="col">
</div> {{ recaptcha.get_code()|safe }}
</div>
{% endif %}
<div class="col"> <div class="col">
<input class="btn btn-primary" type="submit" value="Submit" title="Submit open directory"> <input class="btn btn-primary" type="submit" value="Submit" title="Submit open directory">
</div> </div>
@ -42,9 +44,11 @@
<textarea class="form-control" name="urls" id="urls" rows="10" placeholder="One URL per line, max. 10"></textarea> <textarea class="form-control" name="urls" id="urls" rows="10" placeholder="One URL per line, max. 10"></textarea>
</div> </div>
<div class="row"> <div class="row">
<div class="col"> {% if show_captcha %}
{# {{ recaptcha.get_code()|safe }}#} <div class="col">
</div> {{ recaptcha.get_code()|safe }}
</div>
{% endif %}
<div class="col"> <div class="col">
<input class="btn btn-primary" type="submit" value="Submit" title="Submit open directories"> <input class="btn btn-primary" type="submit" value="Submit" title="Submit open directories">
</div> </div>