simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 02:46:45 +00:00
Code Issues Projects Releases Wiki Activity

Updated README and catpchas are now toggled from the config

Browse Source
This commit is contained in:
Simon 2018-07-10 22:57:52 -04:00
parent 123f38e65d
commit 1b743e7aba
4 changed files with 58 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -11,16 +11,27 @@ sudo pip3 install -r requirements.txt
```
Create `/config.py` and fill out the parameters. Sample config:
```python
# Leave default values for no CAPTCHAs
CAPTCHA_LOGIN = False
CAPTCHA_SUBMIT = False
CAPTCHA_SITE_KEY = ""
CAPTCHA_SECRET_KEY = ""
# Flask secret key for sessions
FLASK_SECRET = ""
RESULTS_PER_PAGE = (25, 50, 100, 250, 500, 1000)
# Headers for http crawler
HEADERS = {}
# Token for the crawl server, used by the server to communicate to the crawl server
CRAWL_SERVER_TOKEN = ""
CRAWL_SERVER_PORT = 5001
# Number of crawler instances (one per task)
CRAWL_SERVER_PROCESSES = 3
# Number of threads per crawler instance
CRAWL_SERVER_THREADS = 20
# Allow ftp websites in /submit
SUBMIT_FTP = False
# Allow http(s) websites in /submit
SUBMIT_HTTP = True
```

68
app.py
View File

@ -13,9 +13,12 @@ from task import TaskDispatcher, Task, CrawlServer
from search.search import ElasticSearchEngine
app = Flask(__name__)
recaptcha = ReCaptcha(app=app,
site_key=config.CAPTCHA_SITE_KEY,
secret_key=config.CAPTCHA_SECRET_KEY)
if config.CAPTCHA_SUBMIT or config.CAPTCHA_LOGIN:
recaptcha = ReCaptcha(app=app,
site_key=config.CAPTCHA_SITE_KEY,
secret_key=config.CAPTCHA_SECRET_KEY)
else:
recaptcha = None
app.secret_key = config.FLASK_SECRET
db = Database("db.sqlite3")
cache = Cache(app, config={'CACHE_TYPE': 'simple'})
@ -317,7 +320,7 @@ def home():
@app.route("/submit")
def submit():
queued_websites = taskDispatcher.get_queued_tasks()
return render_template("submit.html", queue=queued_websites, recaptcha=recaptcha)
return render_template("submit.html", queue=queued_websites, recaptcha=recaptcha, show_captcha=config.CAPTCHA_SUBMIT)
def try_enqueue(url):
@ -356,59 +359,56 @@ def try_enqueue(url):
@app.route("/enqueue", methods=["POST"])
def enqueue():
# if recaptcha.verify():
if not config.CAPTCHA_SUBMIT or recaptcha.verify():
url = os.path.join(request.form.get("url"), "")
message, msg_type = try_enqueue(url)
flash(message, msg_type)
url = os.path.join(request.form.get("url"), "")
message, msg_type = try_enqueue(url)
flash(message, msg_type)
return redirect("/submit")
return redirect("/submit")
# else:
# flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
# return redirect("/submit")
else:
flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
return redirect("/submit")
@app.route("/enqueue_bulk", methods=["POST"])
def enqueue_bulk():
# if recaptcha.verify():
if not config.CAPTCHA_SUBMIT or recaptcha.verify():
urls = request.form.get("urls")
if urls:
urls = urls.split()
urls = request.form.get("urls")
if urls:
urls = urls.split()
if 0 < len(urls) <= 1000000000000:
if 0 < len(urls) <= 1000000000000:
for url in urls:
url = os.path.join(url, "")
message, msg_type = try_enqueue(url)
message += ' <span class="badge badge-' + msg_type + '">' + url + '</span>'
flash(message, msg_type)
return redirect("/submit")
for url in urls:
url = os.path.join(url, "")
message, msg_type = try_enqueue(url)
message += ' <span class="badge badge-' + msg_type + '">' + url + '</span>'
flash(message, msg_type)
return redirect("/submit")
else:
flash("Too few or too many urls, please submit 1-10 urls", "danger")
return redirect("/submit")
else:
flash("Too few or too many urls, please submit 1-10 urls", "danger")
return redirect("/submit")
return abort(500)
else:
return abort(500)
# else:
# flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
# return redirect("/submit")
flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
return redirect("/submit")
@app.route("/admin")
def admin_login_form():
if "username" in session:
return redirect("/dashboard")
return render_template("admin.html", recaptcha=recaptcha)
return render_template("admin.html", recaptcha=recaptcha, show_captcha=config.CAPTCHA_LOGIN)
@app.route("/login", methods=["POST"])
def admin_login():
if recaptcha.verify():
if not config.CAPTCHA_LOGIN or recaptcha.verify():
username = request.form.get("username")
password = request.form.get("password")

4
templates/admin.html
View File

@ -15,7 +15,9 @@
<input class="form-control" name="password" placeholder="Password" type="password">
</div>
{{ recaptcha.get_code()|safe }}
{% if show_captcha %}
{{ recaptcha.get_code()|safe }}
{% endif %}
<input type="submit" value="Login">

16
templates/submit.html
View File

@ -25,9 +25,11 @@
<input class="form-control" name="url" id="url" placeholder="URL">
</div>
<div class="row">
<div class="col">
{# {{ recaptcha.get_code()|safe }}#}
</div>
{% if show_captcha %}
<div class="col">
{{ recaptcha.get_code()|safe }}
</div>
{% endif %}
<div class="col">
<input class="btn btn-primary" type="submit" value="Submit" title="Submit open directory">
</div>
@ -42,9 +44,11 @@
<textarea class="form-control" name="urls" id="urls" rows="10" placeholder="One URL per line, max. 10"></textarea>
</div>
<div class="row">
<div class="col">
{# {{ recaptcha.get_code()|safe }}#}
</div>
{% if show_captcha %}
<div class="col">
{{ recaptcha.get_code()|safe }}
</div>
{% endif %}
<div class="col">
<input class="btn btn-primary" type="submit" value="Submit" title="Submit open directories">
</div>