import json import os from multiprocessing.pool import Pool from urllib.parse import urlparse from flask import render_template, redirect, request, flash, abort, Response, session from flask_caching import Cache import captcha import config import od_util from common import db, taskManager, searchEngine, logger, require_role from database import Website from search.search import InvalidQueryException from tasks import Task def setup_views(app): cache = Cache(app, config={ "CACHE_TYPE": "redis", "CACHE_REDIS_HOST": config.REDIS_HOST, "CACHE_REDIS_PORT": config.REDIS_PORT, }) @app.route("/dl") @cache.cached(120) def downloads(): # Get content of downloads directory dl_dir = "static/downloads/" dir_content = os.listdir(dl_dir) # Make paths relative to working directory # Only allow csv files files = [ (name, os.path.join(dl_dir, name)) for name in dir_content if name.find(".csv") != -1 ] # Stat files # Remove any dirs placed accidentally files = [ (f, full, os.stat(full)) for f, full in files if os.path.isfile(full) ] if len(files) == 0: logger.warning("No export file to display in /dl") return render_template("downloads.html", export_file_stats=files) @app.route("/stats") @cache.cached(120) def stats_page(): return render_template("stats.html") @app.route("/stats/json_chart") @cache.cached(240) def stats_json(): stats = searchEngine.get_global_stats() if stats: db.join_website_on_stats(stats) return Response(json.dumps(stats), mimetype="application/json") return abort(500) @app.route("/website//") def website_info(website_id): website = db.get_website_by_id(website_id) if website: return render_template("website.html", website=website) else: abort(404) @app.route("/website//json_chart") @cache.memoize(60) def website_json_chart(website_id): website = db.get_website_by_id(website_id) if website: stats = searchEngine.get_stats(website_id) stats["base_url"] = website.url stats["report_time"] = website.last_modified return Response(json.dumps(stats), mimetype="application/json") else: abort(404) @app.route("/website//links") def website_links(website_id): website = db.get_website_by_id(website_id) if website: links = searchEngine.get_link_list(website_id, website.url) return Response("\n".join(links), mimetype="text/plain") else: abort(404) @app.route("/website/") def websites(): page = int(request.args.get("p")) if "p" in request.args else 0 url = request.args.get("url") if "url" in request.args else "" if url: parsed_url = urlparse(url) if parsed_url.scheme: search_term = (parsed_url.scheme + "://" + parsed_url.netloc) else: flash("Sorry, I was not able to parse this url format. " "Make sure you include the appropriate scheme (http/https/ftp)", "warning") search_term = "" else: search_term = url return render_template("websites.html", websites=db.get_websites(50, page, search_term), p=page, url=search_term, per_page=50) @app.route("/website/random") def random_website(): rand_id = db.get_random_website_id() if rand_id: return redirect("/website/" + str()) return redirect("/website/") @app.route("/website//clear") def admin_clear_website(website_id): require_role("admin") searchEngine.delete_docs(website_id) flash("Cleared all documents associated with this website", "success") return redirect("/website/" + str(website_id)) @app.route("/website//delete") def admin_delete_website(website_id): require_role("admin") searchEngine.delete_docs(website_id) db.delete_website(website_id) flash("Deleted website " + str(website_id), "success") return redirect("/website/") @app.route("/website//rescan") def admin_rescan_website(website_id): require_role("admin") website = db.get_website_by_id(website_id) if website: priority = request.args.get("priority") if "priority" in request.args else 1 task = Task(website_id, website.url, priority) taskManager.queue_task(task) flash("Enqueued rescan task", "success") else: flash("Website does not exist", "danger") return redirect("/website/" + str(website_id)) @app.route("/search") def search(): results = 0 q = request.args.get("q") if "q" in request.args else "" sort_order = request.args.get("sort_order") if "sort_order" in request.args else "score" page = request.args.get("p") if "p" in request.args else "0" page = int(page) if page.isdigit() else 0 per_page = request.args.get("per_page") if "per_page" in request.args else "50" per_page = int(per_page) if per_page.isdigit() else "50" per_page = per_page if per_page in config.RESULTS_PER_PAGE else 50 extensions = request.args.get("ext") if "ext" in request.args else None extensions = [ext.strip().strip(".").lower() for ext in extensions.split(",")] if extensions else [] size_min = request.args.get("size_min") if "size_min" in request.args else "size_min" size_min = int(size_min) if size_min.isdigit() else 0 size_max = request.args.get("size_max") if "size_max" in request.args else "size_max" size_max = int(size_max) if size_max.isdigit() else 0 date_min = request.args.get("date_min") if "date_min" in request.args else "date_min" date_min = int(date_min) if date_min.isdigit() else 0 date_max = request.args.get("date_max") if "date_max" in request.args else "date_max" date_max = int(date_max) if date_max.isdigit() else 0 match_all = "all" in request.args field_name = "field_name" in request.args field_trigram = "field_trigram" in request.args field_path = "field_path" in request.args if not field_name and not field_trigram and not field_path: # If no fields are selected, search in all field_name = field_path = field_trigram = True fields = [] if field_path: fields.append("path") if field_name: fields.append("name^5") if field_trigram: fields.append("name.nGram^2") if len(q) >= 3: blocked = False hits = None if not config.CAPTCHA_SEARCH or captcha.verify(): try: hits = searchEngine.search(q, page, per_page, sort_order, extensions, size_min, size_max, match_all, fields, date_min, date_max) hits = db.join_website_on_search_result(hits) except InvalidQueryException as e: flash("Invalid query: " + str(e), "warning") blocked = True except: flash("Query failed, this could mean that the search server is overloaded or is not reachable. " "Please try again later", "danger") results = hits["hits"]["total"]["value"] if not isinstance(hits["hits"]["total"], int) else \ hits["hits"]["total"] if hits else -1 took = hits["took"] if hits else -1 forwarded_for = request.headers["X-Forwarded-For"] if "X-Forwarded-For" in request.headers else None logger.info("SEARCH '{}' [res={}, t={}, p={}x{}, ext={}] by {}{}" .format(q, results, took, page, per_page, str(extensions), request.remote_addr, "_" + forwarded_for if forwarded_for else "")) db.log_search(request.remote_addr, forwarded_for, q, extensions, page, blocked, results, took) if blocked: return redirect("/search") else: flash("Error: Invalid captcha please try again", "danger") else: hits = None return render_template("search.html", count=results, results=hits, q=q, p=page, per_page=per_page, sort_order=sort_order, results_set=config.RESULTS_PER_PAGE, extensions=",".join(extensions), size_min=size_min, size_max=size_max, match_all=match_all, field_trigram=field_trigram, field_path=field_path, field_name=field_name, date_min=date_min, date_max=date_max, show_captcha=config.CAPTCHA_SEARCH, captcha=captcha) @app.route("/contribute") @cache.cached(600) def contribute(): return render_template("contribute.html") @app.route("/") def home(): try: stats = searchEngine.get_global_stats() stats["website_count"] = len(db.get_all_websites()) except: stats = {} return render_template("home.html", stats=stats, show_captcha=config.CAPTCHA_SEARCH, captcha=captcha) @app.route("/submit") def submit(): return render_template("submit.html", captcha=captcha, show_captcha=config.CAPTCHA_SUBMIT) def try_enqueue(url): url = os.path.join(url, "") url = od_util.get_top_directory(url) if not od_util.is_valid_url(url): return "Error: Invalid url. Make sure to include the appropriate scheme.", "warning" website = db.get_website_by_url(url) if website: return "Website already exists", "danger" website = db.website_exists(url) if website: return "A parent directory of this url has already been posted", "danger" if db.is_blacklisted(url): return "Error: " \ "Sorry, this website has been blacklisted. If you think " \ "this is an error, please contact me.", "danger" if not od_util.is_od(url): return "Error:" \ "The anti-spam algorithm determined that the submitted url is not " \ "an open directory or the server is not responding. If you think " \ "this is an error, please contact me.", "danger" website_id = db.insert_website(Website(url, str(request.remote_addr + "_" + request.headers.get("X-Forwarded-For", "")), request.user_agent)) task = Task(website_id, url, priority=1) taskManager.queue_task(task) return "The website has been added to the queue", "success" @app.route("/enqueue", methods=["POST"]) def enqueue(): if not config.CAPTCHA_SUBMIT or captcha.verify(): url = os.path.join(request.form.get("url"), "") message, msg_type = try_enqueue(url) flash(message, msg_type) return redirect("/submit") else: flash("Error: Invalid captcha please try again", "danger") return redirect("/submit") def check_url(url): url = os.path.join(url, "") try_enqueue(url) return None @app.route("/enqueue_bulk", methods=["POST"]) def enqueue_bulk(): if not config.CAPTCHA_SUBMIT or captcha.verify(): urls = request.form.get("urls") if urls: urls = urls.split() if 0 < len(urls) <= 1000: # TODO: Load from config & adjust placeholder/messages? pool = Pool(processes=6) pool.map(func=check_url, iterable=urls) pool.close() flash("Submitted websites to the queue", "success") return redirect("/submit") else: flash("Too few or too many urls, please submit 1-10 urls", "danger") return redirect("/submit") else: flash("Too few or too many urls, please submit 1-10 urls", "danger") return redirect("/submit") else: flash("Error: Invalid captcha please try again", "danger") return redirect("/submit") @app.route("/admin") def admin_login_form(): if "username" in session: return redirect("/dashboard") return render_template("admin.html", captcha=captcha, show_captcha=config.CAPTCHA_LOGIN) @app.route("/login", methods=["POST"]) def admin_login(): if not config.CAPTCHA_LOGIN or captcha.verify(): username = request.form.get("username") password = request.form.get("password") if db.check_login(username, password): session["username"] = username flash("Logged in", "success") return redirect("/dashboard") flash("Invalid username/password combo", "danger") return redirect("/admin") else: flash("Invalid captcha", "danger") return redirect("/admin") @app.route("/logout") def admin_logout(): session.clear() flash("Logged out", "info") return redirect("/") @app.route("/dashboard") def admin_dashboard(): require_role("admin") tokens = db.get_tokens() blacklist = db.get_blacklist() return render_template("dashboard.html", api_tokens=tokens, blacklist=blacklist) @app.route("/blacklist/add", methods=["POST"]) def admin_blacklist_add(): require_role("admin") url = request.form.get("url") db.add_blacklist_website(url) flash("Added item to blacklist", "success") return redirect("/dashboard") @app.route("/blacklist//delete") def admin_blacklist_remove(blacklist_id): require_role("admin") db.remove_blacklist_website(blacklist_id) flash("Removed blacklist item", "success") return redirect("/dashboard") @app.route("/generate_token", methods=["POST"]) def admin_generate_token(): require_role("admin") description = request.form.get("description") db.generate_api_token(description) flash("Generated API token", "success") return redirect("/dashboard") @app.route("/del_token", methods=["POST"]) def admin_del_token(): require_role("admin") token = request.form.get("token") db.delete_token(token) flash("Deleted API token", "success") return redirect("/dashboard")