From 290322dfa79ce0eecba2b6429181834c73ce53d9 Mon Sep 17 00:00:00 2001
From: Simon <fortier.simon@hotmail.com>
Date: Thu, 12 Jul 2018 11:26:20 -0400
Subject: [PATCH] Indexing is a bit gentler on server resources and some pages
 have been memory cached

---
 app.py           | 7 +++++--
 search/search.py | 7 ++++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/app.py b/app.py
index d0ddfac..afcf052 100644
--- a/app.py
+++ b/app.py
@@ -58,6 +58,7 @@ def stats_page():
 
 
 @app.route("/stats/json_chart")
+@cache.cached(240)
 def stats_json():
     stats = searchEngine.get_global_stats()
     db.join_website_on_stats(stats)
@@ -82,7 +83,7 @@ def website_info(website_id):
 
 
 @app.route("/website/<int:website_id>/json_chart")
-@cache.memoize(30)
+@cache.memoize(60)
 def website_json_chart(website_id):
     website = db.get_website_by_id(website_id)
 
@@ -310,6 +311,7 @@ def contribute():
 
 
 @app.route("/")
+@cache.cached(240)
 def home():
     stats = searchEngine.get_global_stats()
     stats["website_count"] = len(db.get_all_websites())
@@ -393,7 +395,8 @@ def enqueue_bulk():
                 flash("Too few or too many urls, please submit 1-10 urls", "danger")
                 return redirect("/submit")
         else:
-            return abort(500)
+            flash("Too few or too many urls, please submit 1-10 urls", "danger")
+            return redirect("/submit")
     else:
         flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
         return redirect("/submit")
diff --git a/search/search.py b/search/search.py
index 3d0a33d..75ecae7 100644
--- a/search/search.py
+++ b/search/search.py
@@ -114,7 +114,8 @@ class ElasticSearchEngine(SearchEngine):
 
     def import_json(self, in_lines, website_id: int):
 
-        import_every = 2500
+        import_every = 1000
+        cooldown_time = 0.250
 
         docs = []
 
@@ -129,6 +130,8 @@ class ElasticSearchEngine(SearchEngine):
             if len(docs) >= import_every:
                 self._index(docs)
                 docs.clear()
+                time.sleep(cooldown_time)
+
         if docs:
             self._index(docs)
 
@@ -265,8 +268,6 @@ class ElasticSearchEngine(SearchEngine):
 
     def get_global_stats(self):
 
-        # TODO: mem cache this
-
         size_per_ext = self.es.search(body={
             "query": {
                 "bool": {