Elasticsearch search engine (import from json)

test/test_crawl_server.py

@@ -1,9 +1,7 @@
 from flask_testing import LiveServerTestCase
-import os
 import json
 import requests
 from crawl_server.server import app
-from crawl_server.task_manager import TaskManager
 
 
 class CrawlServerTest(LiveServerTestCase):
@@ -20,15 +18,6 @@ class CrawlServerTest(LiveServerTestCase):
         app.config['LIVESERVER_PORT'] = 9999
         return app
 
-    def test_put_only_accepts_json(self):
-
-        payload = json.dumps({"url": "", "priority": 1, "callback_type": "", "callback_args": "{}"})
-        r = requests.post(self.HOST + "/task/put", data=payload)
-        self.assertEqual(400, r.status_code)
-
-        r2 = requests.post(self.HOST + "/task/put", headers=self.headers, data=payload)
-        self.assertEqual(200, r2.status_code)
-
     def test_put_task(self):
 
         payload = json.dumps({
@@ -43,11 +32,15 @@ class CrawlServerTest(LiveServerTestCase):
         r = requests.get(self.HOST + "/task")
         self.assertEqual(200, r.status_code)
 
-        print(r.text)
         result = json.loads(r.text)[0]
         self.assertEqual(result["url"], "a")
         self.assertEqual(result["priority"], 2)
         self.assertEqual(result["callback_type"], "c")
         self.assertEqual(result["callback_args"], '{"d": 4}')
 
+        payload = json.dumps({"url": "", "priority": 1, "callback_type": "", "callback_args": "{}"})
+        r = requests.post(self.HOST + "/task/put", data=payload)
+        self.assertEqual(400, r.status_code)
 
+        r2 = requests.post(self.HOST + "/task/put", headers=self.headers, data=payload)
+        self.assertEqual(200, r2.status_code)

test/test_search.py

@@ -0,0 +1,38 @@
+from unittest import TestCase
+import time
+import json
+import os
+from search.search import ElasticSearchEngine
+
+
+class SearchTest(TestCase):
+
+    def setUp(self):
+        self.search = ElasticSearchEngine("od-database-test")
+        self.search.reset()
+        time.sleep(1)
+
+    def test_ping(self):
+        self.assertTrue(self.search.ping(), "Search engine not running")
+
+    def test_import_json(self):
+
+        files = [
+            {"name": "a", "size": 1000000000000000000, "path": "c/d", "mtime": 1528765672},
+            {"name": "b", "size": 123, "path": "", "mtime": None},
+            {"name": "c", "size": -1, "path": "c", "mtime": 12345}
+        ]
+
+        with open("tmp.json", "w") as f:
+            for file in files:
+                f.write(json.dumps(file) + "\n")
+
+        self.search.import_json("tmp.json", 123)
+        time.sleep(3)
+        self.assertEqual(3, self.search.es.count(self.search.index_name, "file")["count"])
+
+        os.remove("tmp.json")
+
+
+
+