simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Files are indexed into ES when task is complete

Browse Source
This commit is contained in:
Simon
2018-06-12 15:45:00 -04:00
parent 6c912ea8c5
commit 1718bb91ca
7 changed files with 41 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crawl_server/task_manager.py
View File

@@ -55,14 +55,13 @@ class TaskManager:
print("Starting task " + task.url)
crawler = RemoteDirectoryCrawler(task.url, 100)
crawl_result = crawler.crawl_directory("crawled/" + str(task.website_id) + ".json")
crawl_result = crawler.crawl_directory("./crawled/" + str(task.website_id) + ".json")
result.file_count = crawl_result.file_count
result.status_code = crawl_result.status_code
print("End task " + task.url)
result.end_time = datetime.utcnow()
print("End task " + task.url)
return dict(result=result, db_path=db_path)
@@ -77,6 +76,7 @@ class TaskManager:
db = TaskManagerDatabase(db_path)
db.log_result(result)
print("Logged result to DB")
@staticmethod
def task_error(err):