simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-11 14:08:51 +00:00
Code Issues Projects Releases Wiki Activity

Elasticsearch search engine (import from json)

Browse Source
This commit is contained in:
Simon
2018-06-11 22:35:49 -04:00
parent fcfd7d4acc
commit 72495275b0
9 changed files with 190 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crawl_server/task_manager.py
View File

@@ -49,7 +49,7 @@ class TaskManager:
print("Starting task " + task.url)
crawler = RemoteDirectoryCrawler(task.url, 10)
crawler = RemoteDirectoryCrawler(task.url, 100)
crawl_result = crawler.crawl_directory("12345.json")
result.file_count = crawl_result.file_count
@@ -63,7 +63,6 @@ class TaskManager:
@staticmethod
def task_complete(result: TaskResult):
print("Task done " + str(result))
print(result.status_code)
print(result.file_count)
print(result.start_time)