simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-16 16:19:05 +00:00
Code Issues Projects Releases Wiki Activity

Elasticsearch search engine (import from json)

Browse Source
This commit is contained in:
Simon
2018-06-11 22:35:49 -04:00
parent fcfd7d4acc
commit 72495275b0
9 changed files with 190 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crawler/crawler.py
View File

@@ -12,7 +12,7 @@ class TooManyConnectionsError(Exception):
class File:
def __init__(self, name: str, size: int, mtime: str, path: str, is_dir: bool):
def __init__(self, name: str, size: int, mtime: int, path: str, is_dir: bool):
self.name = name
self.size = size
self.mtime = mtime
@@ -69,8 +69,7 @@ def export_to_json(q: Queue, out_file: str) -> int:
while True:
try:
next_file: File = q.get_nowait()
f.write(next_file.to_json())
f.write("\n")
f.write(next_file.to_json() + "\n")
counter += 1
except Empty:
break