simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 18:36:44 +00:00
Code Issues Projects Releases Wiki Activity

Bug when directory is empty with new file upload

Browse Source
This commit is contained in:
Simon 2018-07-17 18:24:05 -04:00
parent 898ffcf410
commit 8ef1d36c9d
1 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
crawl_server/task_manager.py
View File

@ -48,6 +48,7 @@ class TaskManager:
logger.info("Uploading file list in small chunks") logger.info("Uploading file list in small chunks")
filename = "./crawled/" + str(task_result.website_id) + ".json" filename = "./crawled/" + str(task_result.website_id) + ".json"
CHUNK_SIZE = 1000000 * 10 CHUNK_SIZE = 1000000 * 10
if os.path.exists(filename):
with open(filename) as f: with open(filename) as f:
chunk = f.read(CHUNK_SIZE) chunk = f.read(CHUNK_SIZE)
while chunk: while chunk: