From 8ef1d36c9dced6481d7e36f9d3cbdbfaebfe05b9 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 17 Jul 2018 18:24:05 -0400 Subject: [PATCH] Bug when directory is empty with new file upload --- crawl_server/task_manager.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/crawl_server/task_manager.py b/crawl_server/task_manager.py index e2e6266..39cb2fc 100644 --- a/crawl_server/task_manager.py +++ b/crawl_server/task_manager.py @@ -48,21 +48,22 @@ class TaskManager: logger.info("Uploading file list in small chunks") filename = "./crawled/" + str(task_result.website_id) + ".json" CHUNK_SIZE = 1000000 * 10 - with open(filename) as f: - chunk = f.read(CHUNK_SIZE) - while chunk: - payload = { - "token": config.API_TOKEN, - "website_id": task_result.website_id - } - - files = { - "file_list": chunk - } - - r = requests.post(config.SERVER_URL + "/task/upload", data=payload, files=files) - logger.info("RESPONSE: " + r.text) + if os.path.exists(filename): + with open(filename) as f: chunk = f.read(CHUNK_SIZE) + while chunk: + payload = { + "token": config.API_TOKEN, + "website_id": task_result.website_id + } + + files = { + "file_list": chunk + } + + r = requests.post(config.SERVER_URL + "/task/upload", data=payload, files=files) + logger.info("RESPONSE: " + r.text) + chunk = f.read(CHUNK_SIZE) payload = { "token": config.API_TOKEN,