simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 14:59:02 +00:00
Code Issues Projects Releases Wiki Activity

File upload is made in small chunks

Browse Source
This commit is contained in:
Simon
2018-07-17 17:52:17 -04:00
parent 73afebec28
commit 898ffcf410
2 changed files with 46 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
crawl_server/task_manager.py
View File

@@ -45,21 +45,31 @@ class TaskManager:
try:
logger.info("Uploading file list in small chunks")
filename = "./crawled/" + str(task_result.website_id) + ".json"
CHUNK_SIZE = 1000000 * 10
with open(filename) as f:
chunk = f.read(CHUNK_SIZE)
while chunk:
payload = {
"token": config.API_TOKEN,
"website_id": task_result.website_id
}
files = {
"file_list": chunk
}
r = requests.post(config.SERVER_URL + "/task/upload", data=payload, files=files)
logger.info("RESPONSE: " + r.text)
chunk = f.read(CHUNK_SIZE)
payload = {
"token": config.API_TOKEN,
"result": json.dumps(task_result.to_json())
}
filename = "./crawled/" + str(task_result.website_id) + ".json"
if os.path.exists(filename):
files = {
"file_list": open(filename)
}
else:
files = None
r = requests.post(config.SERVER_URL + "/task/complete", data=payload, files=files)
r = requests.post(config.SERVER_URL + "/task/complete", data=payload)
logger.info("RESPONSE: " + r.text)
if os.path.exists(filename):