simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 02:16:47 +00:00
Code Issues Projects Releases Wiki Activity

Misc bug fixes

Browse Source
This commit is contained in:
Simon 2018-07-20 10:35:17 -04:00
parent df5b01dc83
commit 004ade8935
2 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
crawl_server/task_manager.py
View File

@ -47,22 +47,26 @@ class TaskManager:
logger.info("Uploading file list in small chunks")
filename = "./crawled/" + str(task_result.website_id) + ".json"
CHUNK_SIZE = 1000000 * 10
CHUNK_SIZE = 500000 * 10 # 5Mb
if os.path.exists(filename):
with open(filename) as f:
chunk = f.read(CHUNK_SIZE)
while chunk:
payload = {
"token": config.API_TOKEN,
"website_id": task_result.website_id
}
try:
payload = {
"token": config.API_TOKEN,
"website_id": task_result.website_id
}
files = {
"file_list": chunk
}
files = {
"file_list": chunk
}
r = requests.post(config.SERVER_URL + "/task/upload", data=payload, files=files)
logger.info("RESPONSE: " + r.text)
r = requests.post(config.SERVER_URL + "/task/upload", data=payload, files=files)
logger.info("RESPONSE: " + r.text)
except Exception as e:
logger.error("Exception while sending file_list chunk: " + str(e))
pass
chunk = f.read(CHUNK_SIZE)
payload = {

2
search/search.py
View File

@ -418,7 +418,7 @@ class ElasticSearchEngine(SearchEngine):
"query": {
"match_all": {}
}
}, scroll="5m", client=self.es, index=self.index_name)
}, scroll="1m", client=self.es, index=self.index_name, request_timeout=60)
def are_empty(self, websites):
result = self.es.search(body={