simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-11 14:08:51 +00:00
Code Issues Projects Releases Wiki Activity

Indexing after crawling is a bit more efficient

Browse Source
This commit is contained in:
Simon
2018-06-14 16:41:43 -04:00
parent 83ca579ec7
commit dffd032659
5 changed files with 21 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
crawl_server/server.py
View File

@@ -1,4 +1,4 @@
from flask import Flask, request, abort, Response
from flask import Flask, request, abort, Response, send_file
from flask_httpauth import HTTPTokenAuth
import json
from crawl_server.task_manager import TaskManager, Task
@@ -14,7 +14,6 @@ tm = TaskManager("tm_db.sqlite3", 8)
@auth.verify_token
def verify_token(token):
print(token)
if token in tokens:
return True
@@ -68,15 +67,10 @@ def get_file_list(website_id):
file_name = "./crawled/" + str(website_id) + ".json"
if os.path.exists(file_name):
with open(file_name, "r") as f:
file_list = f.read()
os.remove(file_name)
return file_list
return send_file(file_name)
else:
return abort(404)
if __name__ == "__main__":
app.run(port=5002)
app.run(port=5001)