simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 02:46:45 +00:00
Code Issues Projects Releases Wiki Activity

Some adjustments to make it work on Stretch server

Browse Source
This commit is contained in:
Simon 2018-06-14 17:09:05 -04:00
parent dffd032659
commit f3c7b551d2
3 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/crawler.py
View File

@ -68,7 +68,7 @@ def export_to_json(q: Queue, out_file: str) -> int:
with open(out_file, "w") as f: with open(out_file, "w") as f:
while True: while True:
try: try:
next_file: File = q.get_nowait() next_file = q.get_nowait()
f.write(next_file.to_json() + "\n") f.write(next_file.to_json() + "\n")
counter += 1 counter += 1
except Empty: except Empty:

2
crawl_server/server.py
View File

@ -73,4 +73,4 @@ def get_file_list(website_id):
if __name__ == "__main__": if __name__ == "__main__":
app.run(port=5001) app.run(port=5001, host="0.0.0.0")

1
requirements.txt
View File

@ -14,3 +14,4 @@ lxml
elasticsearch elasticsearch
python-dateutil python-dateutil
flask_httpauth flask_httpauth
ujson