Docker tweaking & bug fixes

app.py

@@ -1,9 +1,11 @@
 from flask import Flask
 
 import api
+import common
 import config
 import template_filters
 import views
+import os
 
 app = Flask(__name__)
 app.secret_key = config.FLASK_SECRET
@@ -12,5 +14,12 @@ template_filters.setup_template_filters(app)
 views.setup_views(app)
 api.setup_api(app)
 
+
+if os.environ.get("ODDB_USER", False) and os.environ.get("ODDB_PASSWORD", False):
+    user = os.environ["ODDB_USER"]
+    password = os.environ["ODDB_PASSWORD"]
+    common.db.generate_login(user, password)
+    print("Generated user %s" % user)
+
 if __name__ == '__main__':
     app.run("0.0.0.0", port=80, threaded=True)

database.py

@@ -121,7 +121,9 @@ class Database:
             cursor = conn.cursor()
             cursor.execute("SELECT id FROM Website ORDER BY random() LIMIT 1")
 
-            return cursor.fetchone()[0]
+            if cursor.fetchone():
+                return cursor.fetchone()[0]
+            return None
 
     def website_exists(self, url):
         """Check if an url or the parent directory of an url already exists"""

docker-compose.yml

@@ -22,6 +22,8 @@ services:
       - "DB_CONN_STR=postgres://od_database:changeme@oddb_db/od_database?sslmode=disable"
       - "RECRAWL_POOL_SIZE=10000"
       - "INDEXER_THREADS=2"
+      - "ODDB_USER=admin"
+      - "ODDB_PASSWORD=changeme"
     depends_on:
       wsb:
         condition: service_started
@@ -73,10 +75,10 @@ services:
   tt_db:
     image: postgres
     volumes:
-      - ./tt_db_data:/var/lib/postgresql/data
+      - ./tt_pg_data:/var/lib/postgresql/data
     environment:
       - "POSTGRES_USER=task_tracker"
-      - "POSTGRES_PASSWORD=task_tracker"
+      - "POSTGRES_PASSWORD=changeme"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U task_tracker"]
       interval: 3s

tasks.py

@@ -2,6 +2,7 @@ import json
 import logging
 import os
 import time
+import traceback
 from multiprocessing.pool import ThreadPool
 from tempfile import NamedTemporaryFile
 from threading import Thread
@@ -10,6 +11,7 @@ from uuid import uuid4
 import requests
 import urllib3
 
+import common
 import config
 import database
 from database import Website
@@ -66,15 +68,19 @@ class TaskManager:
         self.db = database.Database(config.DB_CONN_STR)
         self.tracker = TaskTrackerApi(config.TT_API)
 
+        self.bucket = WsBucketApi(config.WSB_API, config.WSB_SECRET)
+        self._indexer_threads = list()
+
         self.worker = Worker.from_file(self.tracker)
         if not self.worker:
             self.worker = self.tracker.make_worker("$oddb_master")
+            if not self.worker:
+                common.logger.error("Could not create worker: %s" % traceback.format_exc())
+                return
             self.worker.dump_to_file()
             self.worker.request_access(config.TT_CRAWL_PROJECT, False, True)
             self.worker.request_access(config.TT_INDEX_PROJECT, True, False)
 
-        self.bucket = WsBucketApi(config.WSB_API, config.WSB_SECRET)
-        self._indexer_threads = list()
 
     def start_indexer_threads(self):
         logger.info("Starting %s indexer threads " % (config.INDEXER_THREADS, ))

views.py

@@ -117,7 +117,10 @@ def setup_views(app):
 
     @app.route("/website/random")
     def random_website():
-        return redirect("/website/" + str(db.get_random_website_id()))
+        rand_id = db.get_random_website_id()
+        if rand_id:
+            return redirect("/website/" + str())
+        return redirect("/website/")
 
     @app.route("/website/<int:website_id>/clear")
     def admin_clear_website(website_id):