Dockerize

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "docker_viz/feed_viz"]
+	path = docker_viz/feed_viz
+	url = https://github.com/simon987/feed_viz

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.8
+
+WORKDIR /app
+
+ADD requirements.txt /app/requirements.txt
+RUN pip install -r requirements.txt
+
+ENTRYPOINT ["python", "run.py"]
+
+COPY . /app

docker-compose.yml

@@ -0,0 +1,54 @@
+version: "2.1"
+services:
+  influxdb:
+    image: influxdb:alpine
+    volumes:
+      - ./influxdb_data:/var/lib/influxdb
+  grafana:
+    image: grafana/grafana
+    ports:
+      - 127.0.0.1:3006:3000
+    environment:
+      - "GF_SECURITY_ADMIN_PASSWORD=changeme"
+  db:
+    image: postgres
+    volumes:
+      - ./pg_data:/var/lib/postgresql/data
+    environment:
+      - "POSTGRES_USER=feed_archiver"
+      - "POSTGRES_PASSWORD=changeme"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U feed_archiver"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+  rabbitmq:
+    image: rabbitmq:alpine
+  archiver:
+    image: simon987/feed_archiver
+    restart: always
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      - "FA_DB_HOST=db"
+      - "FA_DB_USER=feed_archiver"
+      - "FA_DB_PASSWORD=changeme"
+      - "FA_MQ_CONNSTR=amqp://guest:guest@rabbitmq:5672/"
+      - "FA_EXCHANGES=chan"
+  ws_adapter:
+    image: simon987/ws_feed_adapter
+    environment:
+      - "WSA_MQ_CONNSTR=amqp://guest:guest@rabbitmq:5672/"
+  feed_viz_frontend:
+    build: ./docker_viz/
+    ports:
+    - 127.0.0.1:3005:80
+  chan_4chan:
+    image: simon987/chan_feed
+    restart: always
+    environment:
+      - "CF_CHAN=4chan"
+      - "CF_MQ_HOST=rabbitmq"
+      - "CF_INFLUXDB=influxdb"
+

docker_viz/Dockerfile

@@ -0,0 +1,6 @@
+FROM nginx:alpine
+
+COPY nginx.conf /etc/nginx/
+COPY ["/feed_viz", "/webroot"]
+
+EXPOSE 80

docker_viz/feed_viz

@@ -0,0 +1 @@
+Subproject commit c8e11a73d74e6af19cab581c94abf943daea050e

docker_viz/nginx.conf

@@ -0,0 +1,48 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+
+    keepalive_timeout  65;
+
+    upstream socket {
+        server ws_adapter:3090;
+    }
+
+    server {
+        listen 80;
+
+        index index.html;
+        root /webroot;
+
+        location / {
+            try_files $uri $uri/ /index.html;
+        }
+
+        location /socket {
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "Upgrade";
+            proxy_set_header Host $host;
+            proxy_read_timeout 86400;
+            proxy_pass http://socket;
+        }
+    }
+}

run.py

@@ -1,7 +1,7 @@
 import datetime
 import json
+import os
 import sqlite3
-import sys
 import time
 import traceback
 from collections import defaultdict
@@ -17,12 +17,12 @@ from chan.chan import CHANS
 from post_process import post_process
 from util import logger, Web
 
-MONITORING = True
 BYPASS_RPS = False
 
 DBNAME = "chan_feed"
-if MONITORING:
+if os.environ.get("CF_INFLUXDB"):
-    influxdb = Monitoring(DBNAME, logger=logger, batch_size=100, flush_on_exit=True)
+    influxdb = Monitoring(DBNAME, host=os.environ.get("CF_INFLUXDB"), logger=logger, batch_size=100, flush_on_exit=True)
+    MONITORING = True
 
 
 class ChanScanner:
@@ -163,7 +163,7 @@ def publish_worker(queue: Queue, helper, p):
             queue.task_done()
 
 
-@buffered(batch_size=300, flush_on_exit=True)
+@buffered(batch_size=150, flush_on_exit=True)
 def _publish_buffered(items):
     if not items:
         return
@@ -216,25 +216,27 @@ def publish(item, board, helper, channel, web):
 
 
 def connect():
+    while True:
+        try:
             rabbit = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq_host))
             channel = rabbit.channel()
             channel.exchange_declare(exchange="chan", exchange_type="topic")
             return channel
+        except Exception as e:
+            logger.error(str(e))
+            time.sleep(0.5)
+            pass
 
 
 if __name__ == "__main__":
 
-    if len(sys.argv) < 3:
+    rabbitmq_host = os.environ.get("CF_MQ_HOST", "localhost")
-        logger.error("You must specify RabbitMQ host & chan!")
+    chan = os.environ.get("CF_CHAN", None)
-        quit(1)
-
-    rabbitmq_host = sys.argv[1]
-    chan = sys.argv[2]
     chan_helper = CHANS[chan]
 
     proxy = None
-    if len(sys.argv) > 3:
+    if os.environ.get("CF_PROXY"):
-        proxy = sys.argv[3]
+        proxy = os.environ.get("CF_PROXY")
         logger.info("Using proxy %s" % proxy)
 
     if BYPASS_RPS: