From 408a2523685c74c7c9f93573b59f199d60a65e2a Mon Sep 17 00:00:00 2001 From: simon987 Date: Thu, 19 Mar 2020 11:00:09 -0400 Subject: [PATCH] Dockerize --- .gitmodules | 3 +++ Dockerfile | 10 ++++++++ docker-compose.yml | 54 +++++++++++++++++++++++++++++++++++++++++++ docker_viz/Dockerfile | 6 +++++ docker_viz/feed_viz | 1 + docker_viz/nginx.conf | 48 ++++++++++++++++++++++++++++++++++++++ run.py | 36 +++++++++++++++-------------- 7 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 .gitmodules create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker_viz/Dockerfile create mode 160000 docker_viz/feed_viz create mode 100644 docker_viz/nginx.conf diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2ff4424 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docker_viz/feed_viz"] + path = docker_viz/feed_viz + url = https://github.com/simon987/feed_viz diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d97f1f1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.8 + +WORKDIR /app + +ADD requirements.txt /app/requirements.txt +RUN pip install -r requirements.txt + +ENTRYPOINT ["python", "run.py"] + +COPY . /app diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3f97489 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +version: "2.1" +services: + influxdb: + image: influxdb:alpine + volumes: + - ./influxdb_data:/var/lib/influxdb + grafana: + image: grafana/grafana + ports: + - 127.0.0.1:3006:3000 + environment: + - "GF_SECURITY_ADMIN_PASSWORD=changeme" + db: + image: postgres + volumes: + - ./pg_data:/var/lib/postgresql/data + environment: + - "POSTGRES_USER=feed_archiver" + - "POSTGRES_PASSWORD=changeme" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U feed_archiver"] + interval: 5s + timeout: 5s + retries: 5 + rabbitmq: + image: rabbitmq:alpine + archiver: + image: simon987/feed_archiver + restart: always + depends_on: + db: + condition: service_healthy + environment: + - "FA_DB_HOST=db" + - "FA_DB_USER=feed_archiver" + - "FA_DB_PASSWORD=changeme" + - "FA_MQ_CONNSTR=amqp://guest:guest@rabbitmq:5672/" + - "FA_EXCHANGES=chan" + ws_adapter: + image: simon987/ws_feed_adapter + environment: + - "WSA_MQ_CONNSTR=amqp://guest:guest@rabbitmq:5672/" + feed_viz_frontend: + build: ./docker_viz/ + ports: + - 127.0.0.1:3005:80 + chan_4chan: + image: simon987/chan_feed + restart: always + environment: + - "CF_CHAN=4chan" + - "CF_MQ_HOST=rabbitmq" + - "CF_INFLUXDB=influxdb" + diff --git a/docker_viz/Dockerfile b/docker_viz/Dockerfile new file mode 100644 index 0000000..0b27d59 --- /dev/null +++ b/docker_viz/Dockerfile @@ -0,0 +1,6 @@ +FROM nginx:alpine + +COPY nginx.conf /etc/nginx/ +COPY ["/feed_viz", "/webroot"] + +EXPOSE 80 diff --git a/docker_viz/feed_viz b/docker_viz/feed_viz new file mode 160000 index 0000000..c8e11a7 --- /dev/null +++ b/docker_viz/feed_viz @@ -0,0 +1 @@ +Subproject commit c8e11a73d74e6af19cab581c94abf943daea050e diff --git a/docker_viz/nginx.conf b/docker_viz/nginx.conf new file mode 100644 index 0000000..b3313d6 --- /dev/null +++ b/docker_viz/nginx.conf @@ -0,0 +1,48 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + + keepalive_timeout 65; + + upstream socket { + server ws_adapter:3090; + } + + server { + listen 80; + + index index.html; + root /webroot; + + location / { + try_files $uri $uri/ /index.html; + } + + location /socket { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_read_timeout 86400; + proxy_pass http://socket; + } + } +} \ No newline at end of file diff --git a/run.py b/run.py index f9635f7..399eabe 100644 --- a/run.py +++ b/run.py @@ -1,7 +1,7 @@ import datetime import json +import os import sqlite3 -import sys import time import traceback from collections import defaultdict @@ -17,12 +17,12 @@ from chan.chan import CHANS from post_process import post_process from util import logger, Web -MONITORING = True BYPASS_RPS = False DBNAME = "chan_feed" -if MONITORING: - influxdb = Monitoring(DBNAME, logger=logger, batch_size=100, flush_on_exit=True) +if os.environ.get("CF_INFLUXDB"): + influxdb = Monitoring(DBNAME, host=os.environ.get("CF_INFLUXDB"), logger=logger, batch_size=100, flush_on_exit=True) + MONITORING = True class ChanScanner: @@ -163,7 +163,7 @@ def publish_worker(queue: Queue, helper, p): queue.task_done() -@buffered(batch_size=300, flush_on_exit=True) +@buffered(batch_size=150, flush_on_exit=True) def _publish_buffered(items): if not items: return @@ -216,25 +216,27 @@ def publish(item, board, helper, channel, web): def connect(): - rabbit = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq_host)) - channel = rabbit.channel() - channel.exchange_declare(exchange="chan", exchange_type="topic") - return channel + while True: + try: + rabbit = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq_host)) + channel = rabbit.channel() + channel.exchange_declare(exchange="chan", exchange_type="topic") + return channel + except Exception as e: + logger.error(str(e)) + time.sleep(0.5) + pass if __name__ == "__main__": - if len(sys.argv) < 3: - logger.error("You must specify RabbitMQ host & chan!") - quit(1) - - rabbitmq_host = sys.argv[1] - chan = sys.argv[2] + rabbitmq_host = os.environ.get("CF_MQ_HOST", "localhost") + chan = os.environ.get("CF_CHAN", None) chan_helper = CHANS[chan] proxy = None - if len(sys.argv) > 3: - proxy = sys.argv[3] + if os.environ.get("CF_PROXY"): + proxy = os.environ.get("CF_PROXY") logger.info("Using proxy %s" % proxy) if BYPASS_RPS: