diff --git a/app.py b/app.py index 1f00e69..819d27a 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,3 @@ -import time -time.sleep(60) - from flask import Flask import api @@ -12,7 +9,6 @@ app = Flask(__name__) app.secret_key = config.FLASK_SECRET template_filters.setup_template_filters(app) - views.setup_views(app) api.setup_api(app) diff --git a/docker-compose.yml b/docker-compose.yml index c47f598..615cb5b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3" +version: "2.1" services: oddb: image: simon987/od-database @@ -23,9 +23,14 @@ services: - "RECRAWL_POOL_SIZE=10000" - "INDEXER_THREADS=2" depends_on: - - wsb - - tt - - es + wsb: + condition: service_started + tt: + condition: service_started + oddb_db: + condition: service_healthy + es: + condition: service_healthy restart: always oddb_db: image: postgres @@ -71,22 +76,20 @@ services: ports: - 3010:80 depends_on: - - tt_db + tt_db: + condition: service_healthy es: image: docker.elastic.co/elasticsearch/elasticsearch:7.4.2 environment: -# - bootstrap.memory_lock=true - discovery.type=single-node -# - index.number_of_shards=50 -# - index.number_of_replicas=0 -# - "ES_JAVA_OPTS=-Xms1G -Xmx10G" + - "ES_JAVA_OPTS=-Xms1G -Xmx10G" volumes: - /usr/share/elasticsearch/data healthcheck: test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"] - interval: 30s - timeout: 30s - retries: 3 + interval: 5s + timeout: 5s + retries: 5 diff --git a/search/search.py b/search/search.py index 340134f..986e806 100644 --- a/search/search.py +++ b/search/search.py @@ -2,9 +2,8 @@ import os import time from urllib.parse import urljoin -import ujson - import elasticsearch +import ujson from apscheduler.schedulers.background import BackgroundScheduler from elasticsearch import helpers @@ -49,28 +48,29 @@ class ElasticSearchEngine: logger.info("Elasticsearch first time setup") if self.es.indices.exists(self.index_name): self.es.indices.delete(index=self.index_name) - self.es.indices.create(index=self.index_name) - self.es.indices.close(index=self.index_name) - - # Index settings - self.es.indices.put_settings(body={ - "index": { - "refresh_interval": "30s", - "codec": "best_compression" - }, - "analysis": { - "analyzer": { - "my_nGram": { - "tokenizer": "my_nGram_tokenizer", - "filter": ["lowercase", "asciifolding"] - } + self.es.indices.create(index=self.index_name, body={ + "settings": { + "index": { + "number_of_shards": 50, + "number_of_replicas": 0, + "refresh_interval": "30s", + "codec": "best_compression" }, - "tokenizer": { - "my_nGram_tokenizer": { - "type": "nGram", "min_gram": 3, "max_gram": 3 + "analysis": { + "analyzer": { + "my_nGram": { + "tokenizer": "my_nGram_tokenizer", + "filter": ["lowercase", "asciifolding"] + } + }, + "tokenizer": { + "my_nGram_tokenizer": { + "type": "nGram", "min_gram": 3, "max_gram": 3 + } } } - }}, index=self.index_name) + } + }) # Index Mappings self.es.indices.put_mapping(body={ diff --git a/templates/home.html b/templates/home.html index 2638212..1acf1f4 100644 --- a/templates/home.html +++ b/templates/home.html @@ -26,12 +26,8 @@
-
- -
-
- -
+ +
{% if show_captcha %} {{ captcha.get_code()|safe }} diff --git a/views.py b/views.py index eccbfea..b6f79ac 100644 --- a/views.py +++ b/views.py @@ -3,19 +3,24 @@ import os from multiprocessing.pool import Pool from urllib.parse import urlparse +from flask import render_template, redirect, request, flash, abort, Response, session +from flask_caching import Cache + import captcha import config import od_util from common import db, taskManager, searchEngine, logger, require_role from database import Website -from flask import render_template, redirect, request, flash, abort, Response, session -from flask_caching import Cache from search.search import InvalidQueryException from tasks import Task def setup_views(app): - cache = Cache(app, config={'CACHE_TYPE': 'simple'}) + cache = Cache(app, config={ + "CACHE_TYPE": "redis", + "CACHE_REDIS_HOST": config.REDIS_HOST, + "CACHE_REDIS_PORT": config.REDIS_PORT, + }) @app.route("/dl") @cache.cached(120) @@ -207,7 +212,8 @@ def setup_views(app): flash("Query failed, this could mean that the search server is overloaded or is not reachable. " "Please try again later", "danger") - results = hits["hits"]["total"]["value"] if not isinstance(hits["hits"]["total"], int) else hits["hits"]["total"] if hits else -1 + results = hits["hits"]["total"]["value"] if not isinstance(hits["hits"]["total"], int) else \ + hits["hits"]["total"] if hits else -1 took = hits["took"] if hits else -1 forwarded_for = request.headers["X-Forwarded-For"] if "X-Forwarded-For" in request.headers else None