Update thumbnail-quality parameter in sist2-admin

SQLite search backend

.devcontainer/Dockerfile

@@ -0,0 +1,9 @@
+FROM simon987/sist2-build
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+

.devcontainer/devcontainer.json

@@ -0,0 +1,16 @@
+{
+  "name": "sist2-dev",
+  "dockerComposeFile": [
+    "docker-compose.yml"
+  ],
+  "service": "sist2-dev",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-vscode.cpptools-extension-pack"
+      ]
+    }
+  },
+  "remoteUser": "root",
+  "workspaceFolder": "/app/"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3"
+
+services:
+  sist2-dev:
+    build: .
+    command: sleep infinity
+    volumes:
+      - ../:/app

Dockerfile

@@ -45,6 +45,7 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2

Dockerfile.arm64

@@ -20,16 +20,17 @@ RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-p
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2

README.md

@@ -28,7 +28,7 @@ sist2 (Simple incremental search tool)
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
-\*\*\* See [OCR](#ocr)
+\*\*\* See [OCR](#ocr)    
 \*\*\*\* See [Named-Entity Recognition](#NER)
 
 ## Getting Started
@@ -46,7 +46,7 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
   sist2-admin:
-    image: simon987/sist2:3.0.7-x64-linux
+    image: simon987/sist2:3.1.0-x64-linux
     restart: unless-stopped
     volumes:
       - ./sist2-admin-data/:/sist2-admin/
@@ -126,7 +126,7 @@ Download the language data files with your package manager (`apt install tessera
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
 name must be identical to the `*.traineddata` file installed on your system
@@ -206,7 +206,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
+    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp]
     ```
 
 4. Build

docs/USAGE.md

@@ -17,7 +17,7 @@ Lightning-fast file system indexer and search tool.
 
 Scan options
     -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
     --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
     --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
     --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
@@ -88,8 +88,8 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 
 See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
 
-For example, `--thumbnail-size=500`, `--thumbnail-quality=2` for a directory with 8 million images will create a thumbnail database 
-that is about `8000000 * 36kB = 288GB`.
+For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database 
+that is about `8000000 * 11.8kB = 94.4GB`.
 
 ![thumbnail_size](thumbnail_size.png)
 

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -7,7 +7,7 @@
     <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
-    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+    <b-form-input type="number" min="0" max="100" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailCount") }}</label>
     <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>

sist2-admin/frontend/src/i18n/messages.js

@@ -66,7 +66,7 @@ export default {
             path: "Path",
             threads: "Number of threads",
             memThrottle: "Total memory threshold in MiB for scan throttling",
-            thumbnailQuality: "Thumbnail quality, on a scale of 2 to 32, 2 being the best",
+            thumbnailQuality: "Thumbnail quality, on a scale of 0 to 100, 100 being the best",
             thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
             thumbnailSize: "Thumbnail size, in pixels",
             contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",

sist2-admin/sist2_admin/sist2.py

@@ -46,7 +46,7 @@ class WebOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
-        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
+        args = ["web", f"--es-url={self.es_url}", f"--es-index={self.es_index}", f"--bind={self.bind}",
                 f"--tagline={self.tagline}", f"--lang={self.lang}"]
 
         if self.auth0_audience:
@@ -109,7 +109,7 @@ ARCHIVE_RECURSE = "recurse"
 class ScanOptions(BaseModel):
     path: str
     threads: int = 1
-    thumbnail_quality: int = 2
+    thumbnail_quality: int = 50
     thumbnail_size: int = 552
     thumbnail_count: int = 1
     content_size: int = 32768

sist2-vue/public/index.html

@@ -19,12 +19,6 @@
             We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
             Please enable it to continue.
         </strong>
-        <br/>
-        <strong>
-            Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
-            si JavaScript est activé.
-            Veuillez l'activer pour continuer.
-        </strong>
     </div>
 </noscript>
 <div id="app"></div>

sist2-vue/src/App.vue

@@ -10,7 +10,7 @@
             <b-spinner type="grow" variant="primary"></b-spinner>
         </div>
         <div class="loading-text">
-            Loading • Chargement • 装载 • Wird geladen
+            Loading • Chargement • 装载 • Wird geladen • Ładowanie
         </div>
     </div>
 </template>

sist2-vue/src/Sist2ElasticsearchQuery.ts

@@ -170,14 +170,16 @@ class Sist2ElasticsearchQuery {
                 }
             },
             sort: SORT_MODES[getters.sortMode].mode,
-            aggs:
-                {
-                    total_size: {"sum": {"field": "size"}},
-                    total_count: {"value_count": {"field": "size"}}
-                },
             size: size,
         } as any;
 
+        if (!after) {
+            q.aggs = {
+                total_size: {"sum": {"field": "size"}},
+                total_count: {"value_count": {"field": "size"}}
+            };
+        }
+
         if (!empty && !blankSearch) {
             q.query.bool.must = query;
         }

sist2-vue/src/i18n/messages.ts

@@ -1,7 +1,7 @@
 export default {
     en: {
         filePage: {
-          notFound: "Not found"
+            notFound: "Not found"
         },
         searchBar: {
             simple: "Search",
@@ -92,6 +92,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -184,7 +185,7 @@ export default {
     },
     de: {
         filePage: {
-          notFound: "Nicht gefunden"
+            notFound: "Nicht gefunden"
         },
         searchBar: {
             simple: "Suche",
@@ -271,6 +272,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -445,6 +447,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -619,6 +622,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -703,4 +707,188 @@ export default {
             selectedIndices: "选中索引",
         },
     },
+    pl: {
+        filePage: {
+            notFound: "Nie znaleziono"
+        },
+        searchBar: {
+            simple: "Szukaj",
+            advanced: "Zaawansowane szukanie",
+            fuzzy: "Również podobne"
+        },
+        addTag: "Tag",
+        deleteTag: "Usuń",
+        download: "Pobierz",
+        and: "i",
+        page: "strona",
+        pages: "stron",
+        mimeTypes: "Typy danych",
+        tags: "Tagi",
+        tagFilter: "Filtruj tagi",
+        forExample: "Na przykład:",
+        help: {
+            simpleSearch: "Proste szukanie",
+            advancedSearch: "Zaawansowane szukanie",
+            help: "Pomoc",
+            term: "<WYRAZ>",
+            and: "operator I",
+            or: "operator LUB",
+            not: "zabrania danego wyrazu",
+            quotes: "znajdzie objętą sekwencję wyrazów w podanej kolejności",
+            prefix: "znajdzie dowolny wyraz rozpoczynający się na takie litery, jeśli zastosowane na końcu wyrazu",
+            parens: "używane do grupowania wyrażeń",
+            tildeTerm: "znajdzie wyraz w podanej odległości",
+            tildePhrase: "znajdzie frazę przeplecioną podaną liczbą niepasujących wyrazów",
+            example1:
+                "Na przykład: <code>\"pieczone jajko\" +(kiełbasa | ziemniak) -frytki</code> znajdzie frazę " +
+                "<i>pieczone jajko</i> gdzie występuje też: <i>kiełbasa</i> albo <i>ziemniak</i>, ale zignoruje rezultat " +
+                "zawierający <i>frytki</i>.",
+            defaultOperator:
+                "Kiedy nie podano ani <code>+</code>, ani <code>|</code>, to domyślnym operatorem jest " +
+                "<code>+</code> (i).",
+            fuzzy:
+                "Kiedy opcja <b>Również podobne</b> jest zaznaczona, częściowo zgodne wyrazy są również znajdywane.",
+            moreInfoSimple: "Po więcej informacji sięgnij do <a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">dokumentacji Elasticsearch</a>",
+            moreInfoAdvanced: "Aby uzyskać więcej informacji o zaawansowanym szukaniu, przeczytaj <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">dokumentację Elasticsearch</a>"
+        },
+        config: "Ustawienia",
+        configDescription: "Ustawienia są zapisywane na żywo w tej przeglądarce.",
+        configReset: "Zresetuj ustawienia",
+        searchOptions: "Opcje szukania",
+        treemapOptions: "Opcje mapy",
+        mlOptions: "Opcje uczenia maszynowego",
+        displayOptions: "Opcje wyświetlania",
+        opt: {
+            lang: "Język",
+            highlight: "Zaznaczaj znalezione fragmenty",
+            fuzzy: "Ustaw szukanie również podobnych jako domyślne",
+            searchInPath: "Włącz szukanie również w ścieżce dokumentu",
+            suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
+            fragmentSize: "Podświetl wielkość kontekstu w znakach",
+            queryMode: "Tryb szukania",
+            displayMode: "Wyświetlanie",
+            columns: "Liczba kolumn",
+            treemapType: "Typ mapy",
+            treemapTiling: "Układanie mapy",
+            treemapColorGroupingDepth: "Jak głęboko grupować kolory mapy (na płasko)",
+            treemapColor: "Kolor mapy (kaskadowo)",
+            treemapSize: "Wielkość mapy",
+            theme: "Styl graficzny",
+            lightboxLoadOnlyCurrent: "Nie pobieraj od razu obrazów w pełnej wielkości dla sąsiednich obrazów podglądu.",
+            slideDuration: "Czas trwania jednego slajdu w pokazie slajdów",
+            resultSize: "Liczba wyników na stronę",
+            tagOrOperator: "Użyj operatora LUB przy wyborze kilku tagów",
+            hideDuplicates: "Ukryj zduplikowane wyniki (według sumy kontrolnej)",
+            hideLegacy: "Ukryj powiadomienie Elasticsearch 'legacyES'",
+            updateMimeMap: "Uaktualniaj drzewo typów mediów na żywo",
+            useDatePicker: "Używaj kalendarza do wyboru dat, zamiast suwaka",
+            vidPreviewInterval: "Czas trwania jednej klatki w podglądzie wideo (w ms)",
+            simpleLightbox: "Wyłącz animacje w podglądzie obrazów",
+            showTagPickerFilter: "Pokazuj pole filtrowania tagów",
+            featuredFields: "Wybrane pola szablonu Javascript. Będą pojawiać się przy wynikach wyszukiwania.",
+            featuredFieldsList: "Dostępne zmienne",
+            autoAnalyze: "Automatycznie analizuj tekst",
+            defaultModel: "Domyślny model",
+            mlRepositories: "Repozytoria modeli (każde w osobnej linii)"
+        },
+        queryMode: {
+            simple: "Proste",
+            advanced: "Zaawansowane",
+        },
+        lang: {
+            en: "English",
+            de: "Deutsch",
+            fr: "Français",
+            pl: "Polski",
+            "zh-CN": "简体中文",
+        },
+        displayMode: {
+            grid: "Siatka",
+            list: "Lista",
+        },
+        columns: {
+            auto: "Automatyczna"
+        },
+        treemapType: {
+            cascaded: "Kaskadowa",
+            flat: "Płaska (kompaktowa)"
+        },
+        treemapSize: {
+            small: "Mała",
+            medium: "Średnia",
+            large: "Duża",
+            xLarge: "Bardzo duża",
+            xxLarge: "Ogromna",
+            custom: "Inna",
+        },
+        treemapTiling: {
+            binary: "Binarnie",
+            squarify: "Kwadratowo",
+            slice: "Wycinek",
+            dice: "Kostka",
+            sliceDice: "Wycinek i kostka",
+        },
+        theme: {
+            light: "Jasny",
+            black: "Czarny"
+        },
+        hit: "traf",
+        hits: "trafień",
+        details: "Szczegóły",
+        stats: "Statystyki",
+        queryTime: "Czas szukania",
+        totalSize: "Całkowita wielkość",
+        pathBar: {
+            placeholder: "Filtruj ścieżki",
+            modalTitle: "Wybierz ścieżkę"
+        },
+        debug: "Informacje dla programistów",
+        debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+        tagline: "Slogan",
+        toast: {
+            esConnErrTitle: "Problem z połączeniem z Elasticsearch",
+            esConnErr: "Moduł strony internetowej sist2 napotkał problem przy połączeniu z Elasticsearch." +
+                " Zobacz logi serwera, aby uzyskać więcej informacji.",
+            esQueryErrTitle: "Problem z kwerendą",
+            esQueryErr: "Kwerenda szukania jest niezrozumiała albo nie udało się jej przesłać. Sprawdź dokumentację zaawansowanego szukania. " +
+                "Zobacz logi serwera, aby uzyskać więcej informacji.",
+            dupeTagTitle: "Zduplikowany tag",
+            dupeTag: "Ten dokument już ma taki tag.",
+            copiedToClipboard: "Skopiowano do schowka"
+        },
+        saveTagModalTitle: "Dodaj tag",
+        saveTagPlaceholder: "Nazwa",
+        confirm: "Zatwierdź",
+        indexPickerPlaceholder: "Wybierz indeks",
+        sort: {
+            relevance: "Zgodność z szukanym",
+            dateAsc: "Data (najpierw starsze)",
+            dateDesc: "Data (najpierw nowsze)",
+            sizeAsc: "Wielkość (najpierw mniejsze)",
+            sizeDesc: "Wielkość (najpierw większe)",
+            nameAsc: "Nazwa (A-z)",
+            nameDesc: "Nazwa (Z-a)",
+            random: "Losowo",
+        },
+        d3: {
+            mimeCount: "Dystrybucja liczby plików według typów mediów",
+            mimeSize: "Dystrybucja wielkości plików według typów mediów",
+            dateHistogram: "Dystrybucja dat modyfikacji plików",
+            sizeHistogram: "Dystrybucja wielkości plików",
+        },
+        indexPicker: {
+            selectNone: "Zaznacz nic",
+            selectAll: "Zaznacz wszystko",
+            selectedIndex: "wybrany indeks",
+            selectedIndices: "wybrane indeksy",
+        },
+        ml: {
+            analyzeText: "Analizuj",
+            auto: "Automatycznie",
+            repoFetchError: "Nie udało się uzyskać listy modeli. Zobacz konsolę przeglądarki, aby uzyskać więcej informacji.",
+            repoFetchErrorTitle: "Nie udało się pobrać repozytoriów modeli",
+        }
+    }
 }

sist2-vue/src/views/Configuration.vue

@@ -237,6 +237,7 @@ export default {
                 {value: "fr", text: this.$t("lang.fr")},
                 {value: "zh-CN", text: this.$t("lang.zh-CN")},
                 {value: "de", text: this.$t("lang.de")},
+                {value: "pl", text: this.$t("lang.pl")},
             ],
             queryModeOptions: [
                 {value: "simple", text: this.$t("queryMode.simple")},

src/cli.c

@@ -5,7 +5,7 @@
 #define DEFAULT_OUTPUT "index.sist2"
 #define DEFAULT_NAME "index"
 #define DEFAULT_CONTENT_SIZE 32768
-#define DEFAULT_QUALITY 2
+#define DEFAULT_QUALITY 50
 #define DEFAULT_THUMBNAIL_SIZE 552
 #define DEFAULT_THUMBNAIL_COUNT 1
 #define DEFAULT_REWRITE_URL ""
@@ -100,8 +100,8 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) {
         args->tn_quality = DEFAULT_QUALITY;
-    } else if (args->tn_quality < 2 || args->tn_quality > 31) {
-        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [2, 31].\n",
+    } else if (args->tn_quality < 0 || args->tn_quality > 100) {
+        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [0, 100].\n",
                 args->tn_quality);
         return 1;
     }
@@ -109,7 +109,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (args->tn_size == OPTION_VALUE_UNSPECIFIED) {
         args->tn_size = DEFAULT_THUMBNAIL_SIZE;
     } else if (args->tn_size < 32) {
-        printf("Invalid value --thumbnail-size argument: %d. Must be greater than 32 pixels.\n", args->tn_size);
+        printf("Invalid value --thumbnail-size argument: %d. Must be >= 32 pixels.\n", args->tn_size);
         return 1;
     }
 

src/index/elastic.c

@@ -240,7 +240,10 @@ void print_errors(response_t *r) {
     } else if (errors->valueint != 0) {
         cJSON *err;
         cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
-            if (cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint != 201) {
+
+            int status = cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint;
+
+            if (status != 201 && status != 200) {
                 char *str = cJSON_Print(err);
                 LOG_ERRORF("elastic.c", "%s\n", str);
                 cJSON_free(str);

src/main.c

@@ -490,7 +490,7 @@ int main(int argc, const char *argv[]) {
             OPT_GROUP("Scan options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT: 1"),
             OPT_INTEGER('q', "thumbnail-quality", &scan_args->tn_quality,
-                        "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2",
+                        "Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50",
                         set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_quality),
             OPT_INTEGER(0, "thumbnail-size", &scan_args->tn_size,
                         "Thumbnail size, in pixels. DEFAULT: 552",

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.0.7"
+#define VERSION "3.1.0"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
-static const int VersionMinor = 0;
-static const int VersionPatch = 7;
+static const int VersionMinor = 1;
+static const int VersionPatch = 0;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

third-party/libscan/libscan/arc/arc.c

@@ -4,7 +4,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <fcntl.h>
-#include <openssl/evp.h>
 #include <pcre.h>
 
 #define MAX_DECOMPRESSED_SIZE_RATIO 40.0
@@ -211,11 +210,20 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
 
                 double decompressed_size_ratio = (double) sub_job->vfile.st_size / (double) f->st_size;
                 if (decompressed_size_ratio > MAX_DECOMPRESSED_SIZE_RATIO) {
-                    CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath,
+                    CTX_LOG_ERRORF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)",
+                                   sub_job->filepath,
                                    decompressed_size_ratio);
                     break;
                 }
 
+                if ((archive_entry_is_encrypted(entry) || archive_entry_is_data_encrypted(entry) ||
+                     archive_entry_is_metadata_encrypted(entry)) && ctx->passphrase[0] == 0) {
+                    // Is encrypted but no password is specified, skip
+                    CTX_LOG_ERRORF("arc.c", "Skipped %s, archive is encrypted but no passphrase is supplied",
+                                   doc->filepath);
+                    break;
+                }
+
                 // Handle excludes
                 if (exclude != NULL && EXCLUDED(sub_job->filepath)) {
                     CTX_LOG_DEBUGF("arc.c", "Excluded: %s", sub_job->filepath);

third-party/libscan/libscan/ebook/ebook.c

@@ -153,22 +153,23 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
 
     sws_freeContext(sws_ctx);
 
-    // YUV420p -> JPEG
-    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(pixmap->w, pixmap->h, ctx->tn_qscale);
-    avcodec_send_frame(jpeg_encoder, scaled_frame);
+    // YUV420p -> JPEG/WEBP
+    AVCodecContext *thumbnail_encoder = alloc_webp_encoder(pixmap->w, pixmap->h, ctx->tn_qscale);
+    avcodec_send_frame(thumbnail_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, NULL); // Send EOF
 
-    AVPacket jpeg_packet;
-    av_init_packet(&jpeg_packet);
-    avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+    AVPacket thumbnail_packet;
+    av_init_packet(&thumbnail_packet);
+    avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
     APPEND_LONG_META(doc, MetaThumbnail, 1);
-    ctx->store(doc->doc_id, 0, (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store(doc->doc_id, 0, (char *) thumbnail_packet.data, thumbnail_packet.size);
 
     free(samples);
-    av_packet_unref(&jpeg_packet);
+    av_packet_unref(&thumbnail_packet);
     av_free(*scaled_frame->data);
     av_frame_free(&scaled_frame);
-    avcodec_free_context(&jpeg_encoder);
+    avcodec_free_context(&thumbnail_encoder);
 
     fz_drop_pixmap(fzctx, pixmap);
     fz_drop_page(fzctx, cover);

third-party/libscan/libscan/media/media.c

@@ -68,7 +68,7 @@ void *scale_frame(const AVCodecContext *decoder, const AVFrame *frame, int size)
 
     struct SwsContext *sws_ctx = sws_getContext(
             decoder->width, decoder->height, decoder->pix_fmt,
-            dstW, dstH, AV_PIX_FMT_YUVJ420P,
+            dstW, dstH, AV_PIX_FMT_YUV420P,
             SIST_SWS_ALGO, 0, 0, 0
     );
 
@@ -436,7 +436,8 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
         }
 
         if (seek_ok == FALSE && thumbnail_index != 0) {
-            CTX_LOG_WARNING(doc->filepath, "(media.c) Could not seek media file. Can't generate additional thumbnails.");
+            CTX_LOG_WARNING(doc->filepath,
+                            "(media.c) Could not seek media file. Can't generate additional thumbnails.");
             return SAVE_THUMBNAIL_FAILED;
         }
     }
@@ -470,18 +471,19 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
 
         ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
     } else {
-        // Encode frame to jpeg
-        AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
-                                                          ctx->tn_qscale);
-        avcodec_send_frame(jpeg_encoder, scaled_frame);
+        // Encode frame
+        AVCodecContext *thumbnail_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
+                                                               ctx->tn_qscale);
+        avcodec_send_frame(thumbnail_encoder, scaled_frame);
+        avcodec_send_frame(thumbnail_encoder, NULL); // send EOF
 
-        AVPacket jpeg_packet;
-        av_init_packet(&jpeg_packet);
-        avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+        AVPacket thumbnail_packet;
+        av_init_packet(&thumbnail_packet);
+        avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
         // Save thumbnail
         if (thumbnail_index == 0) {
-            ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size);
+            ctx->store(doc->doc_id, 0, thumbnail_packet.data, thumbnail_packet.size);
             return_value = SAVE_THUMBNAIL_OK;
 
         } else if (thumbnail_index > 1) {
@@ -489,15 +491,15 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
             //  I figure out a better fix.
             thumbnail_index -= 1;
 
-            ctx->store(doc->doc_id, thumbnail_index, jpeg_packet.data, jpeg_packet.size);
+            ctx->store(doc->doc_id, thumbnail_index, thumbnail_packet.data, thumbnail_packet.size);
 
             return_value = SAVE_THUMBNAIL_OK;
         } else {
             return_value = SAVE_THUMBNAIL_SKIPPED;
         }
 
-        avcodec_free_context(&jpeg_encoder);
-        av_packet_unref(&jpeg_packet);
+        avcodec_free_context(&thumbnail_encoder);
+        av_packet_unref(&thumbnail_packet);
         av_free(*scaled_frame->data);
         av_frame_free(&scaled_frame);
     }
@@ -854,9 +856,10 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
         ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
     } else {
         // Encode frame to jpeg
-        AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
+        AVCodecContext *jpeg_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
                                                           ctx->tn_qscale);
         avcodec_send_frame(jpeg_encoder, scaled_frame);
+        avcodec_send_frame(jpeg_encoder, NULL); // Send EOF
 
         AVPacket jpeg_packet;
         av_init_packet(&jpeg_packet);

third-party/libscan/libscan/media/media.h

@@ -48,6 +48,28 @@ static AVCodecContext *alloc_jpeg_encoder(int w, int h, int qscale) {
     return jpeg;
 }
 
+static AVCodecContext *alloc_webp_encoder(int w, int h, int qscale) {
+
+    const AVCodec *webp_codec = avcodec_find_encoder(AV_CODEC_ID_WEBP);
+    AVCodecContext *webp = avcodec_alloc_context3(webp_codec);
+    webp->width = w;
+    webp->height = h;
+    webp->time_base.den = 1000000;
+    webp->time_base.num = 1;
+    webp->compression_level = 6;
+    webp->global_quality = FF_QP2LAMBDA * qscale;
+
+    webp->pix_fmt = AV_PIX_FMT_YUV420P;
+    webp->color_range = AVCOL_RANGE_JPEG;
+    int ret = avcodec_open2(webp, webp_codec, NULL);
+
+    if (ret != 0) {
+        return NULL;
+    }
+
+    return webp;
+}
+
 
 void parse_media(scan_media_ctx_t *ctx, vfile_t *f, document_t *doc, const char *mime_str);
 

third-party/libscan/libscan/raw/raw.c

@@ -52,7 +52,7 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
 
     struct SwsContext *sws_ctx = sws_getContext(
             img->width, img->height, AV_PIX_FMT_RGB24,
-            dstW, dstH, AV_PIX_FMT_YUVJ420P,
+            dstW, dstH, AV_PIX_FMT_YUV420P,
             SIST_SWS_ALGO, 0, 0, 0
     );
 
@@ -76,20 +76,21 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
 
     sws_freeContext(sws_ctx);
 
-    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height, 1.0f);
-    avcodec_send_frame(jpeg_encoder, scaled_frame);
+    AVCodecContext *thumbnail_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height, ctx->tn_qscale);
+    avcodec_send_frame(thumbnail_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, NULL); // Send EOF
 
-    AVPacket jpeg_packet;
-    av_init_packet(&jpeg_packet);
-    avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+    AVPacket thumbnail_packet;
+    av_init_packet(&thumbnail_packet);
+    avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
     APPEND_LONG_META(doc, MetaThumbnail, 1);
-    ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) thumbnail_packet.data, thumbnail_packet.size);
 
-    av_packet_unref(&jpeg_packet);
+    av_packet_unref(&thumbnail_packet);
     av_free(*scaled_frame->data);
     av_frame_free(&scaled_frame);
-    avcodec_free_context(&jpeg_encoder);
+    avcodec_free_context(&thumbnail_encoder);
 
     return TRUE;
 }