console log fixes, version bump

v2.11.4

.drone.yml

@@ -10,7 +10,7 @@ steps:
   - name: build
     image: simon987/sist2-build
     commands:
-      - ./ci/build.sh
+      - ./scripts/build.sh
   - name: docker
     image: plugins/docker
     settings:
@@ -55,7 +55,7 @@ steps:
   - name: build
     image: simon987/sist2-build-arm64
     commands:
-      - ./ci/build_arm64.sh
+      - ./scripts/build_arm64.sh
   - name: scp files
     image: appleboy/drone-scp
     settings:

Dockerfile

@@ -6,12 +6,10 @@ COPY . .
 RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
 RUN make -j$(nproc)
 RUN strip sist2
-RUN ls -lh
-RUN ls -lh sist2-vue/dist/
 
-FROM ubuntu:20.10
+FROM ubuntu:21.10
 
-RUN apt update && apt install -y curl libasan5
+RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -22,9 +20,9 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
     curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-COPY --from=build /build/sist2 /root/sist2
+ENTRYPOINT ["/root/sist2"]
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 
-ENTRYPOINT ["/root/sist2"]
+COPY --from=build /build/sist2 /root/sist2

Dockerfile.arm64

@@ -7,9 +7,9 @@ RUN cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE
 RUN make -j$(nproc)
 RUN strip sist2
 
-FROM ubuntu:20.10
+FROM --platform="linux/arm64/v8" ubuntu:21.10
 
-RUN apt update && apt install -y curl libasan5
+RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -20,9 +20,9 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
     curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-COPY --from=build /build/sist2 /root/sist2
-
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 
-ENTRYPOINT ["/root/sist2"]
+ENTRYPOINT ["/root/sist2"]
+
+COPY --from=build /build/sist2 /root/sist2

README.md

@@ -10,7 +10,7 @@ sist2 (Simple incremental search tool)
 
 *Warning: sist2 is in early development*
 
-![sist2.png](docs/sist2.png)
+![search panel](docs/sist2.png)
 
 ## Features
 

docs/USAGE.md

@@ -315,7 +315,7 @@ See [scripting](scripting.md) documentation.
 # Sidecar files
 
 When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
-original document's metadata. Sidecar metadata files will also work inside archives.
+original document's indexed metadata (does not modify the actual file). Sidecar metadata files will also work inside archives.
 Sidecar files themselves are not saved in the index.
 
 This feature is useful to leverage third-party applications such as speech-to-text or

schema/mappings.json

@@ -78,6 +78,7 @@
     "name": {
       "analyzer": "content_analyzer",
       "type": "text",
+      "fielddata": true,
       "fields": {
         "nGram": {
           "type": "text",

sist2-vue/src/Sist2Query.ts

@@ -43,6 +43,20 @@ const SORT_MODES = {
             {_tie: {order: "asc"}}
         ],
         key: (hit: EsHit) => hit._source.size
+    },
+    nameAsc: {
+        mode: [
+            {name: {order: "asc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._source.name
+    },
+    nameDesc: {
+        mode: [
+            {name: {order: "desc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._source.name
     }
 } as any;
 

sist2-vue/src/components/IndexPicker.vue

@@ -7,11 +7,27 @@
         value-field="id"></b-form-select>
   </div>
   <div v-else>
-    <b-list-group id="index-picker-desktop">
+
+    <div class="d-flex justify-content-between align-content-center">
+      <span>
+        {{ selectedIndices.length }}
+        {{ selectedIndices.length === 1 ? $t("indexPicker.selectedIndex") : $t("indexPicker.selectedIndices") }}
+      </span>
+
+      <div>
+        <b-button variant="link" @click="selectAll()"> {{ $t("indexPicker.selectAll") }}</b-button>
+        <b-button variant="link" @click="selectNone()"> {{ $t("indexPicker.selectNone") }}</b-button>
+      </div>
+    </div>
+
+    <b-list-group id="index-picker-desktop" class="unselectable">
       <b-list-group-item
           v-for="idx in indices"
-          @click="toggleIndex(idx)"
-          class="d-flex justify-content-between align-items-center list-group-item-action pointer">
+          @click="toggleIndex(idx, $event)"
+          @click.shift="shiftClick(idx, $event)"
+          class="d-flex justify-content-between align-items-center list-group-item-action pointer"
+          :class="{active: lastClickIndex === idx}"
+      >
         <div class="d-flex">
           <b-checkbox @change="toggleIndex(idx)" :checked="isSelected(idx)"></b-checkbox>
           {{ idx.name }}
@@ -36,6 +52,7 @@ export default Vue.extend({
   data() {
     return {
       loading: true,
+      lastClickIndex: null
     }
   },
   computed: {
@@ -53,13 +70,50 @@ export default Vue.extend({
     ...mapActions({
       setSelectedIndices: "setSelectedIndices"
     }),
+    shiftClick(index, e) {
+      if (this.lastClickIndex === null) {
+        return;
+      }
+
+      const select = this.isSelected(this.lastClickIndex);
+
+      let leftBoundary = this.indices.indexOf(this.lastClickIndex);
+      let rightBoundary = this.indices.indexOf(index);
+
+      if (rightBoundary < leftBoundary) {
+        let tmp = leftBoundary;
+        leftBoundary = rightBoundary;
+        rightBoundary = tmp;
+      }
+
+      for (let i = leftBoundary; i <= rightBoundary; i++) {
+        if (select) {
+          if (!this.isSelected(this.indices[i])) {
+            this.setSelectedIndices([this.indices[i], ...this.selectedIndices]);
+          }
+        } else {
+          this.setSelectedIndices(this.selectedIndices.filter(idx => idx !== this.indices[i]));
+        }
+      }
+    },
+    selectAll() {
+      this.setSelectedIndices(this.indices);
+    },
+    selectNone() {
+      this.setSelectedIndices([]);
+    },
     onSelect(value) {
       this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
     },
     formatIdxDate(timestamp: number): string {
       return format(new Date(timestamp * 1000), "yyyy-MM-dd");
     },
-    toggleIndex(index) {
+    toggleIndex(index, e) {
+      if (e.shiftKey) {
+        return;
+      }
+
+      this.lastClickIndex = index;
       if (this.isSelected(index)) {
         this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
       } else {
@@ -92,4 +146,21 @@ export default Vue.extend({
   overflow-y: auto;
   max-height: 132px;
 }
+
+.btn-link:focus {
+  box-shadow: none;
+}
+
+.unselectable {
+  user-select: none;
+  -ms-user-select: none;
+  -moz-user-select: none;
+  -webkit-user-select: none;
+}
+
+.list-group-item.active {
+  z-index: 2;
+  background-color: inherit;
+  color: inherit;
+}
 </style>

sist2-vue/src/components/ResultsCard.vue

@@ -23,7 +23,7 @@
 <script lang="ts">
 import {EsResult} from "@/Sist2Api";
 import Vue from "vue";
-import {humanFileSize, humanTime} from "@/util";
+import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
 import SortSelect from "@/components/SortSelect.vue";
 

sist2-vue/src/components/SortSelect.vue

@@ -19,6 +19,14 @@
       {{ $t("sort.sizeDesc") }}
     </b-dropdown-item>
 
+    <b-dropdown-item :class="{'dropdown-active': sort === 'nameDesc'}" @click="onSelect('nameDesc')">
+      {{ $t("sort.nameDesc") }}
+    </b-dropdown-item>
+
+    <b-dropdown-item :class="{'dropdown-active': sort === 'nameAsc'}" @click="onSelect('nameAsc')">
+      {{ $t("sort.nameAsc") }}
+    </b-dropdown-item>
+
     <b-dropdown-item :class="{'dropdown-active': sort === 'random'}" @click="onSelect('random')">
       {{ $t("sort.random") }}
     </b-dropdown-item>

sist2-vue/src/i18n/messages.ts

@@ -131,13 +131,14 @@ export default {
         saveTagModalTitle: "Add tag",
         saveTagPlaceholder: "Tag name",
         confirm: "Confirm",
-        indexPickerPlaceholder: "Select indices",
         sort: {
             relevance: "Relevance",
             dateAsc: "Date (Older first)",
             dateDesc: "Date (Newer first)",
             sizeAsc: "Size (Smaller first)",
             sizeDesc: "Size (Larger first)",
+            nameAsc: "Name (A-z)",
+            nameDesc: "Name (Z-a)",
             random: "Random",
         },
         d3: {
@@ -145,7 +146,13 @@ export default {
             mimeSize: "Size distribution by media type",
             dateHistogram: "File modification time distribution",
             sizeHistogram: "File size distribution",
-        }
+        },
+        indexPicker: {
+            selectNone: "Select None",
+            selectAll: "Select All",
+            selectedIndex: "selected index",
+            selectedIndices: "selected indices",
+        },
     },
     fr: {
         searchBar: {
@@ -288,6 +295,8 @@ export default {
             dateDesc: "Date (Plus récent)",
             sizeAsc: "Taille (Plus petit)",
             sizeDesc: "Taille (Plus grand)",
+            nameAsc: "Nom (A-z)",
+            nameDesc: "Nom (Z-a)",
             random: "Aléatoire",
         },
         d3: {
@@ -295,6 +304,12 @@ export default {
             mimeSize: "Distribution des tailles de fichiers par type de média",
             dateHistogram: "Distribution des dates de modification",
             sizeHistogram: "Distribution des tailles de fichier",
-        }
+        },
+        indexPicker: {
+            selectNone: "Sélectionner aucun",
+            selectAll: "Sélectionner tout",
+            selectedIndex: "indice sélectionné",
+            selectedIndices: "indices sélectionnés",
+        },
     }
 }

src/log.c

@@ -55,10 +55,14 @@ void vsist_logf(const char *filepath, int level, char *format, va_list ap) {
         log_len += 1;
     }
 
-    int ret = write(STDERR_FILENO, log_str, log_len);
-    if (ret == -1) {
-        LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno))
+    if (PrintingProgressBar) {
+        PrintingProgressBar = FALSE;
+        memmove(log_str + 1, log_str, log_len);
+        log_str[0] = '\n';
+        log_len += 1;
     }
+
+    write(STDERR_FILENO, log_str, log_len);
 }
 
 void sist_logf(const char *filepath, int level, char *format, ...) {
@@ -104,8 +108,12 @@ void sist_log(const char *filepath, int level, char *str) {
         );
     }
 
-    int ret = write(STDERR_FILENO, log_str, log_len);
-    if (ret == -1) {
-        LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno));
+    if (PrintingProgressBar) {
+        PrintingProgressBar = FALSE;
+        memmove(log_str + 1, log_str, log_len);
+        log_str[0] = '\n';
+        log_len += 1;
     }
+
+    write(STDERR_FILENO, log_str, log_len);
 }

src/main.c

@@ -433,7 +433,7 @@ void sist2_index(index_args_t *args) {
         cleanup = elastic_cleanup;
     }
 
-    IndexCtx.pool = tpool_create(args->threads, cleanup, FALSE, TRUE);
+    IndexCtx.pool = tpool_create(args->threads, cleanup, FALSE, args->print == 0);
     tpool_start(IndexCtx.pool);
 
     struct dirent *de;
@@ -518,8 +518,8 @@ void sist2_web(web_args_t *args) {
 
 
 int main(int argc, const char *argv[]) {
-//    sigsegv_handler = signal(SIGSEGV, sig_handler);
-//    sigabrt_handler = signal(SIGABRT, sig_handler);
+    sigsegv_handler = signal(SIGSEGV, sig_handler);
+    sigabrt_handler = signal(SIGABRT, sig_handler);
 
     setlocale(LC_ALL, "");
 

src/sist.h

@@ -53,7 +53,7 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "2.11.4"
+#define VERSION "2.11.5"
 static const char *const Version = VERSION;
 
 #ifndef SIST_PLATFORM

src/tpool.c

@@ -191,7 +191,9 @@ void tpool_wait(tpool_t *pool) {
             }
         }
     }
-    progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
+    if (pool->print_progress) {
+        progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
+    }
     pthread_mutex_unlock(&(pool->work_mutex));
 
     LOG_INFO("tpool.c", "Worker threads finished")

src/util.c

@@ -84,6 +84,8 @@ char *expandpath(const char *path) {
     return expanded;
 }
 
+int PrintingProgressBar = 0;
+
 void progress_bar_print(double percentage, size_t tn_size, size_t index_size) {
 
     static int last_val = -1;
@@ -114,13 +116,21 @@ void progress_bar_print(double percentage, size_t tn_size, size_t index_size) {
         index_unit = 'M';
     }
 
-    printf(
-            "\r%3d%%[%.*s>%*s] TN:%3d%c IDX:%3d%c",
-            val, lpad, PBSTR, rpad, "",
-            (int) tn_size, tn_unit,
-            (int) index_size, index_unit
-    );
-    fflush(stdout);
+    if (tn_size == 0 && index_size == 0) {
+        fprintf(stderr,
+                "\r%3d%%[%.*s>%*s]",
+                val, lpad, PBSTR, rpad, ""
+        );
+    } else {
+        fprintf(stderr,
+                "\r%3d%%[%.*s>%*s] TN:%3d%c IDX:%3d%c",
+                val, lpad, PBSTR, rpad, "",
+                (int) tn_size, tn_unit,
+                (int) index_size, index_unit
+        );
+    }
+
+    PrintingProgressBar = TRUE;
 }
 
 GHashTable *incremental_get_table() {

src/util.h

@@ -19,6 +19,8 @@ char *expandpath(const char *path);
 
 dyn_buffer_t url_escape(char *str);
 
+extern int PrintingProgressBar;
+
 void progress_bar_print(double percentage, size_t tn_size, size_t index_size);
 
 GHashTable *incremental_get_table();

third-party/libscan/CMakeLists.txt

@@ -57,7 +57,7 @@ find_library(LZO2_LIB NAMES lzo2)
 
 find_library(RAW_LIB NAMES libraw.a)
 find_library(MUPDF_LIB NAMES liblibmupdf.a)
-find_library(CMS_LIB NAMES lcms)
+find_library(CMS_LIB NAMES lcms2)
 find_library(JAS_LIB NAMES jasper)
 find_library(GUMBO_LIB NAMES gumbo)
 find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/)