Merge 4ec6a14252a5bf93a8e0eaeaa83e9317a1211990 into aff69fb3eb0abf3103428118cd4795f07e1b24a5

Added the version badge-pill to the sist2-admin page so you can check the version without going to the main search frontend.

sist2-admin/frontend/src/components/NavBar.vue

@@ -3,7 +3,11 @@
     <b-navbar-brand to="/">
       <Sist2Icon></Sist2Icon>
     </b-navbar-brand>
-
+    
+    <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
+      v{{ sist2Version() }}
+    </span>
+      
     <b-button class="ml-auto" to="/task" variant="link">{{ $t("tasks") }}</b-button>
   </b-navbar>
 </template>
@@ -66,4 +70,4 @@ export default {
 .btn-link {
   color: #222;
 }
-</style>
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,59 +1,66 @@
 <template>
-    <div>
+  <div>
-        <h4>{{ $t("webOptions.title") }}</h4>
+    <h4>{{ $t("webOptions.title") }}</h4>
-        <b-card>
+    <b-card>
-            <label>{{ $t("webOptions.lang") }}</label>
+      <label>{{ $t("webOptions.lang") }}</label>
-            <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
+      <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
-                           @change="update()"></b-form-select>
+                     @change="update()"></b-form-select>
 
-            <label>{{ $t("webOptions.bind") }}</label>
+      <label>{{ $t("webOptions.bind") }}</label>
-            <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+      <b-form-input v-model="options.bind" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.tagline") }}</label>
+      <label>{{ $t("webOptions.tagline") }}</label>
-            <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+      <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
 
-            <label>{{ $t("webOptions.auth") }}</label>
+      <label>{{ $t("webOptions.auth") }}</label>
-            <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+      <b-form-input v-model="options.auth" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.tagAuth") }}</label>
+      <label>{{ $t("webOptions.tagAuth") }}</label>
-            <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+      <b-form-input v-model="options.tag_auth" @change="update()" :disabled="Boolean(options.auth)"></b-form-input>
-        </b-card>
+    </b-card>
 
-        <br>
+    <br>
-        <h4>Auth0 options</h4>
+    <h4>Auth0 options</h4>
-        <b-card>
+    <b-card>
-            <label>{{ $t("webOptions.auth0Audience") }}</label>
+      <label>{{ $t("webOptions.auth0Audience") }}</label>
-            <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
+      <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.auth0Domain") }}</label>
+      <label>{{ $t("webOptions.auth0Domain") }}</label>
-            <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
+      <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.auth0ClientId") }}</label>
+      <label>{{ $t("webOptions.auth0ClientId") }}</label>
-            <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
+      <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.auth0PublicKey") }}</label>
+      <label>{{ $t("webOptions.auth0PublicKey") }}</label>
-            <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
+      <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-        </b-card>
+    </b-card>
-    </div>
+  </div>
 </template>
 
 <script>
 
 export default {
-    name: "WebOptions",
+  name: "WebOptions",
-    props: ["options", "frontendName"],
+  props: ["options", "frontendName"],
-    data() {
+  data() {
-        return {
+    return {
-            showEsTestAlert: false,
+      showEsTestAlert: false,
-            esTestOk: false,
+      esTestOk: false,
-            esTestMessage: "",
+      esTestMessage: ""
-        }
-    },
-    methods: {
-        update() {
-            this.$emit("change", this.options);
-        },
     }
+  },
+  methods: {
+    update() {
+
+      console.log(this.options)
+      if (this.options.auth && this.options.tag_auth) {
+        // If both are set, remove tagAuth
+        this.options.tag_auth = "";
+      }
+
+      this.$emit("change", this.options);
+    },
+  }
 }
 </script>
 

sist2-admin/frontend/src/i18n/messages.js

@@ -65,6 +65,9 @@ export default {
         gitRepository: "Git repository URL",
         extraArgs: "Extra command line arguments",
 
+        couldNotStartFrontend: "Could not start frontend",
+        couldNotStartFrontendBody: "Unable to start the frontend, check server logs for more details.",
+
         selectJobs: "Available jobs",
         selectJob: "Select a job",
         webOptions: {

sist2-admin/frontend/src/views/Frontend.vue

@@ -1,63 +1,63 @@
 <template>
-    <b-card>
+  <b-card>
-        <b-card-title>
+    <b-card-title>
-            {{ name }}
+      {{ name }}
-            <small style="vertical-align: top">
+      <small style="vertical-align: top">
-                <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
-                <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
-            </small>
+      </small>
-        </b-card-title>
+    </b-card-title>
 
-        <!-- Action buttons-->
+    <!-- Action buttons-->
-        <div class="mb-3" v-if="!loading">
+    <div class="mb-3" v-if="!loading">
-            <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
-                $t("start")
+          $t("start")
-                }}
+        }}
-            </b-button>
+      </b-button>
-            <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
-                $t("stop")
+          $t("stop")
-                }}
+        }}
-            </b-button>
+      </b-button>
-            <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
-                {{ $t("go") }}
+        {{ $t("go") }}
-            </b-button>
+      </b-button>
-            <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
-        </div>
+    </div>
 
-        <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
-        <b-card-body v-else>
+    <b-card-body v-else>
 
-            <h4>{{ $t("backendOptions.title") }}</h4>
+      <h4>{{ $t("backendOptions.title") }}</h4>
-            <b-card>
+      <b-card>
-                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
 
-                <SearchBackendSelect :value="frontend.web_options.search_backend"
+        <SearchBackendSelect :value="frontend.web_options.search_backend"
-                                     @change="onBackendSelect($event)"></SearchBackendSelect>
+                             @change="onBackendSelect($event)"></SearchBackendSelect>
 
-                <br>
+        <br>
-                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-            </b-card>
+      </b-card>
 
-            <br/>
+      <br/>
 
-            <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
+      <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
-                        @change="update()"></WebOptions>
+                  @change="update()"></WebOptions>
-            <br/>
+      <br/>
 
-            <h4>{{ $t("frontendOptions.title") }}</h4>
+      <h4>{{ $t("frontendOptions.title") }}</h4>
-            <b-card>
+      <b-card>
-                <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
-                    {{ $t("autoStart") }}
+          {{ $t("autoStart") }}
-                </b-form-checkbox>
+        </b-form-checkbox>
 
-                <label>{{ $t("extraQueryArgs") }}</label>
+        <label>{{ $t("extraQueryArgs") }}</label>
-                <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
 
-                <label>{{ $t("customUrl") }}</label>
+        <label>{{ $t("customUrl") }}</label>
-                <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
-            </b-card>
+      </b-card>
-        </b-card-body>
+    </b-card-body>
-    </b-card>
+  </b-card>
 </template>
 
 <script>
@@ -68,71 +68,78 @@ import WebOptions from "@/components/WebOptions";
 import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-    name: 'Frontend',
+  name: 'Frontend',
-    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
+  components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
-    data() {
+  data() {
-        return {
+    return {
-            loading: true,
+      loading: true,
-            frontend: null,
+      frontend: null,
-        }
-    },
-    computed: {
-        valid() {
-            return !this.loading && this.frontend.jobs.length > 0;
-        },
-        frontendUrl() {
-            if (this.frontend.custom_url) {
-                return this.frontend.custom_url + this.args;
-            }
-
-            if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
-                return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
-            }
-
-            return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
-        },
-        name() {
-            return this.$route.params.name;
-        },
-        port() {
-            return this.frontend.web_options.bind.split(":")[1]
-        },
-        args() {
-            const args = this.frontend.extra_query_args;
-            if (args !== "") {
-                return "#" + (args.startsWith("?") ? (args) : ("?" + args));
-            }
-            return "";
-        }
-    },
-    mounted() {
-        Sist2AdminApi.getFrontend(this.name).then(resp => {
-            this.frontend = resp.data;
-            this.loading = false;
-        });
-    },
-    methods: {
-        start() {
-            this.frontend.running = true;
-            Sist2AdminApi.startFrontend(this.name)
-        },
-        stop() {
-            this.frontend.running = false;
-            Sist2AdminApi.stopFrontend(this.name)
-        },
-        deleteFrontend() {
-            Sist2AdminApi.deleteFrontend(this.name).then(() => {
-                this.$router.push("/");
-            });
-        },
-        update() {
-            Sist2AdminApi.updateFrontend(this.name, this.frontend);
-        },
-        onBackendSelect(backend) {
-            this.frontend.web_options.search_backend = backend;
-            this.frontend.jobs = [];
-            this.update();
-        }
     }
+  },
+  computed: {
+    valid() {
+      return !this.loading && this.frontend.jobs.length > 0;
+    },
+    frontendUrl() {
+      if (this.frontend.custom_url) {
+        return this.frontend.custom_url + this.args;
+      }
+
+      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+      }
+
+      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+    },
+    name() {
+      return this.$route.params.name;
+    },
+    port() {
+      return this.frontend.web_options.bind.split(":")[1]
+    },
+    args() {
+      const args = this.frontend.extra_query_args;
+      if (args !== "") {
+        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+      }
+      return "";
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getFrontend(this.name).then(resp => {
+      this.frontend = resp.data;
+      this.loading = false;
+    });
+  },
+  methods: {
+    start() {
+      Sist2AdminApi.startFrontend(this.name).then(() => {
+        this.frontend.running = true;
+      }).catch(() => {
+        this.$bvToast.toast(this.$t("couldNotStartFrontendBody"), {
+          title: this.$t("couldNotStartFrontend"),
+          variant: "danger",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    },
+    stop() {
+      this.frontend.running = false;
+      Sist2AdminApi.stopFrontend(this.name)
+    },
+    deleteFrontend() {
+      Sist2AdminApi.deleteFrontend(this.name).then(() => {
+        this.$router.push("/");
+      });
+    },
+    update() {
+      Sist2AdminApi.updateFrontend(this.name, this.frontend);
+    },
+    onBackendSelect(backend) {
+      this.frontend.web_options.search_backend = backend;
+      this.frontend.jobs = [];
+      this.update();
+    }
+  }
 }
 </script>

sist2-admin/sist2_admin/app.py

@@ -2,6 +2,7 @@ import asyncio
 import os
 import signal
 from datetime import datetime
+from time import sleep
 from urllib.parse import urlparse
 
 import requests
@@ -25,6 +26,7 @@ from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEM
     get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
 from script import UserScript, SCRIPT_TEMPLATES
+from util import tail_sync, pid_is_running
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
 db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
@@ -324,7 +326,18 @@ def start_frontend_(frontend: Sist2Frontend):
     logger.debug(f"Fetched search backend options for {backend_name}")
 
     pid = sist2.web(frontend.web_options, search_backend, frontend.name)
+
+    sleep(0.2)
+    if not pid_is_running(pid):
+        frontend_log = frontend.get_log_path(LOG_FOLDER)
+        logger.error(f"Frontend exited too quickly, check {frontend_log} for more details:")
+        for line in tail_sync(frontend.get_log_path(LOG_FOLDER), 3):
+            logger.error(line.strip())
+
+        return False
+
     RUNNING_FRONTENDS[frontend.name] = pid
+    return True
 
 
 @app.post("/api/frontend/{name:str}/start")
@@ -333,7 +346,12 @@ async def start_frontend(name: str):
     if not frontend:
         raise HTTPException(status_code=404)
 
-    start_frontend_(frontend)
+    ok = start_frontend_(frontend)
+
+    if not ok:
+        raise HTTPException(status_code=500)
+
+    return "ok"
 
 
 @app.post("/api/frontend/{name:str}/stop")

sist2-admin/sist2_admin/sist2.py

@@ -257,7 +257,7 @@ class Sist2:
 
         set_pid_cb(proc.pid)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -284,7 +284,7 @@ class Sist2:
 
         set_pid_cb(proc.pid)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -294,7 +294,7 @@ class Sist2:
         return proc.returncode
 
     @staticmethod
-    def _consume_logs_stderr(logs_cb, proc):
+    def _consume_logs_stderr(logs_cb, exit_cb, proc):
         pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8", errors="ignore")
         try:
             for line in pipe_wrapper:
@@ -302,7 +302,9 @@ class Sist2:
                     continue
                 logs_cb({"stderr": line})
         finally:
-            proc.wait()
+            return_code = proc.wait()
+            if exit_cb:
+                exit_cb(return_code)
             pipe_wrapper.close()
 
     @staticmethod
@@ -340,11 +342,14 @@ class Sist2:
         def logs_cb(message):
             web_logger.info(json.dumps(message))
 
+        def exit_cb(return_code):
+            logger.info(f"Web frontend exited with return code {return_code}")
+
         logger.info(f"Starting frontend {' '.join(args)}")
 
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, exit_cb, proc))
         t_stderr.start()
 
         t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))

sist2-admin/sist2_admin/util.py

@@ -0,0 +1,41 @@
+from glob import glob
+import os
+from config import DATA_FOLDER
+
+
+def get_old_index_files(name):
+    files = glob(os.path.join(DATA_FOLDER, f"scan-{name.replace('/', '_')}-*.sist2"))
+    files = list(sorted(files, key=lambda f: os.stat(f).st_mtime))
+    files = files[-1:]
+
+    return files
+
+
+def tail_sync(filename, lines=1, _buffer=4098):
+    with open(filename) as f:
+        lines_found = []
+
+        block_counter = -1
+
+        while len(lines_found) < lines:
+            try:
+                f.seek(block_counter * _buffer, os.SEEK_END)
+            except IOError:
+                f.seek(0)
+                lines_found = f.readlines()
+                break
+
+            lines_found = f.readlines()
+
+            block_counter -= 1
+
+        return lines_found[-lines:]
+
+
+def pid_is_running(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+
+    return True