SQLite backend support for sist2-admin #366

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -33,9 +33,26 @@ class Sist2AdminApi {
         return axios.get(`${this.baseUrl}/api/job/${name}`);
     }
 
-    /**
+    getSearchBackend(name) {
-     * @param {string} name
+        return axios.get(`${this.baseUrl}/api/search_backend/${name}`);
-     */
+    }
+
+    updateSearchBackend(name, data) {
+        return axios.put(`${this.baseUrl}/api/search_backend/${name}`, data);
+    }
+
+    getSearchBackends() {
+        return axios.get(`${this.baseUrl}/api/search_backend/`);
+    }
+
+    deleteBackend(name) {
+        return axios.delete(`${this.baseUrl}/api/search_backend/${name}`)
+    }
+
+    createBackend(name) {
+        return axios.post(`${this.baseUrl}/api/search_backend/${name}`);
+    }
+
     getFrontend(name) {
         return axios.get(`${this.baseUrl}/api/frontend/${name}`);
     }

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -1,64 +0,0 @@
-<template>
-  <div>
-    <label>{{ $t("indexOptions.threads") }}</label>
-    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
-    <b-input-group>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
-
-    <label>{{ $t("indexOptions.esIndex") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
-
-    <br>
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
-      {{ $t("webOptions.esInsecure") }}
-    </b-form-checkbox>
-
-    <label>{{ $t("indexOptions.batchSize") }}</label>
-    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("indexOptions.script") }}</label>
-    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
-  </div>
-
-</template>
-
-<script>
-import sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-  name: "IndexOptions",
-  props: ["options"],
-  data() {
-    return {
-      showEsTestAlert: false,
-      esTestOk: false,
-      esTestMessage: "",
-    }
-  },
-  methods: {
-    update() {
-      this.$emit("change", this.options);
-    },
-    testEs() {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
-        this.showEsTestAlert = true;
-        this.esTestOk = resp.data.ok;
-        this.esTestMessage = resp.data.message;
-      });
-    }
-  },
-}
-</script>
-
-<style scoped>
-
-</style>

sist2-admin/frontend/src/components/SearchBackendListItem.vue

@@ -0,0 +1,24 @@
+<template>
+    <b-list-group-item action :to="`/searchBackend/${backend.name}`">
+
+        <div class="d-flex w-100 justify-content-between">
+            <h5 class="mb-1">
+                {{ backend.name }}
+            </h5>
+
+            <div>
+                <b-badge v-if="backend.backend_type === 'sqlite'" variant="info">SQLite</b-badge>
+                <b-badge v-else variant="info">Elasticsearch</b-badge>
+            </div>
+        </div>
+
+    </b-list-group-item>
+</template>
+
+<script>
+
+export default {
+    name: "SearchBackendListItem",
+    props: ["backend"],
+}
+</script>

sist2-admin/frontend/src/components/SearchBackendSelect.vue

@@ -0,0 +1,37 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <div v-else>
+        <label>{{$t("backendOptions.searchBackend")}}</label>
+        <b-select :options="options" :value="value" @change="$emit('change', $event)"></b-select>
+    </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackendSelect",
+    props: ["value"],
+    data() {
+        return {
+            loading: true,
+            backends: null,
+        }
+    },
+    computed: {
+        options() {
+            return this.backends.map(backend => backend.name)
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackends().then(resp => {
+            this.loading = false;
+            this.backends = resp.data
+        })
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,23 +1,5 @@
 <template>
     <div>
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
-
-    <b-input-group>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
-
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
-      {{ $t("webOptions.esInsecure") }}
-    </b-form-checkbox>
-
-    <label>{{ $t("webOptions.esIndex") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
 
         <label>{{ $t("webOptions.lang") }}</label>
         <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
@@ -47,9 +29,6 @@
 
         <label>{{ $t("webOptions.auth0PublicKey") }}</label>
         <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-
-
-
     </div>
 </template>
 
@@ -75,13 +54,6 @@ export default {
 
             this.$emit("change", this.options);
         },
-    testEs() {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
-        this.showEsTestAlert = true;
-        this.esTestOk = resp.data.ok;
-        this.esTestMessage = resp.data.message;
-      });
-    }
     }
 }
 </script>

sist2-admin/frontend/src/i18n/messages.js

@@ -48,12 +48,13 @@ export default {
         extraQueryArgs: "Extra query arguments when launching from sist2-admin",
         customUrl: "Custom URL when launching from sist2-admin",
 
+        searchBackends: "Search backends",
+        searchBackendTitle: "search backend configuration",
+        newBackendName: "New search backend name",
+
         selectJobs: "Select jobs",
         webOptions: {
             title: "Web options",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
             lang: "UI Language",
             bind: "Listen address",
             tagline: "Tagline in navbar",
@@ -64,6 +65,18 @@ export default {
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
         },
+        backendOptions: {
+            title: "Search backend options",
+            searchBackend: "Search backend",
+            type: "Search backend type",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            threads: "Number of threads",
+            batchSize: "Index batch size",
+            script: "User script",
+            searchIndex: "Search index file location"
+        },
         scanOptions: {
             title: "Scanning options",
             path: "Path",
@@ -90,15 +103,6 @@ export default {
             treemapThreshold: "Relative size threshold for treemap",
             optimizeIndex: "Defragment index file after scan to reduce its file size."
         },
-        indexOptions: {
-            title: "Indexing options",
-            threads: "Number of threads",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
-            batchSize: "Index batch size",
-            script: "User script"
-        },
         jobOptions: {
             title: "Job options",
             cron: "Job schedule",
@@ -106,6 +110,7 @@ export default {
             deleteNow: "Delete now",
             scheduleEnabled: "Enable scheduled re-scan",
             noJobAvailable: "No jobs available.",
+            noBackendError: "You must select a search backend to run this job",
             desktopNotifications: "Desktop notifications"
         },
         frontendOptions: {

sist2-admin/frontend/src/router/index.js

@@ -5,6 +5,7 @@ import Job from "@/views/Job";
 import Tasks from "@/views/Tasks";
 import Frontend from "@/views/Frontend";
 import Tail from "@/views/Tail";
+import SearchBackend from "@/views/SearchBackend.vue";
 
 Vue.use(VueRouter);
 
@@ -29,6 +30,11 @@ const routes = [
     name: "Frontend",
     component: Frontend
   },
+  {
+    path: "/searchBackend/:name",
+    name: "SearchBackend",
+    component: SearchBackend
+  },
   {
     path: "/log/:taskId",
     name: "Tail",

sist2-admin/frontend/src/views/Frontend.vue

@@ -50,10 +50,20 @@
 
             <h4>{{ $t("webOptions.title") }}</h4>
             <b-card>
-        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
+                <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
+                            @change="update()"></WebOptions>
+            </b-card>
+
+            <br>
+
+            <h4>{{ $t("backendOptions.title") }}</h4>
+            <b-card>
+                <SearchBackendSelect :value="frontend.web_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
             </b-card>
         </b-card-body>
 
+
     </b-card>
 </template>
 
@@ -62,10 +72,11 @@
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobCheckboxGroup from "@/components/JobCheckboxGroup";
 import WebOptions from "@/components/WebOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
     name: 'Frontend',
-  components: {JobCheckboxGroup, WebOptions},
+    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
     data() {
         return {
             loading: true,
@@ -118,12 +129,16 @@ export default {
         },
         deleteFrontend() {
             Sist2AdminApi.deleteFrontend(this.name).then(() => {
-        this.$router.push("/frontends");
+                this.$router.push("/");
             });
         },
         update() {
             Sist2AdminApi.updateFrontend(this.name, this.frontend);
         },
+        onBackendSelect(backend) {
+            this.frontend.web_options.search_backend = backend;
+            this.update();
+        }
     }
 }
 </script>

sist2-admin/frontend/src/views/Home.vue

@@ -15,7 +15,8 @@
                     ></b-popover>
                 </b-col>
                 <b-col>
-          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
+                    <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">
+                        {{ $t("create") }}
                     </b-button>
                 </b-col>
             </b-row>
@@ -39,7 +40,8 @@
                     <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
                 </b-col>
                 <b-col>
-          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
+                    <b-button variant="primary" @click="createFrontend()"
+                              :disabled="!frontendNameValid(newFrontendName)">
                         {{ $t("create") }}
                     </b-button>
                 </b-col>
@@ -54,6 +56,33 @@
             </b-list-group>
 
         </b-card>
+
+        <br/>
+
+        <b-card>
+            <b-card-title>{{ $t("searchBackends") }}</b-card-title>
+
+            <b-row>
+                <b-col>
+                    <b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
+                </b-col>
+                <b-col>
+                    <b-button variant="primary" @click="createBackend()"
+                              :disabled="!backendNameValid(newBackendName)">
+                        {{ $t("create") }}
+                    </b-button>
+                </b-col>
+            </b-row>
+
+            <hr/>
+
+            <b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
+            <b-list-group v-else>
+                <SearchBackendListItem v-for="backend in backends"
+                                  :key="backend.name" :backend="backend"></SearchBackendListItem>
+            </b-list-group>
+
+        </b-card>
     </div>
 </template>
 
@@ -62,10 +91,11 @@ import JobListItem from "@/components/JobListItem";
 import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
+import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
 
 export default {
     name: "Jobs",
-  components: {JobListItem, FrontendListItem},
+    components: {SearchBackendListItem, JobListItem, FrontendListItem},
     data() {
         return {
             jobsLoading: true,
@@ -77,6 +107,10 @@ export default {
             formatBindAddress,
             newFrontendName: "",
 
+            backends: [],
+            backendsLoading: true,
+            newBackendName: "",
+
             showHelp: false
         }
     },
@@ -93,7 +127,14 @@ export default {
             return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
         },
         frontendNameValid(name) {
-      if (this.frontends.some(job => job.name === name)) {
+            if (this.frontends.some(frontend => frontend.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        backendNameValid(name) {
+            if (this.backends.some(backend => backend.name === name)) {
                 return false;
             }
 
@@ -110,12 +151,19 @@ export default {
                 this.frontends = resp.data;
                 this.frontendsLoading = false;
             });
+            Sist2AdminApi.getSearchBackends().then(resp => {
+                this.backends = resp.data;
+                this.backendsLoading = false;
+            })
         },
         createJob() {
             Sist2AdminApi.createJob(this.newJobName).then(this.reload);
         },
         createFrontend() {
             Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+        },
+        createBackend() {
+            Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
         }
     }
 }

sist2-admin/frontend/src/views/Job.vue

@@ -6,7 +6,7 @@
         </b-card-title>
 
         <div class="mb-3">
-      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
+            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
             <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
         </div>
 
@@ -31,9 +31,11 @@
 
             <br/>
 
-      <h4>{{ $t("indexOptions.title") }}</h4>
+            <h4>{{ $t("backendOptions.title") }}</h4>
             <b-card>
-        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("jobOptions.noBackendError") }}</b-alert>
+                <SearchBackendSelect :value="job.index_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
             </b-card>
 
         </b-card-body>
@@ -44,20 +46,20 @@
 <script>
 import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
-import IndexOptions from "@/components/IndexOptions";
 import JobOptions from "@/components/JobOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
     name: "Job",
     components: {
-    IndexOptions,
+        SearchBackendSelect,
         ScanOptions,
         JobOptions
     },
     data() {
         return {
             loading: true,
-      job: null
+            job: null,
         }
     },
     methods: {
@@ -77,9 +79,22 @@ export default {
             });
         },
         deleteJob() {
-      Sist2AdminApi.deleteJob(this.getName()).then(() => {
+            Sist2AdminApi.deleteJob(this.getName())
+                .then(() => {
                     this.$router.push("/");
                 })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete job " +
+                        "because it is referenced by a frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        onBackendSelect(backend) {
+            this.job.index_options.search_backend = backend;
+            this.update();
         }
     },
     mounted() {
@@ -87,6 +102,11 @@ export default {
             this.loading = false;
             this.job = resp.data;
         })
+    },
+    computed: {
+        valid() {
+            return this.job?.index_options.search_backend != null;
+        }
     }
 }
 </script>

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -0,0 +1,126 @@
+<template>
+
+    <b-card>
+        <b-card-title>
+            <span class="text-monospace">{{ getName() }}</span>
+            {{ $t("searchBackendTitle") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteBackend()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
+
+            <label>{{ $t("backendOptions.type") }}</label>
+            <b-select :options="backendTypeOptions" v-model="backend.backend_type" @change="update()"></b-select>
+
+            <hr/>
+
+            <template v-if="backend.backend_type === 'elasticsearch'">
+                <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+                    {{ esTestMessage }}
+                </b-alert>
+
+                <label>{{ $t("backendOptions.esUrl") }}</label>
+                <b-input-group>
+                    <b-form-input v-model="backend.es_url" @change="update()"></b-form-input>
+                    <b-input-group-append>
+                        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+                    </b-input-group-append>
+                </b-input-group>
+
+                <b-form-checkbox v-model="backend.es_insecure_ssl" :disabled="!this.backend.es_url.startsWith('https')"
+                                 @change="update()">
+                    {{ $t("backendOptions.esInsecure") }}
+                </b-form-checkbox>
+
+                <label>{{ $t("backendOptions.esIndex") }}</label>
+                <b-form-input v-model="backend.es_index" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.threads") }}</label>
+                <b-form-input v-model="backend.threads" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.batchSize") }}</label>
+                <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.script") }}</label>
+                <b-form-textarea v-model="backend.script" rows="6" @change="update()"></b-form-textarea>
+            </template>
+            <template v-else>
+                <label>{{ $t("backendOptions.searchIndex") }}</label>
+                <b-form-input v-model="backend.search_index" disabled></b-form-input>
+            </template>
+        </b-card-body>
+
+    </b-card>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackend",
+    data() {
+        return {
+            showEsTestAlert: false,
+            esTestOk: false,
+            esTestMessage: "",
+            loading: true,
+            backend: null,
+            backendTypeOptions: [
+                {
+                    text: "Elasticsearch",
+                    value: "elasticsearch"
+                },
+                {
+                    text: "SQLite",
+                    value: "sqlite"
+                }
+            ]
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackend(this.getName()).then(resp => {
+            this.backend = resp.data;
+            this.loading = false;
+        });
+    },
+    methods: {
+        getName() {
+            return this.$route.params.name;
+        },
+        testEs() {
+            sist2AdminApi.pingEs(this.backend.es_url, this.backend.es_insecure_ssl)
+                .then((resp) => {
+                    this.showEsTestAlert = true;
+                    this.esTestOk = resp.data.ok;
+                    this.esTestMessage = resp.data.message;
+                });
+        },
+        update() {
+            Sist2AdminApi.updateSearchBackend(this.getName(), this.backend);
+        },
+        deleteBackend() {
+            Sist2AdminApi.deleteBackend(this.getName())
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete search backend " +
+                        "because it is referenced by a job or frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/sist2_admin/app.py

@@ -20,9 +20,9 @@ import cron
 from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
 from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
 from notifications import Subscribe, Notifications
-from sist2 import Sist2
+from sist2 import Sist2, Sist2SearchBackend
 from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
-    get_log_files_to_remove, delete_log_file
+    get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
@@ -174,12 +174,22 @@ async def task_history(n: int, name: str):
 
 @app.delete("/api/job/{name:str}")
 async def delete_job(name: str):
-    job = db["jobs"][name]
+    job: Sist2Job = db["jobs"][name]
-    if job:
+    if not job:
-        del db["jobs"][name]
-    else:
         raise HTTPException(status_code=404)
 
+    if any(name in frontend.jobs for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    try:
+        os.remove(job.previous_index)
+    except:
+        pass
+
+    del db["jobs"][name]
+
+    return "ok"
+
 
 @app.delete("/api/frontend/{name:str}")
 async def delete_frontend(name: str):
@@ -267,7 +277,16 @@ def check_es_version(es_url: str, insecure: bool):
 def start_frontend_(frontend: Sist2Frontend):
     frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
 
-    pid = sist2.web(frontend.web_options, frontend.name)
+    backend_name = frontend.web_options.search_backend
+    search_backend = db["search_backends"][backend_name]
+    if search_backend is None:
+        logger.error(
+            f"Error while running task: search backend not found: {backend_name}")
+        return -1
+
+    logger.debug(f"Fetched search backend options for {backend_name}")
+
+    pid = sist2.web(frontend.web_options, search_backend, frontend.name)
     RUNNING_FRONTENDS[frontend.name] = pid
 
 
@@ -297,6 +316,62 @@ async def get_frontends():
     return res
 
 
+@app.get("/api/search_backend/")
+async def get_search_backends():
+    return list(db["search_backends"])
+
+
+@app.put("/api/search_backend/{name:str}")
+async def update_search_backend(name: str, backend: Sist2SearchBackend):
+    if not db["search_backends"][name]:
+        raise HTTPException(status_code=404)
+
+    db["search_backends"][name] = backend
+    return "ok"
+
+
+@app.get("/api/search_backend/{name:str}")
+def get_search_backend(name: str):
+    backend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    return backend
+
+
+@app.delete("/api/search_backend/{name:str}")
+def delete_search_backend(name: str):
+    backend: Sist2SearchBackend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    if any(frontend.web_options.search_backend == name for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    if any(job.index_options.search_backend == name for job in db["jobs"]):
+        raise HTTPException(status_code=400, detail="in use (job)")
+
+    del db["search_backends"][name]
+
+    try:
+        os.remove(backend.search_index)
+    except:
+        pass
+
+    return "ok"
+
+
+@app.post("/api/search_backend/{name:str}")
+def create_search_backend(name: str):
+    if db["search_backends"][name] is not None:
+        return HTTPException(status_code=400, detail="already exists")
+
+    backend = Sist2SearchBackend.create_default(name)
+    db["search_backends"][name] = backend
+
+    return backend
+
+
 def tail(filepath: str, n: int):
     with open(filepath) as file:
 
@@ -374,6 +449,8 @@ def initialize_db():
     frontend = Sist2Frontend.create_default("default")
     db["frontends"]["default"] = frontend
 
+    create_default_search_backends(db)
+
     logger.info("Initialized database.")
 
 
@@ -398,6 +475,9 @@ if __name__ == '__main__':
         logger.info("Migrating to v4 database schema")
         migrate_v3_to_v4(db)
 
+    if db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
+        raise Exception(f"Incompatible database version for {db.dbfile}")
+
     start_frontends()
     cron.initialize(db, _run_job)
 

sist2-admin/sist2_admin/jobs.py

@@ -59,11 +59,6 @@ class Sist2Job(BaseModel):
             cron_expression="0 0 * * *"
         )
 
-    # @validator("etag", always=True)
-    # def validate_etag(cls, value, values):
-    #     s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
-    #     return md5(s.encode()).hexdigest()
-
 
 class Sist2TaskProgress:
 
@@ -173,7 +168,14 @@ class Sist2IndexTask(Sist2Task):
 
         self.job.index_options.path = self.job.scan_options.output
 
-        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
+        search_backend = db["search_backends"][self.job.index_options.search_backend]
+        if search_backend is None:
+            logger.error(f"Error while running task: search backend not found: {self.job.index_options.search_backend}")
+            return -1
+
+        logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
+
+        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
         self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
@@ -208,9 +210,17 @@ class Sist2IndexTask(Sist2Task):
             except ChildProcessError:
                 pass
 
+            backend_name = frontend.web_options.search_backend
+            search_backend = db["search_backends"][backend_name]
+            if search_backend is None:
+                logger.error(f"Error while running task: search backend not found: {backend_name}")
+                return -1
+
+            logger.debug(f"Fetched search backend options for {backend_name}")
+
             frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
 
-            pid = sist2.web(frontend.web_options, frontend.name)
+            pid = sist2.web(frontend.web_options, search_backend, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid
 
             self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))

sist2-admin/sist2_admin/sist2.py

@@ -3,6 +3,7 @@ import json
 import logging
 import os.path
 from datetime import datetime
+from enum import Enum
 from io import TextIOWrapper
 from logging import FileHandler
 from subprocess import Popen, PIPE
@@ -12,7 +13,7 @@ from typing import List
 
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER
+from config import logger, LOG_FOLDER, DATA_FOLDER
 
 
 class Sist2Version:
@@ -25,74 +26,54 @@ class Sist2Version:
         return f"{self.major}.{self.minor}.{self.patch}"
 
 
-class WebOptions(BaseModel):
+class SearchBackendType(Enum):
-    indices: List[str] = []
+    SQLITE = "sqlite"
+    ELASTICSEARCH = "elasticsearch"
+
+
+class Sist2SearchBackend(BaseModel):
+    backend_type: SearchBackendType = SearchBackendType("elasticsearch")
+    name: str
+
+    search_index: str = ""
+
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
-    bind: str = "0.0.0.0:4090"
-    auth: str = None
-    tag_auth: str = None
-    tagline: str = "Lightning-fast file system indexer and search tool"
-    dev: bool = False
-    lang: str = "en"
-    auth0_audience: str = None
-    auth0_domain: str = None
-    auth0_client_id: str = None
-    auth0_public_key: str = None
-    auth0_public_key_file: str = None
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def args(self):
-        args = ["web", f"--es-url={self.es_url}", f"--es-index={self.es_index}", f"--bind={self.bind}",
-                f"--tagline={self.tagline}", f"--lang={self.lang}"]
-
-        if self.auth0_audience:
-            args.append(f"--auth0-audience={self.auth0_audience}")
-        if self.auth0_domain:
-            args.append(f"--auth0-domain={self.auth0_domain}")
-        if self.auth0_client_id:
-            args.append(f"--auth0-client-id={self.auth0_client_id}")
-        if self.auth0_public_key_file:
-            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
-        if self.es_insecure_ssl:
-            args.append(f"--es-insecure-ssl")
-        if self.auth:
-            args.append(f"--auth={self.auth}")
-        if self.tag_auth:
-            args.append(f"--tag-auth={self.tag_auth}")
-        if self.dev:
-            args.append(f"--dev")
-
-        args.extend(self.indices)
-
-        return args
-
-
-class IndexOptions(BaseModel):
-    path: str = None
     threads: int = 1
-    es_url: str = "http://elasticsearch:9200"
-    es_insecure_ssl: bool = False
-    es_index: str = "sist2"
-    incremental_index: bool = True
     script: str = ""
     script_file: str = None
     batch_size: int = 70
 
+    @staticmethod
+    def create_default(name: str, backend_type: SearchBackendType = SearchBackendType("elasticsearch")):
+        return Sist2SearchBackend(
+            name=name,
+            search_index=os.path.join(DATA_FOLDER, f"search-index-{name.replace('/', '_')}.sist2"),
+            backend_type=backend_type
+        )
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    incremental_index: bool = True
+    search_backend: str = None
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def args(self):
+    def args(self, search_backend):
+        if search_backend.backend_type == SearchBackendType("sqlite"):
+            args = ["sqlite-index", self.path, "--search-index", search_backend.search_index]
+        else:
+            args = ["index", self.path, f"--threads={search_backend.threads}",
+                    f"--es-url={search_backend.es_url}",
+                    f"--es-index={search_backend.es_index}",
+                    f"--batch-size={search_backend.batch_size}"]
 
-        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
+            if search_backend.script_file:
-                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+                args.append(f"--script-file={search_backend.script_file}")
-
+            if search_backend.es_insecure_ssl:
-        if self.script_file:
-            args.append(f"--script-file={self.script_file}")
-        if self.es_insecure_ssl:
                 args.append(f"--es-insecure-ssl")
             if self.incremental_index:
                 args.append(f"--incremental-index")
@@ -200,6 +181,56 @@ class Sist2Index:
     def name(self) -> str:
         return self._descriptor["name"]
 
+class WebOptions(BaseModel):
+    indices: List[str] = []
+
+    search_backend: str = "elasticsearch"
+
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+    auth0_audience: str = None
+    auth0_domain: str = None
+    auth0_client_id: str = None
+    auth0_public_key: str = None
+    auth0_public_key_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self, search_backend: Sist2SearchBackend):
+        args = ["web", f"--bind={self.bind}", f"--tagline={self.tagline}",
+                f"--lang={self.lang}"]
+
+        if search_backend.backend_type == SearchBackendType("sqlite"):
+            args.append(f"--search-index={search_backend.search_index}")
+        else:
+            args.append(f"--es-url={search_backend.es_url}")
+            args.append(f"--es-index={search_backend.es_index}")
+            if search_backend.es_insecure_ssl:
+                args.append(f"--es-insecure-ssl")
+
+        if self.auth0_audience:
+            args.append(f"--auth0-audience={self.auth0_audience}")
+        if self.auth0_domain:
+            args.append(f"--auth0-domain={self.auth0_domain}")
+        if self.auth0_client_id:
+            args.append(f"--auth0-client-id={self.auth0_client_id}")
+        if self.auth0_public_key_file:
+            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag-auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
 
 class Sist2:
 
@@ -207,21 +238,23 @@ class Sist2:
         self._bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, logs_cb):
+    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
 
-        if options.script:
+        if search_backend.script and search_backend.backend_type == SearchBackendType("elasticsearch"):
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
-                f.write(options.script)
+                f.write(search_backend.script)
-            options.script_file = f.name
+            search_backend.script_file = f.name
         else:
-            options.script_file = None
+            search_backend.script_file = None
 
         args = [
             self._bin_path,
-            *options.args(),
+            *options.args(search_backend),
             "--json-logs",
             "--very-verbose"
         ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
         t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
@@ -290,7 +323,7 @@ class Sist2:
                 except NameError:
                     pass
 
-    def web(self, options: WebOptions, name: str):
+    def web(self, options: WebOptions, search_backend: Sist2SearchBackend, name: str):
 
         if options.auth0_public_key:
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
@@ -301,7 +334,7 @@ class Sist2:
 
         args = [
             self._bin_path,
-            *options.args()
+            *options.args(search_backend)
         ]
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")
@@ -321,3 +354,5 @@ class Sist2:
         t_stdout.start()
 
         return proc.pid
+
+

sist2-admin/sist2_admin/state.py

@@ -7,7 +7,8 @@ import pickle
 
 from tesseract import get_tesseract_langs
 import sqlite3
-from config import LOG_FOLDER
+from config import LOG_FOLDER, logger
+from sist2 import SearchBackendType, Sist2SearchBackend
 
 RUNNING_FRONTENDS: Dict[str, int] = {}
 
@@ -109,13 +110,26 @@ def migrate_v1_to_v2(db: PersistentState):
     }
 
 
+def create_default_search_backends(db: PersistentState):
+    es_backend = Sist2SearchBackend.create_default(name="elasticsearch",
+                                                   backend_type=SearchBackendType("elasticsearch"))
+    db["search_backends"]["elasticsearch"] = es_backend
+    sqlite_backend = Sist2SearchBackend.create_default(name="sqlite", backend_type=SearchBackendType("sqlite"))
+    db["search_backends"]["sqlite"] = sqlite_backend
+
+
 def migrate_v3_to_v4(db: PersistentState):
     shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak")
 
+    create_default_search_backends(db)
+
+    try:
         conn = sqlite3.connect(db.dbfile)
         conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
         conn.commit()
         conn.close()
+    except Exception as e:
+        logger.exception(e)
 
     db["sist2_admin"]["info"] = {
         "version": "4"