simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-10 05:56:46 +00:00
Code Issues Projects Releases Wiki Activity

SQLite backend support for sist2-admin #366

Browse Source
This commit is contained in:
simon987 2023-06-03 18:33:44 -04:00
parent 5522bcfa9b
commit c03c148273
15 changed files with 844 additions and 499 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
sist2-admin/frontend/src/Sist2AdminApi.js
View File

@ -33,9 +33,26 @@ class Sist2AdminApi {
return axios.get(`${this.baseUrl}/api/job/${name}`);
}
/**
* @param {string} name
*/
getSearchBackend(name) {
return axios.get(`${this.baseUrl}/api/search_backend/${name}`);
}
updateSearchBackend(name, data) {
return axios.put(`${this.baseUrl}/api/search_backend/${name}`, data);
}
getSearchBackends() {
return axios.get(`${this.baseUrl}/api/search_backend/`);
}
deleteBackend(name) {
return axios.delete(`${this.baseUrl}/api/search_backend/${name}`)
}
createBackend(name) {
return axios.post(`${this.baseUrl}/api/search_backend/${name}`);
}
getFrontend(name) {
return axios.get(`${this.baseUrl}/api/frontend/${name}`);
}

64
sist2-admin/frontend/src/components/IndexOptions.vue
View File

@ -1,64 +0,0 @@
<template>
<div>
<label>{{ $t("indexOptions.threads") }}</label>
<b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
<label>{{ $t("webOptions.esUrl") }}</label>
<b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
{{ esTestMessage }}
</b-alert>
<b-input-group>
<b-form-input v-model="options.es_url" @change="update()"></b-form-input>
<b-input-group-append>
<b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
</b-input-group-append>
</b-input-group>
<label>{{ $t("indexOptions.esIndex") }}</label>
<b-form-input v-model="options.es_index" @change="update()"></b-form-input>
<br>
<b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
{{ $t("webOptions.esInsecure") }}
</b-form-checkbox>
<label>{{ $t("indexOptions.batchSize") }}</label>
<b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
<label>{{ $t("indexOptions.script") }}</label>
<b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
</div>
</template>
<script>
import sist2AdminApi from "@/Sist2AdminApi";
export default {
name: "IndexOptions",
props: ["options"],
data() {
return {
showEsTestAlert: false,
esTestOk: false,
esTestMessage: "",
}
},
methods: {
update() {
this.$emit("change", this.options);
},
testEs() {
sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
this.showEsTestAlert = true;
this.esTestOk = resp.data.ok;
this.esTestMessage = resp.data.message;
});
}
},
}
</script>
<style scoped>
</style>

24
sist2-admin/frontend/src/components/SearchBackendListItem.vue Normal file
View File

@ -0,0 +1,24 @@
<template>
<b-list-group-item action :to="`/searchBackend/${backend.name}`">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">
{{ backend.name }}
</h5>
<div>
<b-badge v-if="backend.backend_type === 'sqlite'" variant="info">SQLite</b-badge>
<b-badge v-else variant="info">Elasticsearch</b-badge>
</div>
</div>
</b-list-group-item>
</template>
<script>
export default {
name: "SearchBackendListItem",
props: ["backend"],
}
</script>

37
sist2-admin/frontend/src/components/SearchBackendSelect.vue Normal file
View File

@ -0,0 +1,37 @@
<template>
<b-progress v-if="loading" striped animated value="100"></b-progress>
<div v-else>
<label>{{$t("backendOptions.searchBackend")}}</label>
<b-select :options="options" :value="value" @change="$emit('change', $event)"></b-select>
</div>
</template>
<script>
import Sist2AdminApi from "@/Sist2AdminApi";
export default {
name: "SearchBackendSelect",
props: ["value"],
data() {
return {
loading: true,
backends: null,
}
},
computed: {
options() {
return this.backends.map(backend => backend.name)
}
},
mounted() {
Sist2AdminApi.getSearchBackends().then(resp => {
this.loading = false;
this.backends = resp.data
})
}
}
</script>
<style scoped>
</style>

104
sist2-admin/frontend/src/components/WebOptions.vue
View File

@ -1,56 +1,35 @@
<template>
<div>
<label>{{ $t("webOptions.esUrl") }}</label>
<b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
{{ esTestMessage }}
</b-alert>
<div>
<b-input-group>
<b-form-input v-model="options.es_url" @change="update()"></b-form-input>
<b-input-group-append>
<b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
</b-input-group-append>
</b-input-group>
<label>{{ $t("webOptions.lang") }}</label>
<b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
<b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
{{ $t("webOptions.esInsecure") }}
</b-form-checkbox>
<label>{{ $t("webOptions.bind") }}</label>
<b-form-input v-model="options.bind" @change="update()"></b-form-input>
<label>{{ $t("webOptions.esIndex") }}</label>
<b-form-input v-model="options.es_index" @change="update()"></b-form-input>
<label>{{ $t("webOptions.tagline") }}</label>
<b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
<label>{{ $t("webOptions.lang") }}</label>
<b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
<label>{{ $t("webOptions.auth") }}</label>
<b-form-input v-model="options.auth" @change="update()"></b-form-input>
<label>{{ $t("webOptions.bind") }}</label>
<b-form-input v-model="options.bind" @change="update()"></b-form-input>
<label>{{ $t("webOptions.tagAuth") }}</label>
<b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
<label>{{ $t("webOptions.tagline") }}</label>
<b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
<br>
<h5>Auth0 options</h5>
<label>{{ $t("webOptions.auth0Audience") }}</label>
<b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
<label>{{ $t("webOptions.auth") }}</label>
<b-form-input v-model="options.auth" @change="update()"></b-form-input>
<label>{{ $t("webOptions.auth0Domain") }}</label>
<b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
<label>{{ $t("webOptions.tagAuth") }}</label>
<b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
<label>{{ $t("webOptions.auth0ClientId") }}</label>
<b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
<br>
<h5>Auth0 options</h5>
<label>{{ $t("webOptions.auth0Audience") }}</label>
<b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
<label>{{ $t("webOptions.auth0Domain") }}</label>
<b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
<label>{{ $t("webOptions.auth0ClientId") }}</label>
<b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
<label>{{ $t("webOptions.auth0PublicKey") }}</label>
<b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
</div>
<label>{{ $t("webOptions.auth0PublicKey") }}</label>
<b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
</div>
</template>
<script>
@ -58,31 +37,24 @@
import sist2AdminApi from "@/Sist2AdminApi";
export default {
name: "WebOptions",
props: ["options", "frontendName"],
data() {
return {
showEsTestAlert: false,
esTestOk: false,
esTestMessage: "",
}
},
methods: {
update() {
if (!this.options.es_url.startsWith("https")) {
this.options.es_insecure_ssl = false;
}
this.$emit("change", this.options);
name: "WebOptions",
props: ["options", "frontendName"],
data() {
return {
showEsTestAlert: false,
esTestOk: false,
esTestMessage: "",
}
},
testEs() {
sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
this.showEsTestAlert = true;
this.esTestOk = resp.data.ok;
this.esTestMessage = resp.data.message;
});
methods: {
update() {
if (!this.options.es_url.startsWith("https")) {
this.options.es_insecure_ssl = false;
}
this.$emit("change", this.options);
},
}
}
}
</script>

29
sist2-admin/frontend/src/i18n/messages.js
View File

@ -48,12 +48,13 @@ export default {
extraQueryArgs: "Extra query arguments when launching from sist2-admin",
customUrl: "Custom URL when launching from sist2-admin",
searchBackends: "Search backends",
searchBackendTitle: "search backend configuration",
newBackendName: "New search backend name",
selectJobs: "Select jobs",
webOptions: {
title: "Web options",
esUrl: "Elasticsearch URL",
esIndex: "Elasticsearch index name",
esInsecure: "Do not verify SSL connections to Elasticsearch.",
lang: "UI Language",
bind: "Listen address",
tagline: "Tagline in navbar",
@ -64,6 +65,18 @@ export default {
auth0ClientId: "Auth0 client ID",
auth0PublicKey: "Auth0 public key",
},
backendOptions: {
title: "Search backend options",
searchBackend: "Search backend",
type: "Search backend type",
esUrl: "Elasticsearch URL",
esIndex: "Elasticsearch index name",
esInsecure: "Do not verify SSL connections to Elasticsearch.",
threads: "Number of threads",
batchSize: "Index batch size",
script: "User script",
searchIndex: "Search index file location"
},
scanOptions: {
title: "Scanning options",
path: "Path",
@ -90,15 +103,6 @@ export default {
treemapThreshold: "Relative size threshold for treemap",
optimizeIndex: "Defragment index file after scan to reduce its file size."
},
indexOptions: {
title: "Indexing options",
threads: "Number of threads",
esUrl: "Elasticsearch URL",
esIndex: "Elasticsearch index name",
esInsecure: "Do not verify SSL connections to Elasticsearch.",
batchSize: "Index batch size",
script: "User script"
},
jobOptions: {
title: "Job options",
cron: "Job schedule",
@ -106,6 +110,7 @@ export default {
deleteNow: "Delete now",
scheduleEnabled: "Enable scheduled re-scan",
noJobAvailable: "No jobs available.",
noBackendError: "You must select a search backend to run this job",
desktopNotifications: "Desktop notifications"
},
frontendOptions: {

6
sist2-admin/frontend/src/router/index.js
View File

@ -5,6 +5,7 @@ import Job from "@/views/Job";
import Tasks from "@/views/Tasks";
import Frontend from "@/views/Frontend";
import Tail from "@/views/Tail";
import SearchBackend from "@/views/SearchBackend.vue";
Vue.use(VueRouter);
@ -29,6 +30,11 @@ const routes = [
name: "Frontend",
component: Frontend
},
{
path: "/searchBackend/:name",
name: "SearchBackend",
component: SearchBackend
},
{
path: "/log/:taskId",
name: "Tail",

213
sist2-admin/frontend/src/views/Frontend.vue
View File

@ -1,60 +1,70 @@
<template>
<b-card>
<b-card-title>
{{ name }}
<small style="vertical-align: top">
<b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
<b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
</small>
</b-card-title>
<b-card>
<b-card-title>
{{ name }}
<small style="vertical-align: top">
<b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
<b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
</small>
</b-card-title>
<div class="mb-3" v-if="!loading">
<b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
$t("start")
}}
</b-button>
<b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
$t("stop")
}}
</b-button>
<b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
{{ $t("go") }}
</b-button>
<b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
</div>
<div class="mb-3" v-if="!loading">
<b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
$t("start")
}}
</b-button>
<b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
$t("stop")
}}
</b-button>
<b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
{{ $t("go") }}
</b-button>
<b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
</div>
<b-progress v-if="loading" striped animated value="100"></b-progress>
<b-card-body v-else>
<b-progress v-if="loading" striped animated value="100"></b-progress>
<b-card-body v-else>
<h4>{{ $t("frontendOptions.title") }}</h4>
<b-card>
<b-form-checkbox v-model="frontend.auto_start" @change="update()">
{{ $t("autoStart") }}
</b-form-checkbox>
<h4>{{ $t("frontendOptions.title") }}</h4>
<b-card>
<b-form-checkbox v-model="frontend.auto_start" @change="update()">
{{ $t("autoStart") }}
</b-form-checkbox>
<label>{{ $t("extraQueryArgs") }}</label>
<b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
<label>{{ $t("extraQueryArgs") }}</label>
<b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
<label>{{ $t("customUrl") }}</label>
<b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
<label>{{ $t("customUrl") }}</label>
<b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
<br/>
<br/>
<b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
<b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
<JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
</b-card>
<JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
</b-card>
<br/>
<br/>
<h4>{{ $t("webOptions.title") }}</h4>
<b-card>
<WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
</b-card>
</b-card-body>
<h4>{{ $t("webOptions.title") }}</h4>
<b-card>
<WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
@change="update()"></WebOptions>
</b-card>
</b-card>
<br>
<h4>{{ $t("backendOptions.title") }}</h4>
<b-card>
<SearchBackendSelect :value="frontend.web_options.search_backend"
@change="onBackendSelect($event)"></SearchBackendSelect>
</b-card>
</b-card-body>
</b-card>
</template>
<script>
@ -62,68 +72,73 @@
import Sist2AdminApi from "@/Sist2AdminApi";
import JobCheckboxGroup from "@/components/JobCheckboxGroup";
import WebOptions from "@/components/WebOptions";
import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
export default {
name: 'Frontend',
components: {JobCheckboxGroup, WebOptions},
data() {
return {
loading: true,
frontend: null,
}
},
computed: {
valid() {
return !this.loading && this.frontend.jobs.length > 0;
name: 'Frontend',
components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
data() {
return {
loading: true,
frontend: null,
}
},
frontendUrl() {
if (this.frontend.custom_url) {
return this.frontend.custom_url + this.args;
}
computed: {
valid() {
return !this.loading && this.frontend.jobs.length > 0;
},
frontendUrl() {
if (this.frontend.custom_url) {
return this.frontend.custom_url + this.args;
}
if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
}
if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
}
return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
},
name() {
return this.$route.params.name;
},
port() {
return this.frontend.web_options.bind.split(":")[1]
},
args() {
const args = this.frontend.extra_query_args;
if (args !== "") {
return "#" + (args.startsWith("?") ? (args) : ("?" + args));
}
return "";
}
},
name() {
return this.$route.params.name;
mounted() {
Sist2AdminApi.getFrontend(this.name).then(resp => {
this.frontend = resp.data;
this.loading = false;
});
},
port() {
return this.frontend.web_options.bind.split(":")[1]
},
args() {
const args = this.frontend.extra_query_args;
if (args !== "") {
return "#" + (args.startsWith("?") ? (args) : ("?" + args));
}
return "";
methods: {
start() {
this.frontend.running = true;
Sist2AdminApi.startFrontend(this.name)
},
stop() {
this.frontend.running = false;
Sist2AdminApi.stopFrontend(this.name)
},
deleteFrontend() {
Sist2AdminApi.deleteFrontend(this.name).then(() => {
this.$router.push("/");
});
},
update() {
Sist2AdminApi.updateFrontend(this.name, this.frontend);
},
onBackendSelect(backend) {
this.frontend.web_options.search_backend = backend;
this.update();
}
}
},
mounted() {
Sist2AdminApi.getFrontend(this.name).then(resp => {
this.frontend = resp.data;
this.loading = false;
});
},
methods: {
start() {
this.frontend.running = true;
Sist2AdminApi.startFrontend(this.name)
},
stop() {
this.frontend.running = false;
Sist2AdminApi.stopFrontend(this.name)
},
deleteFrontend() {
Sist2AdminApi.deleteFrontend(this.name).then(() => {
this.$router.push("/frontends");
});
},
update() {
Sist2AdminApi.updateFrontend(this.name, this.frontend);
},
}
}
</script>

244
sist2-admin/frontend/src/views/Home.vue
View File

@ -1,60 +1,89 @@
<template>
<div>
<b-card>
<b-card-title>{{ $t("jobs") }}</b-card-title>
<b-row>
<b-col>
<b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
<b-popover
:show.sync="showHelp"
target="new-job"
placement="top"
triggers="manual"
variant="primary"
:content="$t('newJobHelp')"
></b-popover>
</b-col>
<b-col>
<b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
</b-button>
</b-col>
</b-row>
<div>
<b-card>
<b-card-title>{{ $t("jobs") }}</b-card-title>
<b-row>
<b-col>
<b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
<b-popover
:show.sync="showHelp"
target="new-job"
placement="top"
triggers="manual"
variant="primary"
:content="$t('newJobHelp')"
></b-popover>
</b-col>
<b-col>
<b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">
{{ $t("create") }}
</b-button>
</b-col>
</b-row>
<hr/>
<hr/>
<b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
<b-list-group v-else>
<JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
</b-list-group>
</b-card>
<b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
<b-list-group v-else>
<JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
</b-list-group>
</b-card>
<br/>
<br/>
<b-card>
<b-card>
<b-card-title>{{ $t("frontends") }}</b-card-title>
<b-card-title>{{ $t("frontends") }}</b-card-title>
<b-row>
<b-col>
<b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
</b-col>
<b-col>
<b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
{{ $t("create") }}
</b-button>
</b-col>
</b-row>
<b-row>
<b-col>
<b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
</b-col>
<b-col>
<b-button variant="primary" @click="createFrontend()"
:disabled="!frontendNameValid(newFrontendName)">
{{ $t("create") }}
</b-button>
</b-col>
</b-row>
<hr/>
<hr/>
<b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
<b-list-group v-else>
<FrontendListItem v-for="frontend in frontends"
:key="frontend.name" :frontend="frontend"></FrontendListItem>
</b-list-group>
<b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
<b-list-group v-else>
<FrontendListItem v-for="frontend in frontends"
:key="frontend.name" :frontend="frontend"></FrontendListItem>
</b-list-group>
</b-card>
</div>
</b-card>
<br/>
<b-card>
<b-card-title>{{ $t("searchBackends") }}</b-card-title>
<b-row>
<b-col>
<b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
</b-col>
<b-col>
<b-button variant="primary" @click="createBackend()"
:disabled="!backendNameValid(newBackendName)">
{{ $t("create") }}
</b-button>
</b-col>
</b-row>
<hr/>
<b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
<b-list-group v-else>
<SearchBackendListItem v-for="backend in backends"
:key="backend.name" :backend="backend"></SearchBackendListItem>
</b-list-group>
</b-card>
</div>
</template>
<script>
@ -62,61 +91,80 @@ import JobListItem from "@/components/JobListItem";
import {formatBindAddress} from "@/util";
import Sist2AdminApi from "@/Sist2AdminApi";
import FrontendListItem from "@/components/FrontendListItem";
import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
export default {
name: "Jobs",
components: {JobListItem, FrontendListItem},
data() {
return {
jobsLoading: true,
newJobName: "",
jobs: [],
name: "Jobs",
components: {SearchBackendListItem, JobListItem, FrontendListItem},
data() {
return {
jobsLoading: true,
newJobName: "",
jobs: [],
frontendsLoading: true,
frontends: [],
formatBindAddress,
newFrontendName: "",
frontendsLoading: true,
frontends: [],
formatBindAddress,
newFrontendName: "",
showHelp: false
backends: [],
backendsLoading: true,
newBackendName: "",
showHelp: false
}
},
mounted() {
this.loading = true;
this.reload();
},
methods: {
jobNameValid(name) {
if (this.jobs.some(job => job.name === name)) {
return false;
}
return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
},
frontendNameValid(name) {
if (this.frontends.some(frontend => frontend.name === name)) {
return false;
}
return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
},
backendNameValid(name) {
if (this.backends.some(backend => backend.name === name)) {
return false;
}
return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
},
reload() {
Sist2AdminApi.getJobs().then(resp => {
this.jobs = resp.data;
this.jobsLoading = false;
this.showHelp = this.jobs.length === 0;
});
Sist2AdminApi.getFrontends().then(resp => {
this.frontends = resp.data;
this.frontendsLoading = false;
});
Sist2AdminApi.getSearchBackends().then(resp => {
this.backends = resp.data;
this.backendsLoading = false;
})
},
createJob() {
Sist2AdminApi.createJob(this.newJobName).then(this.reload);
},
createFrontend() {
Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
},
createBackend() {
Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
}
}
},
mounted() {
this.loading = true;
this.reload();
},
methods: {
jobNameValid(name) {
if (this.jobs.some(job => job.name === name)) {
return false;
}
return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
},
frontendNameValid(name) {
if (this.frontends.some(job => job.name === name)) {
return false;
}
return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
},
reload() {
Sist2AdminApi.getJobs().then(resp => {
this.jobs = resp.data;
this.jobsLoading = false;
this.showHelp = this.jobs.length === 0;
});
Sist2AdminApi.getFrontends().then(resp => {
this.frontends = resp.data;
this.frontendsLoading = false;
});
},
createJob() {
Sist2AdminApi.createJob(this.newJobName).then(this.reload);
},
createFrontend() {
Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
}
}
}
</script>

160
sist2-admin/frontend/src/views/Job.vue
View File

@ -1,92 +1,112 @@
<template>
<b-card>
<b-card-title>
[{{ getName() }}]
{{ $t("jobTitle") }}
</b-card-title>
<b-card>
<b-card-title>
[{{ getName() }}]
{{ $t("jobTitle") }}
</b-card-title>
<div class="mb-3">
<b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
<b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
</div>
<div class="mb-3">
<b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
<b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
</div>
<div v-if="job">
{{ $t("status") }}: <code>{{ job.status }}</code>
</div>
<div v-if="job">
{{ $t("status") }}: <code>{{ job.status }}</code>
</div>
<b-progress v-if="loading" striped animated value="100"></b-progress>
<b-card-body v-else>
<b-progress v-if="loading" striped animated value="100"></b-progress>
<b-card-body v-else>
<h4>{{ $t("jobOptions.title") }}</h4>
<b-card>
<JobOptions :job="job" @change="update"></JobOptions>
</b-card>
<h4>{{ $t("jobOptions.title") }}</h4>
<b-card>
<JobOptions :job="job" @change="update"></JobOptions>
</b-card>
<br/>
<br/>
<h4>{{ $t("scanOptions.title") }}</h4>
<b-card>
<ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
</b-card>
<h4>{{ $t("scanOptions.title") }}</h4>
<b-card>
<ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
</b-card>
<br/>
<br/>
<h4>{{ $t("indexOptions.title") }}</h4>
<b-card>
<IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
</b-card>
<h4>{{ $t("backendOptions.title") }}</h4>
<b-card>
<b-alert v-if="!valid" variant="warning" show>{{ $t("jobOptions.noBackendError") }}</b-alert>
<SearchBackendSelect :value="job.index_options.search_backend"
@change="onBackendSelect($event)"></SearchBackendSelect>
</b-card>
</b-card-body>
</b-card-body>
</b-card>
</b-card>
</template>
<script>
import ScanOptions from "@/components/ScanOptions";
import Sist2AdminApi from "@/Sist2AdminApi";
import IndexOptions from "@/components/IndexOptions";
import JobOptions from "@/components/JobOptions";
import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
export default {
name: "Job",
components: {
IndexOptions,
ScanOptions,
JobOptions
},
data() {
return {
loading: true,
job: null
name: "Job",
components: {
SearchBackendSelect,
ScanOptions,
JobOptions
},
data() {
return {
loading: true,
job: null,
}
},
methods: {
getName() {
return this.$route.params.name;
},
update() {
Sist2AdminApi.updateJob(this.getName(), this.job);
},
runJob() {
Sist2AdminApi.runJob(this.getName()).then(() => {
this.$bvToast.toast(this.$t("runJobConfirmation"), {
title: this.$t("runJobConfirmationTitle"),
variant: "success",
toaster: "b-toaster-bottom-right"
});
});
},
deleteJob() {
Sist2AdminApi.deleteJob(this.getName())
.then(() => {
this.$router.push("/");
})
.catch(err => {
this.$bvToast.toast("Cannot delete job " +
"because it is referenced by a frontend", {
title: "Error",
variant: "danger",
toaster: "b-toaster-bottom-right"
});
})
},
onBackendSelect(backend) {
this.job.index_options.search_backend = backend;
this.update();
}
},
mounted() {
Sist2AdminApi.getJob(this.getName()).then(resp => {
this.loading = false;
this.job = resp.data;
})
},
computed: {
valid() {
return this.job?.index_options.search_backend != null;
}
}
},
methods: {
getName() {
return this.$route.params.name;
},
update() {
Sist2AdminApi.updateJob(this.getName(), this.job);
},
runJob() {
Sist2AdminApi.runJob(this.getName()).then(() => {
this.$bvToast.toast(this.$t("runJobConfirmation"), {
title: this.$t("runJobConfirmationTitle"),
variant: "success",
toaster: "b-toaster-bottom-right"
});
});
},
deleteJob() {
Sist2AdminApi.deleteJob(this.getName()).then(() => {
this.$router.push("/");
})
}
},
mounted() {
Sist2AdminApi.getJob(this.getName()).then(resp => {
this.loading = false;
this.job = resp.data;
})
}
}
</script>

126
sist2-admin/frontend/src/views/SearchBackend.vue Normal file
View File

@ -0,0 +1,126 @@
<template>
<b-card>
<b-card-title>
<span class="text-monospace">{{ getName() }}</span>
{{ $t("searchBackendTitle") }}
</b-card-title>
<div class="mb-3">
<b-button variant="danger" @click="deleteBackend()">{{ $t("delete") }}</b-button>
</div>
<b-progress v-if="loading" striped animated value="100"></b-progress>
<b-card-body v-else>
<label>{{ $t("backendOptions.type") }}</label>
<b-select :options="backendTypeOptions" v-model="backend.backend_type" @change="update()"></b-select>
<hr/>
<template v-if="backend.backend_type === 'elasticsearch'">
<b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
{{ esTestMessage }}
</b-alert>
<label>{{ $t("backendOptions.esUrl") }}</label>
<b-input-group>
<b-form-input v-model="backend.es_url" @change="update()"></b-form-input>
<b-input-group-append>
<b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
</b-input-group-append>
</b-input-group>
<b-form-checkbox v-model="backend.es_insecure_ssl" :disabled="!this.backend.es_url.startsWith('https')"
@change="update()">
{{ $t("backendOptions.esInsecure") }}
</b-form-checkbox>
<label>{{ $t("backendOptions.esIndex") }}</label>
<b-form-input v-model="backend.es_index" @change="update()"></b-form-input>
<label>{{ $t("backendOptions.threads") }}</label>
<b-form-input v-model="backend.threads" type="number" min="1" @change="update()"></b-form-input>
<label>{{ $t("backendOptions.batchSize") }}</label>
<b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
<label>{{ $t("backendOptions.script") }}</label>
<b-form-textarea v-model="backend.script" rows="6" @change="update()"></b-form-textarea>
</template>
<template v-else>
<label>{{ $t("backendOptions.searchIndex") }}</label>
<b-form-input v-model="backend.search_index" disabled></b-form-input>
</template>
</b-card-body>
</b-card>
</template>
<script>
import sist2AdminApi from "@/Sist2AdminApi";
import Sist2AdminApi from "@/Sist2AdminApi";
export default {
name: "SearchBackend",
data() {
return {
showEsTestAlert: false,
esTestOk: false,
esTestMessage: "",
loading: true,
backend: null,
backendTypeOptions: [
{
text: "Elasticsearch",
value: "elasticsearch"
},
{
text: "SQLite",
value: "sqlite"
}
]
}
},
mounted() {
Sist2AdminApi.getSearchBackend(this.getName()).then(resp => {
this.backend = resp.data;
this.loading = false;
});
},
methods: {
getName() {
return this.$route.params.name;
},
testEs() {
sist2AdminApi.pingEs(this.backend.es_url, this.backend.es_insecure_ssl)
.then((resp) => {
this.showEsTestAlert = true;
this.esTestOk = resp.data.ok;
this.esTestMessage = resp.data.message;
});
},
update() {
Sist2AdminApi.updateSearchBackend(this.getName(), this.backend);
},
deleteBackend() {
Sist2AdminApi.deleteBackend(this.getName())
.then(() => {
this.$router.push("/");
})
.catch(err => {
this.$bvToast.toast("Cannot delete search backend " +
"because it is referenced by a job or frontend", {
title: "Error",
variant: "danger",
toaster: "b-toaster-bottom-right"
});
})
}
}
}
</script>
<style scoped>
</style>

94
sist2-admin/sist2_admin/app.py
View File

@ -20,9 +20,9 @@ import cron
from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
from notifications import Subscribe, Notifications
from sist2 import Sist2
from sist2 import Sist2, Sist2SearchBackend
from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
get_log_files_to_remove, delete_log_file
get_log_files_to_remove, delete_log_file, create_default_search_backends
from web import Sist2Frontend
sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
@ -174,12 +174,22 @@ async def task_history(n: int, name: str):
@app.delete("/api/job/{name:str}")
async def delete_job(name: str):
job = db["jobs"][name]
if job:
del db["jobs"][name]
else:
job: Sist2Job = db["jobs"][name]
if not job:
raise HTTPException(status_code=404)
if any(name in frontend.jobs for frontend in db["frontends"]):
raise HTTPException(status_code=400, detail="in use (frontend)")
try:
os.remove(job.previous_index)
except:
pass
del db["jobs"][name]
return "ok"
@app.delete("/api/frontend/{name:str}")
async def delete_frontend(name: str):
@ -267,7 +277,16 @@ def check_es_version(es_url: str, insecure: bool):
def start_frontend_(frontend: Sist2Frontend):
frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
pid = sist2.web(frontend.web_options, frontend.name)
backend_name = frontend.web_options.search_backend
search_backend = db["search_backends"][backend_name]
if search_backend is None:
logger.error(
f"Error while running task: search backend not found: {backend_name}")
return -1
logger.debug(f"Fetched search backend options for {backend_name}")
pid = sist2.web(frontend.web_options, search_backend, frontend.name)
RUNNING_FRONTENDS[frontend.name] = pid
@ -297,6 +316,62 @@ async def get_frontends():
return res
@app.get("/api/search_backend/")
async def get_search_backends():
return list(db["search_backends"])
@app.put("/api/search_backend/{name:str}")
async def update_search_backend(name: str, backend: Sist2SearchBackend):
if not db["search_backends"][name]:
raise HTTPException(status_code=404)
db["search_backends"][name] = backend
return "ok"
@app.get("/api/search_backend/{name:str}")
def get_search_backend(name: str):
backend = db["search_backends"][name]
if not backend:
raise HTTPException(status_code=404)
return backend
@app.delete("/api/search_backend/{name:str}")
def delete_search_backend(name: str):
backend: Sist2SearchBackend = db["search_backends"][name]
if not backend:
raise HTTPException(status_code=404)
if any(frontend.web_options.search_backend == name for frontend in db["frontends"]):
raise HTTPException(status_code=400, detail="in use (frontend)")
if any(job.index_options.search_backend == name for job in db["jobs"]):
raise HTTPException(status_code=400, detail="in use (job)")
del db["search_backends"][name]
try:
os.remove(backend.search_index)
except:
pass
return "ok"
@app.post("/api/search_backend/{name:str}")
def create_search_backend(name: str):
if db["search_backends"][name] is not None:
return HTTPException(status_code=400, detail="already exists")
backend = Sist2SearchBackend.create_default(name)
db["search_backends"][name] = backend
return backend
def tail(filepath: str, n: int):
with open(filepath) as file:
@ -374,6 +449,8 @@ def initialize_db():
frontend = Sist2Frontend.create_default("default")
db["frontends"]["default"] = frontend
create_default_search_backends(db)
logger.info("Initialized database.")
@ -398,6 +475,9 @@ if __name__ == '__main__':
logger.info("Migrating to v4 database schema")
migrate_v3_to_v4(db)
if db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
raise Exception(f"Incompatible database version for {db.dbfile}")
start_frontends()
cron.initialize(db, _run_job)

24
sist2-admin/sist2_admin/jobs.py
View File

@ -59,11 +59,6 @@ class Sist2Job(BaseModel):
cron_expression="0 0 * * *"
)
# @validator("etag", always=True)
# def validate_etag(cls, value, values):
# s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
# return md5(s.encode()).hexdigest()
class Sist2TaskProgress:
@ -173,7 +168,14 @@ class Sist2IndexTask(Sist2Task):
self.job.index_options.path = self.job.scan_options.output
return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
search_backend = db["search_backends"][self.job.index_options.search_backend]
if search_backend is None:
logger.error(f"Error while running task: search backend not found: {self.job.index_options.search_backend}")
return -1
logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
self.ended = datetime.utcnow()
duration = self.ended - self.started
@ -208,9 +210,17 @@ class Sist2IndexTask(Sist2Task):
except ChildProcessError:
pass
backend_name = frontend.web_options.search_backend
search_backend = db["search_backends"][backend_name]
if search_backend is None:
logger.error(f"Error while running task: search backend not found: {backend_name}")
return -1
logger.debug(f"Fetched search backend options for {backend_name}")
frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
pid = sist2.web(frontend.web_options, frontend.name)
pid = sist2.web(frontend.web_options, search_backend, frontend.name)
RUNNING_FRONTENDS[frontend_name] = pid
self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))

171
sist2-admin/sist2_admin/sist2.py
View File

@ -3,6 +3,7 @@ import json
import logging
import os.path
from datetime import datetime
from enum import Enum
from io import TextIOWrapper
from logging import FileHandler
from subprocess import Popen, PIPE
@ -12,7 +13,7 @@ from typing import List
from pydantic import BaseModel
from config import logger, LOG_FOLDER
from config import logger, LOG_FOLDER, DATA_FOLDER
class Sist2Version:
@ -25,77 +26,57 @@ class Sist2Version:
return f"{self.major}.{self.minor}.{self.patch}"
class WebOptions(BaseModel):
indices: List[str] = []
class SearchBackendType(Enum):
SQLITE = "sqlite"
ELASTICSEARCH = "elasticsearch"
class Sist2SearchBackend(BaseModel):
backend_type: SearchBackendType = SearchBackendType("elasticsearch")
name: str
search_index: str = ""
es_url: str = "http://elasticsearch:9200"
es_insecure_ssl: bool = False
es_index: str = "sist2"
bind: str = "0.0.0.0:4090"
auth: str = None
tag_auth: str = None
tagline: str = "Lightning-fast file system indexer and search tool"
dev: bool = False
lang: str = "en"
auth0_audience: str = None
auth0_domain: str = None
auth0_client_id: str = None
auth0_public_key: str = None
auth0_public_key_file: str = None
def __init__(self, **kwargs):
super().__init__(**kwargs)
def args(self):
args = ["web", f"--es-url={self.es_url}", f"--es-index={self.es_index}", f"--bind={self.bind}",
f"--tagline={self.tagline}", f"--lang={self.lang}"]
if self.auth0_audience:
args.append(f"--auth0-audience={self.auth0_audience}")
if self.auth0_domain:
args.append(f"--auth0-domain={self.auth0_domain}")
if self.auth0_client_id:
args.append(f"--auth0-client-id={self.auth0_client_id}")
if self.auth0_public_key_file:
args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
if self.es_insecure_ssl:
args.append(f"--es-insecure-ssl")
if self.auth:
args.append(f"--auth={self.auth}")
if self.tag_auth:
args.append(f"--tag-auth={self.tag_auth}")
if self.dev:
args.append(f"--dev")
args.extend(self.indices)
return args
class IndexOptions(BaseModel):
path: str = None
threads: int = 1
es_url: str = "http://elasticsearch:9200"
es_insecure_ssl: bool = False
es_index: str = "sist2"
incremental_index: bool = True
script: str = ""
script_file: str = None
batch_size: int = 70
@staticmethod
def create_default(name: str, backend_type: SearchBackendType = SearchBackendType("elasticsearch")):
return Sist2SearchBackend(
name=name,
search_index=os.path.join(DATA_FOLDER, f"search-index-{name.replace('/', '_')}.sist2"),
backend_type=backend_type
)
class IndexOptions(BaseModel):
path: str = None
incremental_index: bool = True
search_backend: str = None
def __init__(self, **kwargs):
super().__init__(**kwargs)
def args(self):
def args(self, search_backend):
if search_backend.backend_type == SearchBackendType("sqlite"):
args = ["sqlite-index", self.path, "--search-index", search_backend.search_index]
else:
args = ["index", self.path, f"--threads={search_backend.threads}",
f"--es-url={search_backend.es_url}",
f"--es-index={search_backend.es_index}",
f"--batch-size={search_backend.batch_size}"]
args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
if self.script_file:
args.append(f"--script-file={self.script_file}")
if self.es_insecure_ssl:
args.append(f"--es-insecure-ssl")
if self.incremental_index:
args.append(f"--incremental-index")
if search_backend.script_file:
args.append(f"--script-file={search_backend.script_file}")
if search_backend.es_insecure_ssl:
args.append(f"--es-insecure-ssl")
if self.incremental_index:
args.append(f"--incremental-index")
return args
@ -200,6 +181,56 @@ class Sist2Index:
def name(self) -> str:
return self._descriptor["name"]
class WebOptions(BaseModel):
indices: List[str] = []
search_backend: str = "elasticsearch"
bind: str = "0.0.0.0:4090"
auth: str = None
tag_auth: str = None
tagline: str = "Lightning-fast file system indexer and search tool"
dev: bool = False
lang: str = "en"
auth0_audience: str = None
auth0_domain: str = None
auth0_client_id: str = None
auth0_public_key: str = None
auth0_public_key_file: str = None
def __init__(self, **kwargs):
super().__init__(**kwargs)
def args(self, search_backend: Sist2SearchBackend):
args = ["web", f"--bind={self.bind}", f"--tagline={self.tagline}",
f"--lang={self.lang}"]
if search_backend.backend_type == SearchBackendType("sqlite"):
args.append(f"--search-index={search_backend.search_index}")
else:
args.append(f"--es-url={search_backend.es_url}")
args.append(f"--es-index={search_backend.es_index}")
if search_backend.es_insecure_ssl:
args.append(f"--es-insecure-ssl")
if self.auth0_audience:
args.append(f"--auth0-audience={self.auth0_audience}")
if self.auth0_domain:
args.append(f"--auth0-domain={self.auth0_domain}")
if self.auth0_client_id:
args.append(f"--auth0-client-id={self.auth0_client_id}")
if self.auth0_public_key_file:
args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
if self.auth:
args.append(f"--auth={self.auth}")
if self.tag_auth:
args.append(f"--tag-auth={self.tag_auth}")
if self.dev:
args.append(f"--dev")
args.extend(self.indices)
return args
class Sist2:
@ -207,21 +238,23 @@ class Sist2:
self._bin_path = bin_path
self._data_dir = data_directory
def index(self, options: IndexOptions, logs_cb):
def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
if options.script:
if search_backend.script and search_backend.backend_type == SearchBackendType("elasticsearch"):
with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
f.write(options.script)
options.script_file = f.name
f.write(search_backend.script)
search_backend.script_file = f.name
else:
options.script_file = None
search_backend.script_file = None
args = [
self._bin_path,
*options.args(),
*options.args(search_backend),
"--json-logs",
"--very-verbose"
]
logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
proc = Popen(args, stdout=PIPE, stderr=PIPE)
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
@ -290,7 +323,7 @@ class Sist2:
except NameError:
pass
def web(self, options: WebOptions, name: str):
def web(self, options: WebOptions, search_backend: Sist2SearchBackend, name: str):
if options.auth0_public_key:
with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
@ -301,7 +334,7 @@ class Sist2:
args = [
self._bin_path,
*options.args()
*options.args(search_backend)
]
web_logger = logging.Logger(name=f"sist2-frontend-{name}")
@ -321,3 +354,5 @@ class Sist2:
t_stdout.start()
return proc.pid

24
sist2-admin/sist2_admin/state.py
View File

@ -7,7 +7,8 @@ import pickle
from tesseract import get_tesseract_langs
import sqlite3
from config import LOG_FOLDER
from config import LOG_FOLDER, logger
from sist2 import SearchBackendType, Sist2SearchBackend
RUNNING_FRONTENDS: Dict[str, int] = {}
@ -109,13 +110,26 @@ def migrate_v1_to_v2(db: PersistentState):
}
def create_default_search_backends(db: PersistentState):
es_backend = Sist2SearchBackend.create_default(name="elasticsearch",
backend_type=SearchBackendType("elasticsearch"))
db["search_backends"]["elasticsearch"] = es_backend
sqlite_backend = Sist2SearchBackend.create_default(name="sqlite", backend_type=SearchBackendType("sqlite"))
db["search_backends"]["sqlite"] = sqlite_backend
def migrate_v3_to_v4(db: PersistentState):
shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak")
conn = sqlite3.connect(db.dbfile)
conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
conn.commit()
conn.close()
create_default_search_backends(db)
try:
conn = sqlite3.connect(db.dbfile)
conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
conn.commit()
conn.close()
except Exception as e:
logger.exception(e)
db["sist2_admin"]["info"] = {
"version": "4"