Rework user scripts, update DB schema to support embeddings

2025-12-12 15:08:53 +00:00 · 2023-08-19 15:46:19 -04:00
parent 27188b6fa0
commit 857f3315c2
62 changed files with 1842 additions and 1250 deletions
--- a/sist2-admin/frontend/src/App.vue
+++ b/sist2-admin/frontend/src/App.vue
@@ -7,7 +7,7 @@
                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
                Github</a>. Thank you!
            </b-alert>
-            <router-view/>
+            <router-view v-if="$store.state.sist2AdminInfo"/>
        </b-container>
    </div>
 </template>
@@ -71,10 +71,12 @@ html, body {

 .info-icon {
    width: 1rem;
+    min-width: 1rem;
    margin-right: 0.2rem;
    cursor: pointer;
    line-height: 1rem;
    height: 1rem;
+    min-height: 1rem;
    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
    filter: brightness(45%);
    display: block;
--- a/sist2-admin/frontend/src/Sist2AdminApi.js
+++ b/sist2-admin/frontend/src/Sist2AdminApi.js
@@ -139,6 +139,38 @@ class Sist2AdminApi {
    deleteTaskLogs(taskId) {
        return axios.post(`${this.baseUrl}/api/task/${taskId}/delete_logs`);
    }
+
+    getUserScripts() {
+        return axios.get(`${this.baseUrl}/api/user_script`);
+    }
+
+    getUserScript(name) {
+        return axios.get(`${this.baseUrl}/api/user_script/${name}`);
+    }
+
+    createUserScript(name, template) {
+        return axios.post(`${this.baseUrl}/api/user_script/${name}`, null, {
+            params: {
+                template: template
+            }
+        });
+    }
+
+    updateUserScript(name, data) {
+        return axios.put(`${this.baseUrl}/api/user_script/${name}`, data);
+    }
+
+    deleteUserScript(name) {
+        return axios.delete(`${this.baseUrl}/api/user_script/${name}`);
+    }
+
+    testUserScript(name, job) {
+        return axios.get(`${this.baseUrl}/api/user_script/${name}/run`, {
+            params: {
+                job: job
+            }
+        });
+    }
 }

 export default new Sist2AdminApi()
--- a/sist2-admin/frontend/src/components/JobSelect.vue
+++ b/sist2-admin/frontend/src/components/JobSelect.vue
@@ -0,0 +1,34 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <span v-else-if="jobs.length === 0"></span>
+    <b-form-select v-else :options="jobs" text-field="name" value-field="name"
+                   @change="$emit('change', $event)" :value="$t('selectJob')"></b-form-select>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "JobSelect",
+    mounted() {
+        Sist2AdminApi.getJobs().then(resp => {
+            this._jobs = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        jobs() {
+            return [
+                {name: this.$t("selectJob"), disabled: true},
+                ...this._jobs.filter(job => job.index_path)
+            ]
+        }
+    },
+    data() {
+        return {
+            loading: true,
+            _jobs: null
+        }
+    }
+}
+</script>
--- a/sist2-admin/frontend/src/components/UserScriptListItem.vue
+++ b/sist2-admin/frontend/src/components/UserScriptListItem.vue
@@ -0,0 +1,18 @@
+<template>
+    <b-list-group-item action :to="`/userScript/${script.name}`">
+
+        <div class="d-flex w-100 justify-content-between">
+            <h5 class="mb-1">
+                {{ script.name }}
+            </h5>
+        </div>
+    </b-list-group-item>
+</template>
+
+<script>
+
+export default {
+    name: "UserScriptListItem",
+    props: ["script"],
+}
+</script>
--- a/sist2-admin/frontend/src/components/UserScriptPicker.vue
+++ b/sist2-admin/frontend/src/components/UserScriptPicker.vue
@@ -0,0 +1,88 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+
+    <b-row v-else>
+        <b-col cols="6">
+            <h5>Selected scripts</h5>
+            <b-list-group>
+                <b-list-group-item v-for="script in selectedScripts" :key="script"
+                                   button
+                                   @click="onRemoveScript(script)"
+                                   class="d-flex justify-content-between align-items-center">
+                    {{ script }}
+                    <b-button-group>
+                        <b-button variant="light" @click.stop="moveUpScript(script)">↑</b-button>
+                        <b-button variant="light" @click.stop="moveDownScript(script)">↓</b-button>
+                    </b-button-group>
+                </b-list-group-item>
+            </b-list-group>
+        </b-col>
+        <b-col cols="6">
+            <h5>Available scripts</h5>
+            <b-list-group>
+                <b-list-group-item v-for="script in availableScripts" :key="script" button
+                                   @click="onSelectScript(script)">
+                    {{ script }}
+                </b-list-group-item>
+            </b-list-group>
+        </b-col>
+    </b-row>
+
+  <!--    <b-checkbox-group v-else :options="scripts" stacked :checked="selectedScripts"-->
+  <!--                      @input="$emit('change', $event)"></b-checkbox-group>-->
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "UserScriptPicker",
+    props: ["selectedScripts"],
+    data() {
+        return {
+            loading: true,
+            scripts: []
+        }
+    },
+    computed: {
+        availableScripts() {
+            return this.scripts.filter(script => !this.selectedScripts.includes(script))
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getUserScripts().then(resp => {
+            this.scripts = resp.data.map(script => script.name);
+            this.loading = false;
+        });
+    },
+    methods: {
+        onSelectScript(name) {
+            this.selectedScripts.push(name);
+            this.$emit("change", this.selectedScripts)
+        },
+        onRemoveScript(name) {
+            this.selectedScripts.splice(this.selectedScripts.indexOf(name), 1);
+            this.$emit("change", this.selectedScripts);
+        },
+        moveUpScript(name) {
+            const index = this.selectedScripts.indexOf(name);
+            if (index > 0) {
+                this.selectedScripts.splice(index, 1);
+                this.selectedScripts.splice(index - 1, 0, name);
+            }
+            this.$emit("change", this.selectedScripts);
+        },
+        moveDownScript(name) {
+            const index = this.selectedScripts.indexOf(name);
+            if (index < this.selectedScripts.length - 1) {
+                this.selectedScripts.splice(index, 1);
+                this.selectedScripts.splice(index + 1, 0, name);
+            }
+            this.$emit("change", this.selectedScripts);
+        }
+    }
+}
+</script>
+
+<style scoped>
+</style>
--- a/sist2-admin/frontend/src/i18n/messages.js
+++ b/sist2-admin/frontend/src/i18n/messages.js
@@ -54,8 +54,18 @@ export default {

        frontendTab: "Frontend",
        backendTab: "Backend",
+        scripts: "User Scripts",
+        script: "User Script",
+        testScript: "Test/debug User Script",
+        newScriptName: "New script name",
+        scriptType: "Script type",
+        scriptCode: "Script code (Python)",
+        scriptOptions: "User scripts",
+        gitRepository: "Git repository URL",
+        extraArgs: "Extra command line arguments",

        selectJobs: "Available jobs",
+        selectJob: "Select a job",
        webOptions: {
            title: "Web options",
            lang: "UI Language",
--- a/sist2-admin/frontend/src/router/index.js
+++ b/sist2-admin/frontend/src/router/index.js
@@ -6,12 +6,18 @@ import Tasks from "@/views/Tasks";
 import Frontend from "@/views/Frontend";
 import Tail from "@/views/Tail";
 import SearchBackend from "@/views/SearchBackend.vue";
+import UserScript from "@/views/UserScript.vue";

 Vue.use(VueRouter);

 const routes = [
  {
-    path: "/",
+    path: "/task",
+    name: "Tasks",
+    component: Tasks
+  },
+  {
+    path: "/:tab?",
    name: "Home",
    component: Home
  },
@@ -20,11 +26,6 @@ const routes = [
    name: "Job",
    component: Job
  },
-  {
-    path: "/task/",
-    name: "Tasks",
-    component: Tasks
-  },
  {
    path: "/frontend/:name",
    name: "Frontend",
@@ -35,6 +36,11 @@ const routes = [
    name: "SearchBackend",
    component: SearchBackend
  },
+  {
+    path: "/userScript/:name",
+    name: "UserScript",
+    component: UserScript
+  },
  {
    path: "/log/:taskId",
    name: "Tail",
--- a/sist2-admin/frontend/src/views/Home.vue
+++ b/sist2-admin/frontend/src/views/Home.vue
@@ -1,6 +1,6 @@
 <template>
    <div>
-        <b-tabs content-class="mt-3">
+        <b-tabs content-class="mt-3" v-model="tab" @input="onTabChange($event)">
            <b-tab :title="$t('backendTab')">

                <b-card>
@@ -25,7 +25,6 @@
                        <SearchBackendListItem v-for="backend in backends"
                                               :key="backend.name" :backend="backend"></SearchBackendListItem>
                    </b-list-group>
-
                </b-card>

                <br/>
@@ -36,12 +35,12 @@
                        <b-col>
                            <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
                            <b-popover
-                                :show.sync="showHelp"
-                                target="new-job"
-                                placement="top"
-                                triggers="manual"
-                                variant="primary"
-                                :content="$t('newJobHelp')"
+                                    :show.sync="showHelp"
+                                    target="new-job"
+                                    placement="top"
+                                    triggers="manual"
+                                    variant="primary"
+                                    :content="$t('newJobHelp')"
                            ></b-popover>
                        </b-col>
                        <b-col>
@@ -59,6 +58,37 @@
                    </b-list-group>
                </b-card>
            </b-tab>
+            <b-tab :title="$t('scripts')">
+
+                <b-progress v-if="scriptsLoading" striped animated value="100"></b-progress>
+                <b-card v-else>
+                    <b-card-title>{{ $t("scripts") }}</b-card-title>
+
+                    <label>Select template</label>
+                    <b-form-radio-group stacked :options="scriptTemplates" v-model="scriptTemplate"></b-form-radio-group>
+                    <br>
+
+                    <b-row>
+                        <b-col>
+                            <b-form-input v-model="newScriptName" :disabled="!scriptTemplate" :placeholder="$t('newScriptName')"></b-form-input>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createScript()"
+                                      :disabled="!scriptNameValid(newScriptName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
+
+                    <hr/>
+
+                    <b-list-group>
+                        <UserScriptListItem v-for="script in scripts"
+                                            :key="script.name" :script="script"></UserScriptListItem>
+                    </b-list-group>
+
+                </b-card>
+            </b-tab>
            <b-tab :title="$t('frontendTab')">
                <b-card>

@@ -96,10 +126,11 @@ import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
 import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
+import UserScriptListItem from "@/components/UserScriptListItem.vue";

 export default {
    name: "Jobs",
-    components: {SearchBackendListItem, JobListItem, FrontendListItem},
+    components: {UserScriptListItem, SearchBackendListItem, JobListItem, FrontendListItem},
    data() {
        return {
            jobsLoading: true,
@@ -115,11 +146,24 @@ export default {
            backendsLoading: true,
            newBackendName: "",

-            showHelp: false
+            scripts: [],
+            scriptTemplates: [],
+            newScriptName: "",
+            scriptTemplate: null,
+            scriptsLoading: true,
+
+            showHelp: false,
+            tab: 0
        }
    },
    mounted() {
        this.loading = true;
+        if (this.$route.params.tab) {
+            console.log("mounted " + this.$route.params.tab)
+            window.setTimeout(() => {
+                this.tab = Math.round(Number(this.$route.params.tab));
+            }, 1)
+        }
        this.reload();
    },
    methods: {
@@ -144,11 +188,20 @@ export default {

            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
        },
+        scriptNameValid(name) {
+            if (this.scripts.some(script => script.name === name)) {
+                return false;
+            }
+            if (name.length > 16) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
        reload() {
            Sist2AdminApi.getJobs().then(resp => {
                this.jobs = resp.data;
                this.jobsLoading = false;
-
                this.showHelp = this.jobs.length === 0;
            });
            Sist2AdminApi.getFrontends().then(resp => {
@@ -159,6 +212,11 @@ export default {
                this.backends = resp.data;
                this.backendsLoading = false;
            })
+            Sist2AdminApi.getUserScripts().then(resp => {
+                this.scripts = resp.data;
+                this.scriptTemplates = this.$store.state.sist2AdminInfo.user_script_templates;
+                this.scriptsLoading = false;
+            })
        },
        createJob() {
            Sist2AdminApi.createJob(this.newJobName).then(this.reload);
@@ -168,6 +226,14 @@ export default {
        },
        createBackend() {
            Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
+        },
+        createScript() {
+            Sist2AdminApi.createUserScript(this.newScriptName, this.scriptTemplate).then(this.reload)
+        },
+        onTabChange(tab) {
+            if (this.$route.params.tab != tab) {
+                this.$router.push({params: {tab: tab}})
+            }
        }
    }
 }
--- a/sist2-admin/frontend/src/views/Job.vue
+++ b/sist2-admin/frontend/src/views/Job.vue
@@ -30,6 +30,13 @@
                <SearchBackendSelect :value="job.index_options.search_backend"
                                     @change="onBackendSelect($event)"></SearchBackendSelect>
            </b-card>
+            <br/>
+
+            <h4>{{ $t("scriptOptions") }}</h4>
+            <b-card>
+                <UserScriptPicker :selected-scripts="job.user_scripts"
+                                  @change="onScriptChange($event)"></UserScriptPicker>
+            </b-card>

            <br/>

@@ -48,10 +55,12 @@ import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobOptions from "@/components/JobOptions";
 import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
+import UserScriptPicker from "@/components/UserScriptPicker.vue";

 export default {
    name: "Job",
    components: {
+        UserScriptPicker,
        SearchBackendSelect,
        ScanOptions,
        JobOptions
@@ -95,6 +104,10 @@ export default {
        onBackendSelect(backend) {
            this.job.index_options.search_backend = backend;
            this.update();
+        },
+        onScriptChange(scripts) {
+            this.job.user_scripts = scripts;
+            this.update();
        }
    },
    mounted() {
--- a/sist2-admin/frontend/src/views/SearchBackend.vue
+++ b/sist2-admin/frontend/src/views/SearchBackend.vue
@@ -44,9 +44,6 @@

                <label>{{ $t("backendOptions.batchSize") }}</label>
                <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-                <label>{{ $t("backendOptions.script") }}</label>
-                <b-form-textarea v-model="backend.script" rows="6" @change="update()"></b-form-textarea>
            </template>
            <template v-else>
                <label>{{ $t("backendOptions.searchIndex") }}</label>
--- a/sist2-admin/frontend/src/views/Tail.vue
+++ b/sist2-admin/frontend/src/views/Tail.vue
@@ -92,6 +92,9 @@ export default {
                    if ("stderr" in message) {
                        message.level = "ERROR";
                        message.message = message["stderr"];
+                    } else if ("stdout" in message) {
+                        message.level = "INFO";
+                        message.message = message["stdout"];
                    } else {
                        message.level = "ADMIN";
                        message.message = message["sist2-admin"];
--- a/sist2-admin/frontend/src/views/UserScript.vue
+++ b/sist2-admin/frontend/src/views/UserScript.vue
@@ -0,0 +1,117 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card v-else>
+        <b-card-title>
+            {{ $route.params.name }}
+            {{ $t("script") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteScript()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-card>
+            <h5>{{ $t("testScript") }}</h5>
+
+            <b-row>
+                <b-col cols="11">
+                    <JobSelect @change="onJobSelect($event)"></JobSelect>
+                </b-col>
+                <b-col cols="1">
+                    <b-button :disabled="!selectedTestJob" variant="primary" @click="testScript()">{{ $t("test") }}
+                    </b-button>
+                </b-col>
+            </b-row>
+
+        </b-card>
+        <br/>
+
+        <label>{{ $t("scriptType") }}</label>
+        <b-form-select :options="['git', 'simple']" v-model="script.type" @change="update()"></b-form-select>
+
+        <template v-if="script.type === 'git'">
+            <label>{{ $t("gitRepository") }}</label>
+            <b-form-input v-model="script.git_repository" placeholder="https://github.com/example/example.git"
+                          @change="update()"></b-form-input>
+
+            <label>{{ $t("extraArgs") }}</label>
+            <b-form-input v-model="script.extra_args" @change="update()" class="text-monospace"></b-form-input>
+        </template>
+
+        <template v-if="script.type === 'simple'">
+
+            <label>{{ $t("scriptCode") }}</label>
+            <p>Find sist2-python documentation <a href="https://sist2-python.readthedocs.io/" target="_blank">here</a></p>
+            <b-textarea rows="15" class="text-monospace" v-model="script.script" @change="update()" spellcheck="false"></b-textarea>
+        </template>
+
+        <template v-if="script.type === 'local'">
+            <!-- TODO-->
+        </template>
+
+
+    </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobOptions from "@/components/JobOptions.vue";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup.vue";
+import JobSelect from "@/components/JobSelect.vue";
+
+export default {
+    name: "UserScript",
+    components: {JobSelect, JobCheckboxGroup, JobOptions},
+    data() {
+        return {
+            loading: true,
+            script: null,
+            selectedTestJob: null
+        }
+    },
+    methods: {
+        update() {
+            Sist2AdminApi.updateUserScript(this.name, this.script);
+        },
+        onJobSelect(job) {
+            this.selectedTestJob = job;
+        },
+        deleteScript() {
+            Sist2AdminApi.deleteUserScript(this.name)
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete user script " +
+                        "because it is referenced by a job", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        testScript() {
+            Sist2AdminApi.testUserScript(this.name, this.selectedTestJob)
+                .then(() => {
+                    this.$bvToast.toast(this.$t("runJobConfirmation"), {
+                        title: this.$t("runJobConfirmationTitle"),
+                        variant: "success",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getUserScript(this.name).then(resp => {
+            this.script = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        name() {
+            return this.$route.params.name;
+        },
+    },
+}
+</script>