Rework user scripts, update DB schema to support embeddings

sist2-admin/frontend/src/views/Home.vue

@@ -1,6 +1,6 @@
 <template>
     <div>
-        <b-tabs content-class="mt-3">
+        <b-tabs content-class="mt-3" v-model="tab" @input="onTabChange($event)">
             <b-tab :title="$t('backendTab')">
 
                 <b-card>
@@ -25,7 +25,6 @@
                         <SearchBackendListItem v-for="backend in backends"
                                                :key="backend.name" :backend="backend"></SearchBackendListItem>
                     </b-list-group>
-
                 </b-card>
 
                 <br/>
@@ -36,12 +35,12 @@
                         <b-col>
                             <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
                             <b-popover
-                                :show.sync="showHelp"
-                                target="new-job"
-                                placement="top"
-                                triggers="manual"
-                                variant="primary"
-                                :content="$t('newJobHelp')"
+                                    :show.sync="showHelp"
+                                    target="new-job"
+                                    placement="top"
+                                    triggers="manual"
+                                    variant="primary"
+                                    :content="$t('newJobHelp')"
                             ></b-popover>
                         </b-col>
                         <b-col>
@@ -59,6 +58,37 @@
                     </b-list-group>
                 </b-card>
             </b-tab>
+            <b-tab :title="$t('scripts')">
+
+                <b-progress v-if="scriptsLoading" striped animated value="100"></b-progress>
+                <b-card v-else>
+                    <b-card-title>{{ $t("scripts") }}</b-card-title>
+
+                    <label>Select template</label>
+                    <b-form-radio-group stacked :options="scriptTemplates" v-model="scriptTemplate"></b-form-radio-group>
+                    <br>
+
+                    <b-row>
+                        <b-col>
+                            <b-form-input v-model="newScriptName" :disabled="!scriptTemplate" :placeholder="$t('newScriptName')"></b-form-input>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createScript()"
+                                      :disabled="!scriptNameValid(newScriptName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
+
+                    <hr/>
+
+                    <b-list-group>
+                        <UserScriptListItem v-for="script in scripts"
+                                            :key="script.name" :script="script"></UserScriptListItem>
+                    </b-list-group>
+
+                </b-card>
+            </b-tab>
             <b-tab :title="$t('frontendTab')">
                 <b-card>
 
@@ -96,10 +126,11 @@ import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
 import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
+import UserScriptListItem from "@/components/UserScriptListItem.vue";
 
 export default {
     name: "Jobs",
-    components: {SearchBackendListItem, JobListItem, FrontendListItem},
+    components: {UserScriptListItem, SearchBackendListItem, JobListItem, FrontendListItem},
     data() {
         return {
             jobsLoading: true,
@@ -115,11 +146,24 @@ export default {
             backendsLoading: true,
             newBackendName: "",
 
-            showHelp: false
+            scripts: [],
+            scriptTemplates: [],
+            newScriptName: "",
+            scriptTemplate: null,
+            scriptsLoading: true,
+
+            showHelp: false,
+            tab: 0
         }
     },
     mounted() {
         this.loading = true;
+        if (this.$route.params.tab) {
+            console.log("mounted " + this.$route.params.tab)
+            window.setTimeout(() => {
+                this.tab = Math.round(Number(this.$route.params.tab));
+            }, 1)
+        }
         this.reload();
     },
     methods: {
@@ -144,11 +188,20 @@ export default {
 
             return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
         },
+        scriptNameValid(name) {
+            if (this.scripts.some(script => script.name === name)) {
+                return false;
+            }
+            if (name.length > 16) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
         reload() {
             Sist2AdminApi.getJobs().then(resp => {
                 this.jobs = resp.data;
                 this.jobsLoading = false;
-
                 this.showHelp = this.jobs.length === 0;
             });
             Sist2AdminApi.getFrontends().then(resp => {
@@ -159,6 +212,11 @@ export default {
                 this.backends = resp.data;
                 this.backendsLoading = false;
             })
+            Sist2AdminApi.getUserScripts().then(resp => {
+                this.scripts = resp.data;
+                this.scriptTemplates = this.$store.state.sist2AdminInfo.user_script_templates;
+                this.scriptsLoading = false;
+            })
         },
         createJob() {
             Sist2AdminApi.createJob(this.newJobName).then(this.reload);
@@ -168,6 +226,14 @@ export default {
         },
         createBackend() {
             Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
+        },
+        createScript() {
+            Sist2AdminApi.createUserScript(this.newScriptName, this.scriptTemplate).then(this.reload)
+        },
+        onTabChange(tab) {
+            if (this.$route.params.tab != tab) {
+                this.$router.push({params: {tab: tab}})
+            }
         }
     }
 }

sist2-admin/frontend/src/views/Job.vue

@@ -30,6 +30,13 @@
                 <SearchBackendSelect :value="job.index_options.search_backend"
                                      @change="onBackendSelect($event)"></SearchBackendSelect>
             </b-card>
+            <br/>
+
+            <h4>{{ $t("scriptOptions") }}</h4>
+            <b-card>
+                <UserScriptPicker :selected-scripts="job.user_scripts"
+                                  @change="onScriptChange($event)"></UserScriptPicker>
+            </b-card>
 
             <br/>
 
@@ -48,10 +55,12 @@ import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobOptions from "@/components/JobOptions";
 import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
+import UserScriptPicker from "@/components/UserScriptPicker.vue";
 
 export default {
     name: "Job",
     components: {
+        UserScriptPicker,
         SearchBackendSelect,
         ScanOptions,
         JobOptions
@@ -95,6 +104,10 @@ export default {
         onBackendSelect(backend) {
             this.job.index_options.search_backend = backend;
             this.update();
+        },
+        onScriptChange(scripts) {
+            this.job.user_scripts = scripts;
+            this.update();
         }
     },
     mounted() {

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -44,9 +44,6 @@
 
                 <label>{{ $t("backendOptions.batchSize") }}</label>
                 <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-                <label>{{ $t("backendOptions.script") }}</label>
-                <b-form-textarea v-model="backend.script" rows="6" @change="update()"></b-form-textarea>
             </template>
             <template v-else>
                 <label>{{ $t("backendOptions.searchIndex") }}</label>

sist2-admin/frontend/src/views/Tail.vue

@@ -92,6 +92,9 @@ export default {
                     if ("stderr" in message) {
                         message.level = "ERROR";
                         message.message = message["stderr"];
+                    } else if ("stdout" in message) {
+                        message.level = "INFO";
+                        message.message = message["stdout"];
                     } else {
                         message.level = "ADMIN";
                         message.message = message["sist2-admin"];

sist2-admin/frontend/src/views/UserScript.vue

@@ -0,0 +1,117 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card v-else>
+        <b-card-title>
+            {{ $route.params.name }}
+            {{ $t("script") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteScript()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-card>
+            <h5>{{ $t("testScript") }}</h5>
+
+            <b-row>
+                <b-col cols="11">
+                    <JobSelect @change="onJobSelect($event)"></JobSelect>
+                </b-col>
+                <b-col cols="1">
+                    <b-button :disabled="!selectedTestJob" variant="primary" @click="testScript()">{{ $t("test") }}
+                    </b-button>
+                </b-col>
+            </b-row>
+
+        </b-card>
+        <br/>
+
+        <label>{{ $t("scriptType") }}</label>
+        <b-form-select :options="['git', 'simple']" v-model="script.type" @change="update()"></b-form-select>
+
+        <template v-if="script.type === 'git'">
+            <label>{{ $t("gitRepository") }}</label>
+            <b-form-input v-model="script.git_repository" placeholder="https://github.com/example/example.git"
+                          @change="update()"></b-form-input>
+
+            <label>{{ $t("extraArgs") }}</label>
+            <b-form-input v-model="script.extra_args" @change="update()" class="text-monospace"></b-form-input>
+        </template>
+
+        <template v-if="script.type === 'simple'">
+
+            <label>{{ $t("scriptCode") }}</label>
+            <p>Find sist2-python documentation <a href="https://sist2-python.readthedocs.io/" target="_blank">here</a></p>
+            <b-textarea rows="15" class="text-monospace" v-model="script.script" @change="update()" spellcheck="false"></b-textarea>
+        </template>
+
+        <template v-if="script.type === 'local'">
+            <!-- TODO-->
+        </template>
+
+
+    </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobOptions from "@/components/JobOptions.vue";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup.vue";
+import JobSelect from "@/components/JobSelect.vue";
+
+export default {
+    name: "UserScript",
+    components: {JobSelect, JobCheckboxGroup, JobOptions},
+    data() {
+        return {
+            loading: true,
+            script: null,
+            selectedTestJob: null
+        }
+    },
+    methods: {
+        update() {
+            Sist2AdminApi.updateUserScript(this.name, this.script);
+        },
+        onJobSelect(job) {
+            this.selectedTestJob = job;
+        },
+        deleteScript() {
+            Sist2AdminApi.deleteUserScript(this.name)
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete user script " +
+                        "because it is referenced by a job", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        testScript() {
+            Sist2AdminApi.testUserScript(this.name, this.selectedTestJob)
+                .then(() => {
+                    this.$bvToast.toast(this.$t("runJobConfirmation"), {
+                        title: this.$t("runJobConfirmationTitle"),
+                        variant: "success",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getUserScript(this.name).then(resp => {
+            this.script = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        name() {
+            return this.$route.params.name;
+        },
+    },
+}
+</script>