simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-12-12 06:58:54 +00:00
Code Issues Projects Releases Wiki Activity

Rework user scripts, update DB schema to support embeddings

Browse Source
This commit is contained in:
simon987
2023-08-19 15:46:19 -04:00
parent 27188b6fa0
commit 857f3315c2
62 changed files with 1842 additions and 1250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

609
sist2-vue/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

7
sist2-vue/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "sist2",
"version": "2.11.0",
"version": "1.0.0",
"private": true,
"scripts": {
"serve": "vue-cli-service serve",
@@ -9,7 +9,6 @@
"dependencies": {
"@auth0/auth0-spa-js": "^2.0.2",
"@egjs/vue-infinitegrid": "3.3.0",
"@tensorflow/tfjs": "^4.4.0",
"axios": "^0.25.0",
"bootstrap-vue": "^2.21.2",
"core-js": "^3.6.5",
@@ -45,8 +44,8 @@
"portal-vue": "^2.1.7",
"sass": "^1.26.11",
"sass-loader": "^10.0.2",
"typescript": "~4.1.5",
"vue-cli-plugin-bootstrap-vue": "~0.7.0",
"typescript": "^4.9.5",
"vue-cli-plugin-bootstrap-vue": "~0.8.2",
"vue-template-compiler": "^2.6.11"
},
"browserslist": [

6
sist2-vue/src/App.vue
View File

@@ -308,15 +308,21 @@ html, body {
.info-icon {
width: 1rem;
min-width: 1rem;
margin-right: 0.2rem;
cursor: pointer;
line-height: 1rem;
height: 1rem;
min-height: 1rem;
background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
filter: brightness(45%);
display: block;
}
.theme-black .info-icon {
filter: brightness(80%);
}
.tabs {
margin-top: 10px;
}

27
sist2-vue/src/Sist2Api.ts
View File

@@ -25,6 +25,7 @@ export interface Index {
id: string
idPrefix: string
timestamp: number
models: []
}
export interface EsHit {
@@ -117,6 +118,15 @@ class Sist2Api {
return this.sist2Info.searchBackend;
}
models() {
const allModels = this.sist2Info.indices
.map(idx => idx.models)
.flat();
return allModels
.filter((v, i, a) => a.findIndex(v2 => (v2.id === v.id)) === i)
}
getSist2Info(): Promise<any> {
return axios.get(`${this.baseUrl}i`).then(resp => {
const indices = resp.data.indices as Index[];
@@ -127,7 +137,8 @@ class Sist2Api {
name: idx.name,
timestamp: idx.timestamp,
version: idx.version,
idPrefix: getIdPrefix(indices, idx.id)
models: idx.models,
idPrefix: getIdPrefix(indices, idx.id),
} as Index;
});
@@ -618,6 +629,15 @@ class Sist2Api {
}
}
if ("knn" in query) {
query.query = {
bool: {
must: []
}
};
delete query.knn;
}
if ("function_score" in query.query) {
query.query = query.query.function_score.query;
}
@@ -702,6 +722,11 @@ class Sist2Api {
return result;
});
}
getEmbeddings(indexId, docId, modelId) {
return axios.post(`${this.baseUrl}/e/${indexId}/${docId}/${modelId.toString().padStart(3, '0')}`)
.then(resp => (resp.data));
}
}
export default new Sist2Api("");

59
sist2-vue/src/Sist2ElasticsearchQuery.ts
View File

@@ -1,5 +1,5 @@
import store from "./store";
import {EsHit, Index} from "@/Sist2Api";
import sist2Api, {EsHit, Index} from "@/Sist2Api";
const SORT_MODES = {
score: {
@@ -79,8 +79,10 @@ class Sist2ElasticsearchQuery {
const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
const selectedMimeTypes = getters.selectedMimeTypes;
const selectedTags = getters.selectedTags;
const sortMode = getters.embedding ? "score" : getters.sortMode;
const legacyES = store.state.sist2Info.esVersionLegacy;
const hasKnn = store.state.sist2Info.esVersionHasKnn;
const filters = [
{terms: {index: selectedIndexIds}}
@@ -162,14 +164,14 @@ class Sist2ElasticsearchQuery {
const q = {
_source: {
excludes: ["content", "_tie"]
excludes: ["content", "_tie", "emb.*"]
},
query: {
bool: {
filter: filters,
}
},
sort: SORT_MODES[getters.sortMode].mode,
sort: SORT_MODES[sortMode].mode,
size: size,
} as any;
@@ -181,14 +183,57 @@ class Sist2ElasticsearchQuery {
}
if (!empty && !blankSearch) {
q.query.bool.must = query;
if (getters.embedding) {
filters.push(query)
} else {
q.query.bool.must = query;
}
}
if (getters.embedding) {
delete q.query;
const field = "emb." + sist2Api.models().find(m => m.id == getters.embeddingsModel).path;
if (hasKnn) {
// Use knn (8.8+)
q.knn = {
field: field,
query_vector: getters.embedding,
k: 600,
num_candidates: 600,
filter: filters
}
} else {
// Use brute-force as a fallback
filters.push({exists: {field: field}});
q.query = {
function_score: {
query: {
bool: {
must: filters,
}
},
script_score: {
script: {
source: `cosineSimilarity(params.query_vector, "${field}") + 1.0`,
params: {query_vector: getters.embedding}
}
}
}
}
}
}
if (after) {
q.search_after = [SORT_MODES[getters.sortMode].key(after), after["_id"]];
q.search_after = [SORT_MODES[sortMode].key(after), after["_id"]];
}
if (getters.optHighlight) {
if (getters.optHighlight && !getters.embedding) {
q.highlight = {
pre_tags: ["<mark>"],
post_tags: ["</mark>"],
@@ -214,7 +259,7 @@ class Sist2ElasticsearchQuery {
}
}
if (getters.sortMode === "random") {
if (sortMode === "random") {
q.query = {
function_score: {
query: {

2
sist2-vue/src/Sist2SqliteQuery.ts
View File

@@ -103,7 +103,7 @@ class Sist2ElasticsearchQuery {
q["highlightContextSize"] = Number(getters.optFragmentSize);
}
if (getters.embeddingText) {
if (getters.embedding) {
q["model"] = getters.embeddingsModel;
q["embedding"] = getters.embedding;
q["sort"] = "embedding";

3
sist2-vue/src/components/DebugInfo.vue
View File

@@ -45,7 +45,8 @@ export default {
items.push(
{key: "esVersion", value: this.$store.state.sist2Info.esVersion},
{key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
{key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy}
{key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
{key: "esVersionHasKnn", value: this.$store.state.sist2Info.esVersionHasKnn},
);
}

12
sist2-vue/src/components/DocCard.vue
View File

@@ -24,6 +24,7 @@
<!-- Title line -->
<div style="display: flex">
<span class="info-icon" @click="onInfoClick()"></span>
<MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
<DocFileTitle :doc="doc"></DocFileTitle>
</div>
@@ -49,10 +50,12 @@ import DocInfoModal from "@/components/DocInfoModal.vue";
import ContentDiv from "@/components/ContentDiv.vue";
import FullThumbnail from "@/components/FullThumbnail";
import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
import MLIcon from "@/components/icons/MlIcon.vue";
import Sist2Api from "@/Sist2Api";
export default {
components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
components: {MLIcon, FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
props: ["doc", "width"],
data() {
return {
@@ -71,6 +74,13 @@ export default {
onInfoClick() {
this.showInfo = true;
},
onEmbeddingClick() {
Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
this.$store.commit("setEmbeddingText", "");
this.$store.commit("setEmbedding", embeddings);
this.$store.commit("setEmbeddingDoc", this.doc);
})
},
async onThumbnailClick() {
this.$store.commit("setUiLightboxSlide", this.doc._seq);
await this.$store.dispatch("showLightbox");

75
sist2-vue/src/components/DocFileTitle.vue
View File

@@ -1,63 +1,70 @@
<template>
<a :href="`f/${doc._source.index}/${doc._id}`" class="file-title-anchor" target="_blank">
<div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
v-html="fileName() + ext(doc)"></div>
</a>
<a :href="`f/${doc._source.index}/${doc._id}`"
:class="doc._source.embedding ? 'file-title-anchor-with-embedding' : 'file-title-anchor'" target="_blank">
<div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
v-html="fileName() + ext(doc)"></div>
</a>
</template>
<script>
import {ext} from "@/util";
export default {
name: "DocFileTitle",
props: ["doc"],
methods: {
ext: ext,
fileName() {
if (!this.doc.highlight) {
return this.doc._source.name;
}
if (this.doc.highlight["name.nGram"]) {
return this.doc.highlight["name.nGram"];
}
if (this.doc.highlight.name) {
return this.doc.highlight.name;
}
return this.doc._source.name;
name: "DocFileTitle",
props: ["doc"],
methods: {
ext: ext,
fileName() {
if (!this.doc.highlight) {
return this.doc._source.name;
}
if (this.doc.highlight["name.nGram"]) {
return this.doc.highlight["name.nGram"];
}
if (this.doc.highlight.name) {
return this.doc.highlight.name;
}
return this.doc._source.name;
}
}
}
}
</script>
<style scoped>
.file-title-anchor {
max-width: calc(100% - 1.2rem);
max-width: calc(100% - 1.2rem);
}
.file-title-anchor-with-embedding {
max-width: calc(100% - 2.2rem);
}
.file-title {
width: 100%;
line-height: 1rem;
height: 1.1rem;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
font-size: 16px;
font-family: "Source Sans Pro", sans-serif;
font-weight: bold;
width: 100%;
max-width: 100%;
line-height: 1rem;
height: 1.1rem;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
font-size: 16px;
font-family: "Source Sans Pro", sans-serif;
font-weight: bold;
}
.theme-black .file-title {
color: #ddd;
color: #ddd;
}
.theme-black .file-title:hover {
color: #fff;
color: #fff;
}
.theme-light .file-title {
color: black;
color: black;
}
.doc-card .file-title {
font-size: 12px;
font-size: 12px;
}
</style>

262
sist2-vue/src/components/DocListItem.vue
View File

@@ -1,63 +1,64 @@
<template>
<b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}"
@mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
<b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}"
@mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
<!-- Info modal-->
<DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
<!-- Info modal-->
<DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
<div class="media ml-2">
<div class="media ml-2">
<!-- Thumbnail-->
<div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
<div class="img-wrapper">
<div v-if="doc._props.isPlayableVideo" class="play">
<svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
<path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
</svg>
</div>
<!-- Thumbnail-->
<div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
<div class="img-wrapper">
<div v-if="doc._props.isPlayableVideo" class="play">
<svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
<path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
</svg>
</div>
<img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
:src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
alt=""
class="pointer fit-sm" @click="onThumbnailClick()">
<img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
class="fit-sm">
</div>
</div>
<div v-else class="file-icon-wrapper" style="">
<FileIcon></FileIcon>
</div>
<img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
:src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
alt=""
class="pointer fit-sm" @click="onThumbnailClick()">
<img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
class="fit-sm">
</div>
</div>
<div v-else class="file-icon-wrapper" style="">
<FileIcon></FileIcon>
</div>
<!-- Doc line-->
<div class="doc-line ml-3">
<div style="display: flex">
<span class="info-icon" @click="showInfo = true"></span>
<DocFileTitle :doc="doc"></DocFileTitle>
</div>
<!-- Doc line-->
<div class="doc-line ml-3">
<div style="display: flex">
<span class="info-icon" @click="showInfo = true"></span>
<MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
<DocFileTitle :doc="doc"></DocFileTitle>
</div>
<!-- Content highlight -->
<ContentDiv :doc="doc"></ContentDiv>
<!-- Content highlight -->
<ContentDiv :doc="doc"></ContentDiv>
<div class="path-row">
<div class="path-line" v-html="path()"></div>
<TagContainer :hit="doc"></TagContainer>
</div>
<div class="path-row">
<div class="path-line" v-html="path()"></div>
<TagContainer :hit="doc"></TagContainer>
</div>
<div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
<div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
<span v-if="doc._source.pages">{{ doc._source.pages }} {{
doc._source.pages > 1 ? $t("pages") : $t("page")
}}</span>
<span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
<span v-if="doc._source.author">{{ doc._source.author }}</span>
</div>
}}</span>
<span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
<span v-if="doc._source.author">{{ doc._source.author }}</span>
</div>
<!-- Featured line -->
<div style="display: flex">
<FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
<!-- Featured line -->
<div style="display: flex">
<FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
</div>
</div>
</div>
</div>
</div>
</b-list-group-item>
</b-list-group-item>
</template>
<script>
@@ -67,131 +68,140 @@ import DocInfoModal from "@/components/DocInfoModal";
import ContentDiv from "@/components/ContentDiv";
import FileIcon from "@/components/icons/FileIcon";
import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
import MLIcon from "@/components/icons/MlIcon.vue";
import Sist2Api from "@/Sist2Api";
export default {
name: "DocListItem",
components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
props: ["doc"],
data() {
return {
hover: false,
showInfo: false
}
},
methods: {
async onThumbnailClick() {
this.$store.commit("setUiLightboxSlide", this.doc._seq);
await this.$store.dispatch("showLightbox");
name: "DocListItem",
components: {MLIcon, FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
props: ["doc"],
data() {
return {
hover: false,
showInfo: false
}
},
path() {
if (!this.doc.highlight) {
return this.doc._source.path + "/"
}
if (this.doc.highlight["path.text"]) {
return this.doc.highlight["path.text"] + "/"
}
methods: {
async onThumbnailClick() {
this.$store.commit("setUiLightboxSlide", this.doc._seq);
await this.$store.dispatch("showLightbox");
},
onEmbeddingClick() {
Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
this.$store.commit("setEmbeddingText", "");
this.$store.commit("setEmbedding", embeddings);
this.$store.commit("setEmbeddingDoc", this.doc);
})
},
path() {
if (!this.doc.highlight) {
return this.doc._source.path + "/"
}
if (this.doc.highlight["path.text"]) {
return this.doc.highlight["path.text"] + "/"
}
if (this.doc.highlight["path.nGram"]) {
return this.doc.highlight["path.nGram"] + "/"
}
return this.doc._source.path + "/"
},
onTnEnter() {
this.hover = true;
},
onTnLeave() {
this.hover = false;
},
}
if (this.doc.highlight["path.nGram"]) {
return this.doc.highlight["path.nGram"] + "/"
}
return this.doc._source.path + "/"
},
onTnEnter() {
this.hover = true;
},
onTnLeave() {
this.hover = false;
},
}
}
</script>
<style scoped>
.sub-document {
background: #AB47BC1F !important;
background: #AB47BC1F !important;
}
.theme-black .sub-document {
background: #37474F !important;
background: #37474F !important;
}
.list-group {
margin-top: 1em;
margin-top: 1em;
}
.list-group-item {
padding: .25rem 0.5rem;
padding: .25rem 0.5rem;
box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
border-radius: 0;
border: none;
box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
border-radius: 0;
border: none;
}
.path-row {
display: -ms-flexbox;
display: flex;
-ms-flex-align: start;
align-items: flex-start;
display: -ms-flexbox;
display: flex;
-ms-flex-align: start;
align-items: flex-start;
}
.path-line {
color: #808080;
text-overflow: ellipsis;
overflow: hidden;
white-space: nowrap;
margin-right: 0.3em;
color: #808080;
text-overflow: ellipsis;
overflow: hidden;
white-space: nowrap;
margin-right: 0.3em;
}
.theme-black .path-line {
color: #bbb;
color: #bbb;
}
.play {
position: absolute;
width: 18px;
height: 18px;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
pointer-events: none;
position: absolute;
width: 18px;
height: 18px;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
pointer-events: none;
}
.play svg {
fill: rgba(0, 0, 0, 0.7);
fill: rgba(0, 0, 0, 0.7);
}
.list-group-item .img-wrapper {
width: 88px;
height: 88px;
position: relative;
width: 88px;
height: 88px;
position: relative;
}
.fit-sm {
max-height: 100%;
max-width: 100%;
width: auto;
height: auto;
position: absolute;
top: 0;
bottom: 0;
left: 0;
right: 0;
margin: auto;
max-height: 100%;
max-width: 100%;
width: auto;
height: auto;
position: absolute;
top: 0;
bottom: 0;
left: 0;
right: 0;
margin: auto;
/*box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.12);*/
/*box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.12);*/
}
.doc-line {
max-width: calc(100% - 88px - 1.5rem);
flex: 1;
vertical-align: middle;
margin-top: auto;
margin-bottom: auto;
max-width: calc(100% - 88px - 1.5rem);
flex: 1;
vertical-align: middle;
margin-top: auto;
margin-bottom: auto;
}
.file-icon-wrapper {
width: calc(88px + .5rem);
height: 88px;
position: relative;
width: calc(88px + .5rem);
height: 88px;
position: relative;
}
</style>

142
sist2-vue/src/components/EmbeddingsSearchBar.vue
View File

@@ -1,29 +1,38 @@
<template>
<div>
<b-progress v-if="modelLoading" :value="modelLoadingProgress" max="1" class="mb-1" variant="warning"
<b-progress v-if="modelLoading && [0, 1].includes(modelLoadingProgress)" max="1" class="mb-1" variant="primary"
striped animated :value="1">
</b-progress>
<b-progress v-else-if="modelLoading" :value="modelLoadingProgress" max="1" class="mb-1" variant="warning"
show-progress>
</b-progress>
<b-input-group>
<b-form-input :value="embeddingText"
:placeholder="$t('embeddingsSearchPlaceholder')"
@input="onInput($event)"
:disabled="modelLoading"
></b-form-input>
<div style="display: flex">
<b-select :options="modelOptions()" class="mr-2 input-prepend" :value="modelName"
@change="onModelChange($event)"></b-select>
<b-input-group>
<b-form-input :value="embeddingText"
:placeholder="$store.state.embeddingDoc ? ' ' : $t('embeddingsSearchPlaceholder')"
@input="onInput($event)"
:disabled="modelLoading"
:style="{'pointer-events': $store.state.embeddingDoc ? 'none' : undefined}"
></b-form-input>
<b-badge v-if="$store.state.embeddingDoc" pill variant="primary" class="overlay-badge" href="#"
@click="onBadgeClick()">{{ docName }}
</b-badge>
<template #prepend>
</template>
<template #append>
<b-input-group-text>
<MLIcon class="ml-append" big></MLIcon>
</b-input-group-text>
</template>
</b-input-group>
</div>
<!-- TODO: dropdown of available models-->
<!-- <template #prepend>-->
<!-- <b-input-group-text>-->
<!-- <b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">-->
<!-- {{ $t("searchBar.fuzzy") }}-->
<!-- </b-form-checkbox>-->
<!-- </b-input-group-text>-->
<!-- </template>-->
<template #append>
<b-input-group-text>
<MLIcon></MLIcon>
</b-input-group-text>
</template>
</b-input-group>
</div>
</template>
@@ -32,6 +41,7 @@ import {mapGetters, mapMutations} from "vuex";
import {CLIPTransformerModel} from "@/ml/CLIPTransformerModel"
import _debounce from "lodash/debounce";
import MLIcon from "@/components/icons/MlIcon.vue";
import Sist2AdminApi from "@/Sist2Api";
export default {
components: {MLIcon},
@@ -40,7 +50,8 @@ export default {
modelLoading: false,
modelLoadingProgress: 0,
modelLoaded: false,
model: null
model: null,
modelName: null
}
},
computed: {
@@ -49,9 +60,18 @@ export default {
embeddingText: "embeddingText",
fuzzy: "fuzzy",
}),
docName() {
const ext = this.$store.state.embeddingDoc._source.extension;
return this.$store.state.embeddingDoc._source.name +
(ext ? "." + ext : "")
}
},
mounted() {
this.onInput = _debounce(this._onInput, 300, {leading: false});
// Set default model
this.modelName = Sist2AdminApi.models()[0].name;
this.onModelChange(this.modelName);
this.onInput = _debounce(this._onInput, 450, {leading: false});
},
methods: {
...mapMutations({
@@ -61,11 +81,6 @@ export default {
}),
async loadModel() {
this.modelLoading = true;
this.model = new CLIPTransformerModel(
// TODO: add a config for this (?)
"https://github.com/simon987/sist2-models/raw/main/clip/models/clip-vit-base-patch32-q8.onnx",
"https://github.com/simon987/sist2-models/raw/main/clip/models/tokenizer.json",
);
await this.model.init(async progress => {
this.modelLoadingProgress = progress;
@@ -74,26 +89,67 @@ export default {
this.modelLoaded = true;
},
async _onInput(text) {
if (!this.modelLoaded) {
await this.loadModel();
this.setEmbeddingModel(1); // TODO
try {
if (!this.modelLoaded) {
await this.loadModel();
}
if (text.length === 0) {
this.setEmbeddingText("");
this.setEmbedding(null);
return;
}
const embeddings = await this.model.predict(text);
this.setEmbeddingText(text);
this.setEmbedding(embeddings);
} catch (e) {
alert(e)
}
if (text.length === 0) {
this.setEmbeddingText("");
this.setEmbedding(null);
return;
}
const embeddings = await this.model.predict(text);
this.setEmbeddingText(text);
this.setEmbedding(embeddings);
},
mounted() {
modelOptions() {
return Sist2AdminApi.models().map(model => model.name);
},
onModelChange(name) {
this.modelLoaded = false;
this.modelLoadingProgress = 0;
const modelInfo = Sist2AdminApi.models().find(m => m.name === name);
if (modelInfo.name === "CLIP") {
const tokenizerUrl = new URL("./tokenizer.json", modelInfo.url).href;
this.model = new CLIPTransformerModel(modelInfo.url, tokenizerUrl)
this.setEmbeddingModel(modelInfo.id);
} else {
throw new Error("Unknown model: " + name);
}
},
onBadgeClick() {
this.$store.commit("setEmbedding", null);
this.$store.commit("setEmbeddingDoc", null);
}
}
}
</script>
<style>
.overlay-badge {
position: absolute;
z-index: 1;
left: 0.375rem;
top: 8px;
line-height: 1.1rem;
overflow: hidden;
max-width: 200px;
text-overflow: ellipsis;
}
.input-prepend {
max-width: 100px;
}
.theme-black .ml-append {
filter: brightness(0.95) !important;
}
</style>

262
sist2-vue/src/components/IndexPicker.vue
View File

@@ -1,42 +1,46 @@
<template>
<div v-if="isMobile">
<b-form-select
:value="selectedIndicesIds"
@change="onSelect($event)"
:options="indices" multiple :select-size="6" text-field="name"
value-field="id"></b-form-select>
</div>
<div v-else>
<div v-if="isMobile">
<b-form-select
:value="selectedIndicesIds"
@change="onSelect($event)"
:options="indices" multiple :select-size="6" text-field="name"
value-field="id"></b-form-select>
</div>
<div v-else>
<div class="d-flex justify-content-between align-content-center">
<div class="d-flex justify-content-between align-content-center">
<span>
{{ selectedIndices.length }}
{{ selectedIndices.length === 1 ? $t("indexPicker.selectedIndex") : $t("indexPicker.selectedIndices") }}
</span>
<div>
<b-button variant="link" @click="selectAll()"> {{ $t("indexPicker.selectAll") }}</b-button>
<b-button variant="link" @click="selectNone()"> {{ $t("indexPicker.selectNone") }}</b-button>
</div>
</div>
<b-list-group id="index-picker-desktop" class="unselectable">
<b-list-group-item
v-for="idx in indices"
@click="toggleIndex(idx, $event)"
@click.shift="shiftClick(idx, $event)"
class="d-flex justify-content-between align-items-center list-group-item-action pointer"
:class="{active: lastClickIndex === idx}"
>
<div class="d-flex">
<b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
{{ idx.name }}
<span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
<div>
<b-button variant="link" @click="selectAll()"> {{ $t("indexPicker.selectAll") }}</b-button>
<b-button variant="link" @click="selectNone()"> {{ $t("indexPicker.selectNone") }}</b-button>
</div>
</div>
<b-badge class="version-badge">v{{ idx.version }}</b-badge>
</b-list-group-item>
</b-list-group>
</div>
<b-list-group id="index-picker-desktop" class="unselectable">
<b-list-group-item
v-for="idx in indices"
@click="toggleIndex(idx, $event)"
@click.shift="shiftClick(idx, $event)"
class="d-flex justify-content-between align-items-center list-group-item-action pointer"
:class="{active: lastClickIndex === idx}"
>
<div class="d-flex">
<b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
{{ idx.name }}
<div style="vertical-align: center; margin-left: 5px">
<MLIcon small style="top: -1px; position: relative"></MLIcon>
</div>
<span class="text-muted timestamp-text ml-2"
style="top: 1px; position: relative">{{ formatIdxDate(idx.timestamp) }}</span>
</div>
<b-badge class="version-badge">v{{ idx.version }}</b-badge>
</b-list-group-item>
</b-list-group>
</div>
</template>
<script lang="ts">
@@ -44,148 +48,150 @@ import SmallBadge from "./SmallBadge.vue"
import {mapActions, mapGetters} from "vuex";
import Vue from "vue";
import {format} from "date-fns";
import MLIcon from "@/components/icons/MlIcon.vue";
export default Vue.extend({
components: {
SmallBadge
},
data() {
return {
loading: true,
lastClickIndex: null
}
},
computed: {
...mapGetters([
"indices", "selectedIndices"
]),
selectedIndicesIds() {
return this.selectedIndices.map(idx => idx.id)
components: {
MLIcon,
SmallBadge
},
isMobile() {
return window.innerWidth <= 650;
}
},
methods: {
...mapActions({
setSelectedIndices: "setSelectedIndices"
}),
shiftClick(index, e) {
if (this.lastClickIndex === null) {
return;
}
const select = this.isSelected(this.lastClickIndex);
let leftBoundary = this.indices.indexOf(this.lastClickIndex);
let rightBoundary = this.indices.indexOf(index);
if (rightBoundary < leftBoundary) {
let tmp = leftBoundary;
leftBoundary = rightBoundary;
rightBoundary = tmp;
}
for (let i = leftBoundary; i <= rightBoundary; i++) {
if (select) {
if (!this.isSelected(this.indices[i])) {
this.setSelectedIndices([this.indices[i], ...this.selectedIndices]);
}
} else {
this.setSelectedIndices(this.selectedIndices.filter(idx => idx !== this.indices[i]));
data() {
return {
loading: true,
lastClickIndex: null
}
}
},
selectAll() {
this.setSelectedIndices(this.indices);
computed: {
...mapGetters([
"indices", "selectedIndices"
]),
selectedIndicesIds() {
return this.selectedIndices.map(idx => idx.id)
},
isMobile() {
return window.innerWidth <= 650;
}
},
selectNone() {
this.setSelectedIndices([]);
},
onSelect(value) {
this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
},
formatIdxDate(timestamp: number): string {
return format(new Date(timestamp * 1000), "yyyy-MM-dd");
},
toggleIndex(index, e) {
if (e.shiftKey) {
return;
}
methods: {
...mapActions({
setSelectedIndices: "setSelectedIndices"
}),
shiftClick(index, e) {
if (this.lastClickIndex === null) {
return;
}
this.lastClickIndex = index;
if (this.isSelected(index)) {
this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
} else {
this.setSelectedIndices([index, ...this.selectedIndices]);
}
const select = this.isSelected(this.lastClickIndex);
let leftBoundary = this.indices.indexOf(this.lastClickIndex);
let rightBoundary = this.indices.indexOf(index);
if (rightBoundary < leftBoundary) {
let tmp = leftBoundary;
leftBoundary = rightBoundary;
rightBoundary = tmp;
}
for (let i = leftBoundary; i <= rightBoundary; i++) {
if (select) {
if (!this.isSelected(this.indices[i])) {
this.setSelectedIndices([this.indices[i], ...this.selectedIndices]);
}
} else {
this.setSelectedIndices(this.selectedIndices.filter(idx => idx !== this.indices[i]));
}
}
},
selectAll() {
this.setSelectedIndices(this.indices);
},
selectNone() {
this.setSelectedIndices([]);
},
onSelect(value) {
this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
},
formatIdxDate(timestamp: number): string {
return format(new Date(timestamp * 1000), "yyyy-MM-dd");
},
toggleIndex(index, e) {
if (e.shiftKey) {
return;
}
this.lastClickIndex = index;
if (this.isSelected(index)) {
this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
} else {
this.setSelectedIndices([index, ...this.selectedIndices]);
}
},
isSelected(index) {
return this.selectedIndices.find(idx => idx.id == index.id) != null;
}
},
isSelected(index) {
return this.selectedIndices.find(idx => idx.id == index.id) != null;
}
},
})
</script>
<style scoped>
.timestamp-text {
line-height: 24px;
font-size: 80%;
line-height: 24px;
font-size: 80%;
}
.theme-black .version-badge {
color: #eee !important;
background: none;
color: #eee !important;
background: none;
}
.version-badge {
color: #222 !important;
background: none;
color: #222 !important;
background: none;
}
.list-group-item {
padding: 0.2em 0.4em;
padding: 0.2em 0.4em;
}
#index-picker-desktop {
overflow-y: auto;
max-height: 132px;
overflow-y: auto;
max-height: 132px;
}
.btn-link:focus {
box-shadow: none;
box-shadow: none;
}
.unselectable {
user-select: none;
-ms-user-select: none;
-moz-user-select: none;
-webkit-user-select: none;
user-select: none;
-ms-user-select: none;
-moz-user-select: none;
-webkit-user-select: none;
}
.list-group-item.active {
z-index: 2;
background-color: inherit;
color: inherit;
z-index: 2;
background-color: inherit;
color: inherit;
}
.theme-black .list-group-item {
border: 1px solid rgba(255,255,255, 0.1);
border: 1px solid rgba(255, 255, 255, 0.1);
}
.theme-black .list-group-item:first-child {
border: 1px solid rgba(255,255,255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.05);
}
.theme-black .list-group-item.active {
z-index: 2;
background-color: inherit;
color: inherit;
border: 1px solid rgba(255,255,255, 0.3);
border-radius: 0;
z-index: 2;
background-color: inherit;
color: inherit;
border: 1px solid rgba(255, 255, 255, 0.3);
border-radius: 0;
}
.theme-black .list-group {
border-radius: 0;
border-radius: 0;
}
</style>

2
sist2-vue/src/components/SortSelect.vue
View File

@@ -1,5 +1,5 @@
<template>
<b-dropdown variant="primary" :disabled="$store.getters.embeddingText !== ''">
<b-dropdown variant="primary" :disabled="$store.getters.embedding !== null">
<b-dropdown-item :class="{'dropdown-active': sort === 'score'}" @click="onSelect('score')">{{
$t("sort.relevance")
}}

32
sist2-vue/src/components/icons/MlIcon.vue
View File

@@ -1,6 +1,6 @@
<template>
<svg height="20px" width="20px" xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 512 512" xml:space="preserve">
<svg class="ml-icon" :class="{'m-icon': 1, 'ml-icon-big': big, 'ml-icon-clickable': clickable}" xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 512 512" xml:space="preserve" fill="currentColor" stroke="currentColor" @click="$emit('click')">
<g>
<path class="st0" d="M167.314,14.993C167.314,6.712,160.602,0,152.332,0h-5.514c-8.27,0-14.982,6.712-14.982,14.993v41.466h35.478
V14.993z"/>
@@ -42,9 +42,35 @@
<script>
export default {
name: "MLIcon"
name: "MLIcon",
props: {
"big": Boolean,
"clickable": Boolean
}
}
</script>
<style scoped>
.ml-icon-clickable {
cursor: pointer;
}
.ml-icon-big {
width: 24px !important;
height: 24px !important;
}
.ml-icon {
width: 1rem;
min-width: 1rem;
margin-right: 0.2rem;
line-height: 1rem;
height: 1rem;
min-height: 1rem;
filter: brightness(45%);
}
.theme-black .ml-icon {
filter: brightness(80%);
}
</style>

1
sist2-vue/src/main.js
View File

@@ -1,4 +1,3 @@
import '@babel/polyfill'
import 'mutationobserver-shim'
import Vue from 'vue'
import './plugins/bootstrap-vue'

17
sist2-vue/src/ml/CLIPTransformerModel.js
View File

@@ -2,6 +2,7 @@ import * as ort from "onnxruntime-web";
import {BPETokenizer} from "@/ml/BPETokenizer";
import axios from "axios";
import {downloadToBuffer, ORT_WASM_PATHS} from "@/ml/mlUtils";
import ModelStore from "@/ml/ModelStore";
export class CLIPTransformerModel {
@@ -21,9 +22,17 @@ export class CLIPTransformerModel {
async loadModel(onProgress) {
ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
const buf = await downloadToBuffer(this._modelUrl, onProgress);
ort.env.wasm.numThreads = 2;
this._model = await ort.InferenceSession.create(buf.buffer, {executionProviders: ["wasm"]});
let buf = await ModelStore.get(this._modelUrl);
if (!buf) {
buf = await downloadToBuffer(this._modelUrl, onProgress);
await ModelStore.set(this._modelUrl, buf);
}
this._model = await ort.InferenceSession.create(buf.buffer, {
executionProviders: ["wasm"],
});
}
async loadTokenizer() {
@@ -34,11 +43,11 @@ export class CLIPTransformerModel {
async predict(text) {
const tokenized = this._tokenizer.encode(text);
const feeds = {
const inputs = {
input_ids: new ort.Tensor("int32", tokenized, [1, 77])
};
const results = await this._model.run(feeds);
const results = await this._model.run(inputs);
return Array.from(
Object.values(results)

67
sist2-vue/src/ml/ModelStore.js Normal file
View File

@@ -0,0 +1,67 @@
class ModelStore {
_ok;
_db;
_resolve;
_loadingPromise;
constructor() {
const request = window.indexedDB.open("ModelStore", 1);
request.onerror = () => {
this._ok = false;
}
request.onupgradeneeded = event => {
const db = event.target.result;
db.createObjectStore("models");
}
request.onsuccess = () => {
this._ok = true;
this._db = request.result;
this._resolve();
}
this._loadingPromise = new Promise(resolve => this._resolve = resolve);
}
async get(key) {
await this._loadingPromise;
const req = this._db.transaction(["models"], "readwrite")
.objectStore("models")
.get(key);
return new Promise(resolve => {
req.onsuccess = event => {
resolve(event.target.result);
};
req.onerror = event => {
console.log("ERROR:");
console.log(event);
resolve(null);
};
});
}
async set(key, val) {
await this._loadingPromise;
const req = this._db.transaction(["models"], "readwrite")
.objectStore("models")
.put(val, key);
return new Promise(resolve => {
req.onsuccess = () => {
resolve(true);
};
req.onerror = () => {
resolve(false);
};
});
}
}
export default new ModelStore();

1
sist2-vue/src/ml/mlUtils.js
View File

@@ -17,7 +17,6 @@ export async function downloadToBuffer(url, onProgress) {
break;
}
console.log(`Sending ${value.length} bytes into ${buf.length} at offset ${cursor} (${buf.length - cursor} free)`)
buf.set(value, cursor);
cursor += value.length;

4
sist2-vue/src/store/index.ts
View File

@@ -25,6 +25,7 @@ export default new Vuex.Store({
searchText: "",
embeddingText: "",
embedding: null,
embeddingDoc: null,
pathText: "",
sortMode: "score",
@@ -133,7 +134,8 @@ export default new Vuex.Store({
setDateBoundsMax: (state, val) => state.dateBoundsMax = val,
setSearchText: (state, val) => state.searchText = val,
setEmbeddingText: (state, val) => state.embeddingText = val,
setEmbedding: (state, val) => state.embedding= val,
setEmbedding: (state, val) => state.embedding = val,
setEmbeddingDoc: (state, val) => state.embeddingDoc = val,
setFuzzy: (state, val) => state.fuzzy = val,
setLastQueryResult: (state, val) => state.lastQueryResults = val,
setFirstQueryResult: (state, val) => state.firstQueryResults = val,

9
sist2-vue/src/views/SearchPage.vue
View File

@@ -13,7 +13,7 @@
<b-card v-show="!uiLoading && !showEsConnectionError" id="search-panel">
<SearchBar @show-help="showHelp=true"></SearchBar>
<EmbeddingsSearchBar class="mt-3"></EmbeddingsSearchBar>
<EmbeddingsSearchBar v-if="hasEmbeddings" class="mt-3"></EmbeddingsSearchBar>
<b-row>
<b-col style="height: 70px;" sm="6">
<SizeSlider></SizeSlider>
@@ -172,6 +172,12 @@ export default Vue.extend({
setDateBoundsMax: "setDateBoundsMax",
setTags: "setTags",
}),
hasEmbeddings() {
if (!this.loading) {
return false;
}
return Sist2Api.models().some();
},
showErrorToast() {
this.$bvToast.toast(
this.$t("toast.esConnErr"),
@@ -203,6 +209,7 @@ export default Vue.extend({
await this.handleSearch(resp);
this.searchBusy = false;
}).catch(err => {
console.log(err)
if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
this.showSyntaxErrorToast();
} else {

16
sist2-vue/vue.config.js
View File

@@ -1,3 +1,5 @@
const TerserPlugin = require("terser-webpack-plugin");
module.exports = {
filenameHashing: false,
productionSourceMap: false,
@@ -6,5 +8,19 @@ module.exports = {
index: {
entry: "src/main.js"
}
},
configureWebpack: config => {
config.optimization.minimizer = [new TerserPlugin({
terserOptions: {
compress: {
passes: 2,
module: true,
hoist_funs: true,
// https://github.com/microsoft/onnxruntime/issues/16984
unused: false,
},
mangle: true,
}
})]
}
}