simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-19 02:06:46 +00:00
Code Issues Projects Releases Wiki Activity

Add NER support

Browse Source
This commit is contained in:
simon987 2023-04-23 12:53:27 -04:00
parent b5cdd9a5df
commit dc39c0ec4b
15 changed files with 1826 additions and 742 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
README.md
View File

@ -24,10 +24,12 @@ sist2 (Simple incremental search tool)
* Recursive scan inside archive files \*\* * Recursive scan inside archive files \*\*
* OCR support with tesseract \*\*\* * OCR support with tesseract \*\*\*
* Stats page & disk utilisation visualization * Stats page & disk utilisation visualization
* Named-entity recognition (client-side) \*\*\*\*
\* See [format support](#format-support) \* See [format support](#format-support)
\*\* See [Archive files](#archive-files) \*\* See [Archive files](#archive-files)
\*\*\* See [OCR](#ocr) \*\*\* See [OCR](#ocr)
\*\*\*\* See [Named-Entity Recognition](#NER)
## Getting Started ## Getting Started
@ -68,10 +70,9 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
``` ```
2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases). 2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`. Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
3. See [usage guide](docs/USAGE.md) for command line usage. 3. See [usage guide](docs/USAGE.md) for command line usage.
Example usage: Example usage:
1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2` 1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
@ -135,6 +136,29 @@ sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/ sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
``` ```
### NER
sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to
**Configuration** > **Machine learning options** > **Model repositories**
to enable it.
The text processing is done in your browser, no data is sent to any third-party services.
See [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) for more details.
#### List of available repositories:
| URL | Maintainer | Purpose |
|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
<details>
<summary>Screenshot</summary>
![ner](docs/ner.png)
</details>
## Build from source ## Build from source
You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries

BIN
docs/ner.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 448 KiB

720
sist2-vue/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
sist2-vue/package.json
View File

@ -9,6 +9,7 @@
"dependencies": { "dependencies": {
"@auth0/auth0-spa-js": "^2.0.2", "@auth0/auth0-spa-js": "^2.0.2",
"@egjs/vue-infinitegrid": "3.3.0", "@egjs/vue-infinitegrid": "3.3.0",
"@tensorflow/tfjs": "^4.4.0",
"axios": "^0.25.0", "axios": "^0.25.0",
"bootstrap-vue": "^2.21.2", "bootstrap-vue": "^2.21.2",
"core-js": "^3.6.5", "core-js": "^3.6.5",

424
sist2-vue/src/App.vue
View File

@ -1,383 +1,395 @@
<template> <template>
<div id="app" :class="getClass()" v-if="!authLoading"> <div id="app" :class="getClass()" v-if="!authLoading">
<NavBar></NavBar> <NavBar></NavBar>
<router-view v-if="!configLoading"/> <router-view v-if="!configLoading"/>
</div>
<div class="loading-page" v-else>
<div class="loading-spinners">
<b-spinner type="grow" variant="primary"></b-spinner>
<b-spinner type="grow" variant="primary"></b-spinner>
<b-spinner type="grow" variant="primary"></b-spinner>
</div> </div>
<div class="loading-text"> <div class="loading-page" v-else>
Loading • Chargement • 装载 • Wird geladen <div class="loading-spinners">
<b-spinner type="grow" variant="primary"></b-spinner>
<b-spinner type="grow" variant="primary"></b-spinner>
<b-spinner type="grow" variant="primary"></b-spinner>
</div>
<div class="loading-text">
Loading • Chargement • 装载 • Wird geladen
</div>
</div> </div>
</div>
</template> </template>
<script> <script>
import NavBar from "@/components/NavBar"; import NavBar from "@/components/NavBar";
import {mapActions, mapGetters, mapMutations} from "vuex"; import {mapActions, mapGetters, mapMutations} from "vuex";
import Sist2Api from "@/Sist2Api"; import Sist2Api from "@/Sist2Api";
import ModelsRepo from "@/ml/modelsRepo";
import {setupAuth0} from "@/main"; import {setupAuth0} from "@/main";
export default { export default {
components: {NavBar}, components: {NavBar},
data() { data() {
return { return {
configLoading: false, configLoading: false,
authLoading: true, authLoading: true,
sist2InfoLoading: true sist2InfoLoading: true
} }
}, },
computed: { computed: {
...mapGetters(["optTheme"]), ...mapGetters(["optTheme"]),
}, },
mounted() { mounted() {
this.$store.dispatch("loadConfiguration").then(() => { this.$store.dispatch("loadConfiguration").then(() => {
this.$root.$i18n.locale = this.$store.state.optLang; this.$root.$i18n.locale = this.$store.state.optLang;
}); ModelsRepo.init(this.$store.getters.mlRepositoryList).catch(err => {
this.$bvToast.toast(
this.$t("ml.repoFetchError"),
{
title: this.$t("ml.repoFetchErrorTitle"),
noAutoHide: true,
toaster: "b-toaster-bottom-right",
headerClass: "toast-header-warning",
bodyClass: "toast-body-warning",
});
});
});
this.$store.subscribe((mutation) => { this.$store.subscribe((mutation) => {
if (mutation.type === "setOptLang") { if (mutation.type === "setOptLang") {
this.$root.$i18n.locale = mutation.payload; this.$root.$i18n.locale = mutation.payload;
this.configLoading = true; this.configLoading = true;
window.setTimeout(() => this.configLoading = false, 10); window.setTimeout(() => this.configLoading = false, 10);
}
if (mutation.type === "setAuth0Token") {
this.authLoading = false;
}
});
Sist2Api.getSist2Info().then(data => {
if (data.auth0Enabled) {
this.authLoading = true;
setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
this.$auth.$watch("loading", loading => {
if (loading === false) {
if (!this.$auth.isAuthenticated) {
this.$auth.loginWithRedirect();
return;
} }
// Remove "code" param if (mutation.type === "setAuth0Token") {
window.history.replaceState({}, "", "/" + window.location.hash); this.authLoading = false;
}
this.$store.dispatch("loadAuth0Token");
}
}); });
} else {
this.authLoading = false;
}
this.setSist2Info(data); Sist2Api.getSist2Info().then(data => {
this.setIndices(data.indices)
}); if (data.auth0Enabled) {
}, this.authLoading = true;
methods: { setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
...mapActions(["setSist2Info",]),
...mapMutations(["setIndices",]), this.$auth.$watch("loading", loading => {
getClass() { if (loading === false) {
return {
"theme-light": this.optTheme === "light", if (!this.$auth.isAuthenticated) {
"theme-black": this.optTheme === "black", this.$auth.loginWithRedirect();
} return;
}
// Remove "code" param
window.history.replaceState({}, "", "/" + window.location.hash);
this.$store.dispatch("loadAuth0Token");
}
});
} else {
this.authLoading = false;
}
this.setSist2Info(data);
this.setIndices(data.indices)
});
},
methods: {
...mapActions(["setSist2Info",]),
...mapMutations(["setIndices",]),
getClass() {
return {
"theme-light": this.optTheme === "light",
"theme-black": this.optTheme === "black",
}
}
} }
} ,
,
} }
</script> </script>
<style> <style>
html, body { html, body {
height: 100%; height: 100%;
} }
#app { #app {
/*font-family: Avenir, Helvetica, Arial, sans-serif;*/ /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-webkit-font-smoothing: antialiased; -webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale; -moz-osx-font-smoothing: grayscale;
/*text-align: center;*/ /*text-align: center;*/
color: #2c3e50; color: #2c3e50;
padding-bottom: 1em; padding-bottom: 1em;
min-height: 100%; min-height: 100%;
} }
/*Black theme*/ /*Black theme*/
.theme-black { .theme-black {
background-color: #000; background-color: #000;
} }
.theme-black .card, .theme-black .modal-content { .theme-black .card, .theme-black .modal-content {
background: #212121; background: #212121;
color: #e0e0e0; color: #e0e0e0;
border-radius: 1px; border-radius: 1px;
border: none; border: none;
} }
.theme-black .table { .theme-black .table {
color: #e0e0e0; color: #e0e0e0;
} }
.theme-black .table td, .theme-black .table th { .theme-black .table td, .theme-black .table th {
border: none; border: none;
} }
.theme-black .table thead th { .theme-black .table thead th {
border-bottom: 1px solid #646464; border-bottom: 1px solid #646464;
} }
.theme-black .custom-select { .theme-black .custom-select {
overflow: auto; overflow: auto;
background-color: #37474F; background-color: #37474F;
border: 1px solid #616161; border: 1px solid #616161;
color: #bdbdbd; color: #bdbdbd;
} }
.theme-black .custom-select:focus { .theme-black .custom-select:focus {
border-color: #757575; border-color: #757575;
outline: 0; outline: 0;
box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25); box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
} }
.theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow { .theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow {
background: none !important; background: none !important;
} }
.theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before { .theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before {
background-color: black !important; background-color: black !important;
} }
.theme-black .inspire-tree .title { .theme-black .inspire-tree .title {
color: #eee; color: #eee;
} }
.theme-black .inspire-tree { .theme-black .inspire-tree {
font-weight: 400; font-weight: 400;
font-size: 14px; font-size: 14px;
font-family: Helvetica, Nueue, Verdana, sans-serif; font-family: Helvetica, Nueue, Verdana, sans-serif;
max-height: 350px; max-height: 350px;
overflow: auto; overflow: auto;
} }
.inspire-tree [type=checkbox] { .inspire-tree [type=checkbox] {
left: 22px !important; left: 22px !important;
top: 7px !important; top: 7px !important;
} }
.theme-black .form-control { .theme-black .form-control {
background-color: #37474F; background-color: #37474F;
border: 1px solid #616161; border: 1px solid #616161;
color: #dbdbdb !important; color: #dbdbdb !important;
} }
.theme-black .form-control:focus { .theme-black .form-control:focus {
background-color: #546E7A; background-color: #546E7A;
color: #fff; color: #fff;
} }
.theme-black .input-group-text, .theme-black .default-input { .theme-black .input-group-text, .theme-black .default-input {
background: #37474F !important; background: #37474F !important;
border: 1px solid #616161 !important; border: 1px solid #616161 !important;
color: #dbdbdb !important; color: #dbdbdb !important;
} }
.theme-black ::placeholder { .theme-black ::placeholder {
color: #BDBDBD !important; color: #BDBDBD !important;
opacity: 1; opacity: 1;
} }
.theme-black .nav-tabs .nav-link { .theme-black .nav-tabs .nav-link {
color: #e0e0e0; color: #e0e0e0;
border-radius: 0; border-radius: 0;
} }
.theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active { .theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
background-color: #212121; background-color: #212121;
border-color: #616161 #616161 #212121; border-color: #616161 #616161 #212121;
color: #e0e0e0; color: #e0e0e0;
} }
.theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus { .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus {
border-color: #616161 #616161 #212121; border-color: #616161 #616161 #212121;
color: #e0e0e0; color: #e0e0e0;
} }
.theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover { .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover {
border-color: #e0e0e0 #e0e0e0 #212121; border-color: #e0e0e0 #e0e0e0 #212121;
color: #e0e0e0; color: #e0e0e0;
} }
.theme-black .nav-tabs { .theme-black .nav-tabs {
border-bottom: #616161; border-bottom: #616161;
} }
.theme-black a:hover, .theme-black .btn:hover { .theme-black a:hover, .theme-black .btn:hover {
color: #fff; color: #fff;
} }
.theme-black .b-dropdown a:hover { .theme-black .b-dropdown a:hover {
color: inherit; color: inherit;
} }
.theme-black .btn { .theme-black .btn {
color: #eee; color: #eee;
} }
.theme-black .modal-header .close { .theme-black .modal-header .close {
color: #e0e0e0; color: #e0e0e0;
text-shadow: none; text-shadow: none;
} }
.theme-black .modal-header { .theme-black .modal-header {
border-bottom: 1px solid #646464; border-bottom: 1px solid #646464;
} }
/* -------------------------- */ /* -------------------------- */
#nav { #nav {
padding: 30px; padding: 30px;
} }
#nav a { #nav a {
font-weight: bold; font-weight: bold;
color: #2c3e50; color: #2c3e50;
} }
#nav a.router-link-exact-active { #nav a.router-link-exact-active {
color: #42b983; color: #42b983;
} }
.mobile { .mobile {
display: none; display: none;
} }
.container { .container {
padding-top: 1em; padding-top: 1em;
} }
@media (max-width: 650px) { @media (max-width: 650px) {
.mobile { .mobile {
display: initial; display: initial;
} }
.not-mobile { .not-mobile {
display: none; display: none;
} }
.grid-single-column .fit { .grid-single-column .fit {
max-height: none !important; max-height: none !important;
} }
.container { .container {
padding-left: 0; padding-left: 0;
padding-right: 0; padding-right: 0;
padding-top: 0 padding-top: 0
} }
.lightbox-caption { .lightbox-caption {
display: none; display: none;
} }
} }
.info-icon { .info-icon {
width: 1rem; width: 1rem;
margin-right: 0.2rem; margin-right: 0.2rem;
cursor: pointer; cursor: pointer;
line-height: 1rem; line-height: 1rem;
height: 1rem; height: 1rem;
background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==); background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
filter: brightness(45%); filter: brightness(45%);
display: block; display: block;
} }
.tabs { .tabs {
margin-top: 10px; margin-top: 10px;
} }
.modal-title { .modal-title {
text-overflow: ellipsis; text-overflow: ellipsis;
overflow: hidden; overflow: hidden;
white-space: nowrap; white-space: nowrap;
} }
@media screen and (min-width: 1500px) { @media screen and (min-width: 1500px) {
.container { .container {
max-width: 1440px; max-width: 1440px;
} }
} }
.noUi-connects { .noUi-connects {
border-radius: 1px !important; border-radius: 1px !important;
} }
mark { mark {
background: #fff217; background: #fff217;
border-radius: 0; border-radius: 0;
padding: 1px 0; padding: 1px 0;
color: inherit; color: inherit;
} }
.theme-black mark { .theme-black mark {
background: rgba(251, 191, 41, 0.25); background: rgba(251, 191, 41, 0.25);
border-radius: 0; border-radius: 0;
padding: 1px 0; padding: 1px 0;
color: inherit; color: inherit;
} }
.theme-black .content-div mark { .theme-black .content-div mark {
background: rgba(251, 191, 41, 0.40); background: rgba(251, 191, 41, 0.40);
color: white; color: white;
} }
.content-div { .content-div {
font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
font-size: 13px; font-size: 13px;
padding: 1em; padding: 1em;
background-color: #f5f5f5; background-color: #f5f5f5;
border: 1px solid #ccc; border: 1px solid #ccc;
border-radius: 4px; border-radius: 4px;
margin: 3px; margin: 3px;
white-space: normal; white-space: normal;
color: #000; color: #000;
overflow: hidden; overflow: hidden;
} }
.theme-black .content-div { .theme-black .content-div {
background-color: #37474F; background-color: #37474F;
border: 1px solid #616161; border: 1px solid #616161;
color: #E0E0E0FF; color: #E0E0E0FF;
} }
.graph { .graph {
display: inline-block; display: inline-block;
width: 40%; width: 40%;
} }
.pointer { .pointer {
cursor: pointer; cursor: pointer;
} }
.loading-page { .loading-page {
display: flex; display: flex;
justify-content: center; justify-content: center;
align-items: center; align-items: center;
flex-direction: column; flex-direction: column;
height: 100%; height: 100%;
gap: 15px gap: 15px
} }
.loading-spinners { .loading-spinners {
display: flex; display: flex;
gap: 10px; gap: 10px;
} }
.loading-text { .loading-text {
text-align: center; text-align: center;
} }
</style> </style>

21
sist2-vue/src/components/AnalyzedContentSpan.vue Normal file
View File

@ -0,0 +1,21 @@
<template>
<span :style="getStyle()">{{span.text}}</span>
</template>
<script>
import ModelsRepo from "@/ml/modelsRepo";
export default {
name: "AnalyzedContentSpan",
props: ["span", "text"],
methods: {
getStyle() {
return ModelsRepo.data[this.$store.getters.mlModel.name].labelStyles[this.span.label];
}
}
}
</script>
<style scoped></style>

75
sist2-vue/src/components/AnalyzedContentSpanContainer.vue Normal file
View File

@ -0,0 +1,75 @@
<template>
<div>
<b-card class="mb-2">
<AnalyzedContentSpan v-for="span of legend" :key="span.id" :span="span"
class="mr-2"></AnalyzedContentSpan>
</b-card>
<div class="content-div">
<AnalyzedContentSpan v-for="span of mergedSpans" :key="span.id" :span="span"></AnalyzedContentSpan>
</div>
</div>
</template>
<script>
import AnalyzedContentSpan from "@/components/AnalyzedContentSpan.vue";
import ModelsRepo from "@/ml/modelsRepo";
export default {
name: "AnalyzedContentSpanContainer",
components: {AnalyzedContentSpan},
props: ["spans", "text"],
computed: {
legend() {
return Object.entries(ModelsRepo.data[this.$store.state.mlModel.name].legend)
.map(([label, name]) => ({
text: name,
id: label,
label: label
}));
},
mergedSpans() {
const spans = this.spans;
const merged = [];
let lastLabel = null;
let fixSpace = false;
for (let i = 0; i < spans.length; i++) {
if (spans[i].label !== lastLabel) {
let start = spans[i].wordIndex;
const nextSpan = spans.slice(i + 1).find(s => s.label !== spans[i].label)
let end = nextSpan ? nextSpan.wordIndex : undefined;
if (end !== undefined && this.text[end - 1] === " ") {
end -= 1;
fixSpace = true;
}
merged.push({
text: this.text.slice(start, end),
label: spans[i].label,
id: spans[i].wordIndex
});
if (fixSpace) {
merged.push({
text: " ",
label: "O",
id: end
});
fixSpace = false;
}
lastLabel = spans[i].label;
}
}
return merged;
},
},
}
</script>
<style scoped></style>

270
sist2-vue/src/components/LazyContentDiv.vue
View File

@ -1,6 +1,36 @@
<template> <template>
<Preloader v-if="loading"></Preloader> <Preloader v-if="loading"></Preloader>
<div v-else-if="content" class="content-div" v-html="content"></div> <div v-else-if="content">
<b-form inline class="my-2" v-if="ModelsRepo.getOptions().length > 0">
<b-checkbox class="ml-auto mr-2" :checked="optAutoAnalyze"
@input="setOptAutoAnalyze($event); $store.dispatch('updateConfiguration')">
{{ $t("ml.auto") }}
</b-checkbox>
<b-button :disabled="mlPredictionsLoading || mlLoading" @click="mlAnalyze" variant="primary"
>{{ $t("ml.analyzeText") }}
</b-button>
<b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="mlModel">
<b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
</b-select-option>
</b-select>
</b-form>
<b-progress v-if="mlLoading" variant="warning" show-progress :max="1" class="mb-3"
>
<b-progress-bar :value="modelLoadingProgress">
<strong>{{ ((modelLoadingProgress * modelSize) / (1024*1024)).toFixed(1) }}MB / {{
(modelSize / (1024 * 1024)).toFixed(1)
}}MB</strong>
</b-progress-bar>
</b-progress>
<b-progress v-if="mlPredictionsLoading" variant="primary" :value="modelPredictionProgress"
:max="content.length" class="mb-3"></b-progress>
<AnalyzedContentSpansContainer v-if="analyzedContentSpans.length > 0"
:spans="analyzedContentSpans" :text="rawContent"></AnalyzedContentSpansContainer>
<div v-else class="content-div" v-html="content"></div>
</div>
</template> </template>
<script> <script>
@ -8,87 +38,169 @@ import Sist2Api from "@/Sist2Api";
import Preloader from "@/components/Preloader"; import Preloader from "@/components/Preloader";
import Sist2Query from "@/Sist2Query"; import Sist2Query from "@/Sist2Query";
import store from "@/store"; import store from "@/store";
import BertNerModel from "@/ml/BertNerModel";
import AnalyzedContentSpansContainer from "@/components/AnalyzedContentSpanContainer.vue";
import ModelsRepo from "@/ml/modelsRepo";
import {mapGetters, mapMutations} from "vuex";
export default { export default {
name: "LazyContentDiv", name: "LazyContentDiv",
components: {Preloader}, components: {AnalyzedContentSpansContainer, Preloader},
props: ["docId"], props: ["docId"],
data() { data() {
return { return {
content: "", ModelsRepo,
loading: true content: "",
rawContent: "",
loading: true,
modelLoadingProgress: 0,
modelPredictionProgress: 0,
mlPredictionsLoading: false,
mlLoading: false,
mlModel: null,
analyzedContentSpans: []
}
},
mounted() {
if (this.$store.getters.optMlDefaultModel) {
this.mlModel = this.$store.getters.optMlDefaultModel
} else {
this.mlModel = ModelsRepo.getDefaultModel();
}
const query = Sist2Query.searchQuery();
if (this.$store.state.optHighlight) {
const fields = this.$store.state.fuzzy
? {"content.nGram": {}}
: {content: {}};
query.highlight = {
pre_tags: ["<mark>"],
post_tags: ["</mark>"],
number_of_fragments: 0,
fields,
};
if (!store.state.sist2Info.esVersionLegacy) {
query.highlight.max_analyzed_offset = 999_999;
}
}
if ("function_score" in query.query) {
query.query = query.query.function_score.query;
}
if (!("must" in query.query.bool)) {
query.query.bool.must = [];
} else if (!Array.isArray(query.query.bool.must)) {
query.query.bool.must = [query.query.bool.must];
}
query.query.bool.must.push({match: {_id: this.docId}});
delete query["sort"];
delete query["aggs"];
delete query["search_after"];
delete query.query["function_score"];
query._source = {
includes: ["content", "name", "path", "extension"]
}
query.size = 1;
Sist2Api.esQuery(query).then(resp => {
this.loading = false;
if (resp.hits.hits.length === 1) {
this.content = this.getContent(resp.hits.hits[0]);
}
if (this.optAutoAnalyze) {
this.mlAnalyze();
}
});
},
computed: {
...mapGetters(["optAutoAnalyze"]),
modelSize() {
const modelData = ModelsRepo.data[this.mlModel];
if (!modelData) {
return 0;
}
return modelData.size;
}
},
methods: {
...mapMutations(["setOptAutoAnalyze"]),
getContent(doc) {
this.rawContent = doc._source.content;
if (!doc.highlight) {
return doc._source.content;
}
if (doc.highlight["content.nGram"]) {
return doc.highlight["content.nGram"][0];
}
if (doc.highlight.content) {
return doc.highlight.content[0];
}
},
async getMlModel() {
if (this.$store.getters.mlModel.name !== this.mlModel) {
this.mlLoading = true;
this.modelLoadingProgress = 0;
const modelInfo = ModelsRepo.data[this.mlModel];
const model = new BertNerModel(
modelInfo.vocabUrl,
modelInfo.modelUrl,
modelInfo.id2label,
)
await model.init(progress => this.modelLoadingProgress = progress);
this.$store.commit("setMlModel", {model, name: this.mlModel});
this.mlLoading = false;
return model
}
return this.$store.getters.mlModel.model;
},
async mlAnalyze() {
if (!this.content) {
return;
}
const modelInfo = ModelsRepo.data[this.mlModel];
if (modelInfo === undefined) {
return;
}
this.$store.commit("setOptMlDefaultModel", this.mlModel);
await this.$store.dispatch("updateConfiguration");
const model = await this.getMlModel();
this.analyzedContentSpans = [];
this.mlPredictionsLoading = true;
await model.predict(this.rawContent, results => {
results.forEach(result => result.label = modelInfo.humanLabels[result.label]);
this.analyzedContentSpans.push(...results);
this.modelPredictionProgress = results[results.length - 1].wordIndex;
});
this.mlPredictionsLoading = false;
}
} }
},
mounted() {
const query = Sist2Query.searchQuery();
if (this.$store.state.optHighlight) {
const fields = this.$store.state.fuzzy
? {"content.nGram": {}}
: {content: {}};
query.highlight = {
pre_tags: ["<mark>"],
post_tags: ["</mark>"],
number_of_fragments: 0,
fields,
};
if (!store.state.sist2Info.esVersionLegacy) {
query.highlight.max_analyzed_offset = 999_999;
}
}
if ("function_score" in query.query) {
query.query = query.query.function_score.query;
}
if (!("must" in query.query.bool)) {
query.query.bool.must = [];
} else if (!Array.isArray(query.query.bool.must)) {
query.query.bool.must = [query.query.bool.must];
}
query.query.bool.must.push({match: {_id: this.docId}});
delete query["sort"];
delete query["aggs"];
delete query["search_after"];
delete query.query["function_score"];
query._source = {
includes: ["content", "name", "path", "extension"]
}
query.size = 1;
Sist2Api.esQuery(query).then(resp => {
this.loading = false;
if (resp.hits.hits.length === 1) {
this.content = this.getContent(resp.hits.hits[0]);
} else {
console.log("FIXME: could not get content")
console.log(resp)
}
});
},
methods: {
getContent(doc) {
if (!doc.highlight) {
return doc._source.content;
}
if (doc.highlight["content.nGram"]) {
return doc.highlight["content.nGram"][0];
}
if (doc.highlight.content) {
return doc.highlight.content[0];
}
}
}
} }
</script> </script>
<style scoped> <style>
.progress-bar {
transition: none;
}
</style> </style>

12
sist2-vue/src/i18n/messages.ts
View File

@ -49,6 +49,7 @@ export default {
configReset: "Reset configuration", configReset: "Reset configuration",
searchOptions: "Search options", searchOptions: "Search options",
treemapOptions: "Treemap options", treemapOptions: "Treemap options",
mlOptions: "Machine learning options",
displayOptions: "Display options", displayOptions: "Display options",
opt: { opt: {
lang: "Language", lang: "Language",
@ -78,7 +79,10 @@ export default {
simpleLightbox: "Disable animations in image viewer", simpleLightbox: "Disable animations in image viewer",
showTagPickerFilter: "Display the tag filter bar", showTagPickerFilter: "Display the tag filter bar",
featuredFields: "Featured fields Javascript template string. Will appear in the search results.", featuredFields: "Featured fields Javascript template string. Will appear in the search results.",
featuredFieldsList: "Available variables" featuredFieldsList: "Available variables",
autoAnalyze: "Automatically analyze text",
defaultModel: "Default model",
mlRepositories: "Model repositories (one per line)"
}, },
queryMode: { queryMode: {
simple: "Simple", simple: "Simple",
@ -171,6 +175,12 @@ export default {
selectedIndex: "selected index", selectedIndex: "selected index",
selectedIndices: "selected indices", selectedIndices: "selected indices",
}, },
ml: {
analyzeText: "Analyze",
auto: "Auto",
repoFetchError: "Failed to get list of models. Check browser console for more details.",
repoFetchErrorTitle: "Could not fetch model repositories",
}
}, },
de: { de: {
filePage: { filePage: {

77
sist2-vue/src/ml/BertNerModel.js Normal file
View File

@ -0,0 +1,77 @@
import BertTokenizer from "@/ml/BertTokenizer";
import * as tf from "@tensorflow/tfjs";
import axios from "axios";
export default class BertNerModel {
vocabUrl;
modelUrl;
id2label;
_tokenizer;
_model;
inputSize = 128;
_previousWordId = null;
constructor(vocabUrl, modelUrl, id2label) {
this.vocabUrl = vocabUrl;
this.modelUrl = modelUrl;
this.id2label = id2label;
}
async init(onProgress) {
await Promise.all([this.loadTokenizer(), this.loadModel(onProgress)]);
}
async loadTokenizer() {
const vocab = (await axios.get(this.vocabUrl)).data;
this._tokenizer = new BertTokenizer(vocab);
}
async loadModel(onProgress) {
this._model = await tf.loadGraphModel(this.modelUrl, {onProgress});
}
alignLabels(labels, wordIds, words) {
const result = [];
for (let i = 0; i < this.inputSize; i++) {
const label = labels[i];
const wordId = wordIds[i];
if (wordId === -1) {
continue;
}
if (wordId === this._previousWordId) {
continue;
}
result.push({
word: words[wordId].text, wordIndex: words[wordId].index, label: label
});
this._previousWordId = wordId;
}
return result;
}
async predict(text, callback) {
this._previousWordId = null;
const encoded = this._tokenizer.encodeText(text, this.inputSize)
for (let chunk of encoded.inputChunks) {
const rawResult = tf.tidy(() => this._model.execute({
input_ids: tf.tensor2d(chunk.inputIds, [1, this.inputSize], "int32"),
token_type_ids: tf.tensor2d(chunk.segmentIds, [1, this.inputSize], "int32"),
attention_mask: tf.tensor2d(chunk.inputMask, [1, this.inputSize], "int32"),
}));
const labelIds = await tf.argMax(rawResult, -1);
const labelIdsArray = await labelIds.array();
const labels = labelIdsArray[0].map(id => this.id2label[id]);
rawResult.dispose()
callback(this.alignLabels(labels, chunk.wordIds, encoded.words))
}
}
}

184
sist2-vue/src/ml/BertTokenizer.js Normal file
View File

@ -0,0 +1,184 @@
import {zip, chunk} from "underscore";
const UNK_INDEX = 100;
const CLS_INDEX = 101;
const SEP_INDEX = 102;
const CONTINUING_SUBWORD_PREFIX = "##";
function isWhitespace(ch) {
return /\s/.test(ch);
}
function isInvalid(ch) {
return (ch.charCodeAt(0) === 0 || ch.charCodeAt(0) === 0xfffd);
}
const punctuations = '[~`!@#$%^&*(){}[];:"\'<,.>?/\\|-_+=';
/** To judge whether it's a punctuation. */
function isPunctuation(ch) {
return punctuations.indexOf(ch) !== -1;
}
export default class BertTokenizer {
vocab;
constructor(vocab) {
this.vocab = vocab;
}
tokenize(text) {
const charOriginalIndex = [];
const cleanedText = this.cleanText(text, charOriginalIndex);
const origTokens = cleanedText.split(' ');
let charCount = 0;
const tokens = origTokens.map((token) => {
token = token.toLowerCase();
const tokens = this.runSplitOnPunctuation(token, charCount, charOriginalIndex);
charCount += token.length + 1;
return tokens;
});
let flattenTokens = [];
for (let index = 0; index < tokens.length; index++) {
flattenTokens = flattenTokens.concat(tokens[index]);
}
return flattenTokens;
}
/* Performs invalid character removal and whitespace cleanup on text. */
cleanText(text, charOriginalIndex) {
text = text.replace(/\?/g, "").trim();
const stringBuilder = [];
let originalCharIndex = 0;
let newCharIndex = 0;
for (const ch of text) {
// Skip the characters that cannot be used.
if (isInvalid(ch)) {
originalCharIndex += ch.length;
continue;
}
if (isWhitespace(ch)) {
if (stringBuilder.length > 0 && stringBuilder[stringBuilder.length - 1] !== ' ') {
stringBuilder.push(' ');
charOriginalIndex[newCharIndex] = originalCharIndex;
originalCharIndex += ch.length;
} else {
originalCharIndex += ch.length;
continue;
}
} else {
stringBuilder.push(ch);
charOriginalIndex[newCharIndex] = originalCharIndex;
originalCharIndex += ch.length;
}
newCharIndex++;
}
return stringBuilder.join('');
}
/* Splits punctuation on a piece of text. */
runSplitOnPunctuation(text, count, charOriginalIndex) {
const tokens = [];
let startNewWord = true;
for (const ch of text) {
if (isPunctuation(ch)) {
tokens.push({text: ch, index: charOriginalIndex[count]});
count += ch.length;
startNewWord = true;
} else {
if (startNewWord) {
tokens.push({text: '', index: charOriginalIndex[count]});
startNewWord = false;
}
tokens[tokens.length - 1].text += ch;
count += ch.length;
}
}
return tokens;
}
encode(words) {
let outputTokens = [];
const wordIds = [];
for (let i = 0; i < words.length; i++) {
let chars = [...words[i].text];
let isUnknown = false;
let start = 0;
let subTokens = [];
while (start < chars.length) {
let end = chars.length;
let currentSubstring = null;
while (start < end) {
let substr = chars.slice(start, end).join('');
if (start > 0) {
substr = CONTINUING_SUBWORD_PREFIX + substr;
}
if (this.vocab.includes(substr)) {
currentSubstring = this.vocab.indexOf(substr);
break;
}
--end;
}
if (currentSubstring == null) {
isUnknown = true;
break;
}
subTokens.push(currentSubstring);
start = end;
}
if (isUnknown) {
outputTokens.push(UNK_INDEX);
wordIds.push(i);
} else {
subTokens.forEach(tok => {
outputTokens.push(tok);
wordIds.push(i)
});
}
}
return {tokens: outputTokens, wordIds};
}
encodeText(inputText, inputSize) {
const tokenized = this.tokenize(inputText);
const encoded = this.encode(tokenized);
const encodedTokenChunks = chunk(encoded.tokens, inputSize - 2);
const encodedWordIdChunks = chunk(encoded.wordIds, inputSize - 2);
const chunks = [];
zip(encodedTokenChunks, encodedWordIdChunks).forEach(([tokens, wordIds]) => {
const inputIds = [CLS_INDEX, ...tokens, SEP_INDEX];
const segmentIds = Array(inputIds.length).fill(0);
const inputMask = Array(inputIds.length).fill(1);
wordIds = [-1, ...wordIds, -1];
while (inputIds.length < inputSize) {
inputIds.push(0);
inputMask.push(0);
segmentIds.push(0);
wordIds.push(-1);
}
chunks.push({inputIds, inputMask, segmentIds, wordIds})
});
return {
inputChunks: chunks,
words: tokenized
};
}
}

43
sist2-vue/src/ml/modelsRepo.js Normal file
View File

@ -0,0 +1,43 @@
import axios from "axios";
class ModelsRepo {
_repositories;
data = {};
async init(repositories) {
this._repositories = repositories;
const data = await Promise.all(this._repositories.map(this._loadRepository));
data.forEach(models => {
models.forEach(model => {
this.data[model.name] = model;
})
});
}
async _loadRepository(repository) {
const data = (await axios.get(repository)).data;
data.forEach(model => {
model["modelUrl"] = new URL(model["modelPath"], repository).href;
model["vocabUrl"] = new URL(model["vocabPath"], repository).href;
});
return data;
}
getOptions() {
return Object.values(this.data).map(model => ({
text: `${model.name} (${Math.round(model.size / (1024*1024))}MB)`,
value: model.name
}));
}
getDefaultModel() {
if (Object.values(this.data).length === 0) {
return null;
}
return Object.values(this.data).find(model => model.default).name;
}
}
export default new ModelsRepo();

21
sist2-vue/src/store/index.ts
View File

@ -57,6 +57,9 @@ export default new Vuex.Store({
optVidPreviewInterval: 700, optVidPreviewInterval: 700,
optSimpleLightbox: true, optSimpleLightbox: true,
optShowTagPickerFilter: true, optShowTagPickerFilter: true,
optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
optAutoAnalyze: false,
optMlDefaultModel: null,
_onLoadSelectedIndices: [] as string[], _onLoadSelectedIndices: [] as string[],
_onLoadSelectedMimeTypes: [] as string[], _onLoadSelectedMimeTypes: [] as string[],
@ -86,7 +89,11 @@ export default new Vuex.Store({
uiMimeMap: [] as any[], uiMimeMap: [] as any[],
auth0Token: null auth0Token: null,
mlModel: {
model: null,
name: null
},
}, },
mutations: { mutations: {
setUiShowDetails: (state, val) => state.uiShowDetails = val, setUiShowDetails: (state, val) => state.uiShowDetails = val,
@ -172,6 +179,9 @@ export default new Vuex.Store({
setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val, setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val, setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val, setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
setOptAutoAnalyze: (state, val) => {state.optAutoAnalyze = val},
setOptMlRepositories: (state, val) => {state.optMlRepositories = val},
setOptMlDefaultModel: (state, val) => {state.optMlDefaultModel = val},
setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val, setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val, setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@ -194,6 +204,7 @@ export default new Vuex.Store({
// noop // noop
}, },
setAuth0Token: (state, val) => state.auth0Token = val, setAuth0Token: (state, val) => state.auth0Token = val,
setMlModel: (state, val) => state.mlModel = val,
}, },
actions: { actions: {
setSist2Info: (store, val) => { setSist2Info: (store, val) => {
@ -350,6 +361,7 @@ export default new Vuex.Store({
}, },
modules: {}, modules: {},
getters: { getters: {
mlModel: (state) => state.mlModel,
seed: (state) => state.seed, seed: (state) => state.seed,
getPathText: (state) => state.pathText, getPathText: (state) => state.pathText,
indices: state => state.indices, indices: state => state.indices,
@ -416,5 +428,12 @@ export default new Vuex.Store({
optSimpleLightbox: state => state.optSimpleLightbox, optSimpleLightbox: state => state.optSimpleLightbox,
optShowTagPickerFilter: state => state.optShowTagPickerFilter, optShowTagPickerFilter: state => state.optShowTagPickerFilter,
optFeaturedFields: state => state.optFeaturedFields, optFeaturedFields: state => state.optFeaturedFields,
optMlRepositories: state => state.optMlRepositories,
mlRepositoryList: state => {
const repos = state.optMlRepositories.split("\n")
return repos[0] == "" ? [] : repos;
},
optMlDefaultModel: state => state.optMlDefaultModel,
optAutoAnalyze: state => state.optAutoAnalyze,
} }
}) })

680
sist2-vue/src/views/Configuration.vue
View File

@ -1,202 +1,218 @@
<template> <template>
<!-- <div :style="{width: `${$store.getters.optContainerWidth}px`}"--> <!-- <div :style="{width: `${$store.getters.optContainerWidth}px`}"-->
<div <div
v-if="!configLoading" v-if="!configLoading"
style="margin-left: auto; margin-right: auto;" class="container"> style="margin-left: auto; margin-right: auto;" class="container">
<b-card>
<b-card-title>
<GearIcon></GearIcon>
{{ $t("config") }}
</b-card-title>
<p>{{ $t("configDescription") }}</p>
<b-card-body>
<h4>{{ $t("displayOptions") }}</h4>
<b-card> <b-card>
<b-card-title>
<GearIcon></GearIcon>
{{ $t("config") }}
</b-card-title>
<p>{{ $t("configDescription") }}</p>
<label> <b-card-body>
<LanguageIcon/> <h4>{{ $t("displayOptions") }}</h4>
<span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
<b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
<label>{{ $t("opt.theme") }}</label> <b-card>
<b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
<label>{{ $t("opt.displayMode") }}</label> <label>
<b-form-select :options="displayModeOptions" :value="optDisplay" @input="setOptDisplay"></b-form-select> <LanguageIcon/>
<span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
<b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
<label>{{ $t("opt.columns") }}</label> <label>{{ $t("opt.theme") }}</label>
<b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select> <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
<div style="height: 10px"></div> <label>{{ $t("opt.displayMode") }}</label>
<b-form-select :options="displayModeOptions" :value="optDisplay"
@input="setOptDisplay"></b-form-select>
<b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent"> <label>{{ $t("opt.columns") }}</label>
{{ $t("opt.lightboxLoadOnlyCurrent") }} <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
</b-form-checkbox>
<b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy"> <div style="height: 10px"></div>
{{ $t("opt.hideLegacy") }}
</b-form-checkbox>
<b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap"> <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
{{ $t("opt.updateMimeMap") }} {{ $t("opt.lightboxLoadOnlyCurrent") }}
</b-form-checkbox> </b-form-checkbox>
<b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker"> <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
{{ $t("opt.useDatePicker") }} {{ $t("opt.hideLegacy") }}
</b-form-checkbox> </b-form-checkbox>
<b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{ <b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
$t("opt.simpleLightbox") {{ $t("opt.updateMimeMap") }}
}} </b-form-checkbox>
</b-form-checkbox>
<b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{ <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
$t("opt.showTagPickerFilter") {{ $t("opt.useDatePicker") }}
}} </b-form-checkbox>
</b-form-checkbox>
<br/> <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
<label>{{ $t("opt.featuredFields") }}</label> $t("opt.simpleLightbox")
}}
</b-form-checkbox>
<br> <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
<b-button v-b-toggle.collapse-1 variant="secondary" class="dropdown-toggle">{{ $t("opt.showTagPickerFilter")
$t("opt.featuredFieldsList") }}
}} </b-form-checkbox>
</b-button>
<b-collapse id="collapse-1" class="mt-2">
<ul>
<li><code>doc.checksum</code></li>
<li><code>doc.path</code></li>
<li><code>doc.mime</code></li>
<li><code>doc.videoc</code></li>
<li><code>doc.audioc</code></li>
<li><code>doc.pages</code></li>
<li><code>doc.mtime</code></li>
<li><code>doc.font_name</code></li>
<li><code>doc.album</code></li>
<li><code>doc.artist</code></li>
<li><code>doc.title</code></li>
<li><code>doc.genre</code></li>
<li><code>doc.album_artist</code></li>
<li><code>doc.exif_make</code></li>
<li><code>doc.exif_model</code></li>
<li><code>doc.exif_software</code></li>
<li><code>doc.exif_exposure_time</code></li>
<li><code>doc.exif_fnumber</code></li>
<li><code>doc.exif_iso_speed_ratings</code></li>
<li><code>doc.exif_focal_length</code></li>
<li><code>doc.exif_user_comment</code></li>
<li><code>doc.exif_user_comment</code></li>
<li><code>doc.exif_gps_longitude_ref</code></li>
<li><code>doc.exif_gps_longitude_dms</code></li>
<li><code>doc.exif_gps_longitude_dec</code></li>
<li><code>doc.exif_gps_latitude_ref</code></li>
<li><code>doc.exif_gps_latitude_dec</code></li>
<li><code>humanDate()</code></li>
<li><code>humanFileSize()</code></li>
</ul>
<p>{{ $t("forExample") }}</p> <br/>
<label>{{ $t("opt.featuredFields") }}</label>
<ul> <br>
<li> <b-button v-b-toggle.collapse-1 variant="secondary" class="dropdown-toggle">{{
<code>&lt;b&gt;${humanDate(doc.mtime)}&lt;/b&gt; • ${doc.videoc || ''}</code> $t("opt.featuredFieldsList")
</li> }}
<li> </b-button>
<code>${doc.pages ? (doc.pages + ' pages') : ''}</code> <b-collapse id="collapse-1" class="mt-2">
</li> <ul>
</ul> <li><code>doc.checksum</code></li>
</b-collapse> <li><code>doc.path</code></li>
<br/> <li><code>doc.mime</code></li>
<br/> <li><code>doc.videoc</code></li>
<b-textarea rows="3" :value="optFeaturedFields" @input="setOptFeaturedFields"></b-textarea> <li><code>doc.audioc</code></li>
<li><code>doc.pages</code></li>
<li><code>doc.mtime</code></li>
<li><code>doc.font_name</code></li>
<li><code>doc.album</code></li>
<li><code>doc.artist</code></li>
<li><code>doc.title</code></li>
<li><code>doc.genre</code></li>
<li><code>doc.album_artist</code></li>
<li><code>doc.exif_make</code></li>
<li><code>doc.exif_model</code></li>
<li><code>doc.exif_software</code></li>
<li><code>doc.exif_exposure_time</code></li>
<li><code>doc.exif_fnumber</code></li>
<li><code>doc.exif_iso_speed_ratings</code></li>
<li><code>doc.exif_focal_length</code></li>
<li><code>doc.exif_user_comment</code></li>
<li><code>doc.exif_user_comment</code></li>
<li><code>doc.exif_gps_longitude_ref</code></li>
<li><code>doc.exif_gps_longitude_dms</code></li>
<li><code>doc.exif_gps_longitude_dec</code></li>
<li><code>doc.exif_gps_latitude_ref</code></li>
<li><code>doc.exif_gps_latitude_dec</code></li>
<li><code>humanDate()</code></li>
<li><code>humanFileSize()</code></li>
</ul>
<p>{{ $t("forExample") }}</p>
<ul>
<li>
<code>&lt;b&gt;${humanDate(doc.mtime)}&lt;/b&gt; • ${doc.videoc || ''}</code>
</li>
<li>
<code>${doc.pages ? (doc.pages + ' pages') : ''}</code>
</li>
</ul>
</b-collapse>
<br/>
<br/>
<b-textarea rows="3" :value="optFeaturedFields" @input="setOptFeaturedFields"></b-textarea>
</b-card>
<br/>
<h4>{{ $t("searchOptions") }}</h4>
<b-card>
<b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
$t("opt.hideDuplicates")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{
$t("opt.highlight")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
$t("opt.tagOrOperator")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
<b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
$t("opt.searchInPath")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
$t("opt.suggestPath")
}}
</b-form-checkbox>
<br/>
<label>{{ $t("opt.fragmentSize") }}</label>
<b-form-input :value="optFragmentSize" step="10" type="number" min="0"
@input="setOptFragmentSize"></b-form-input>
<label>{{ $t("opt.resultSize") }}</label>
<b-form-input :value="optResultSize" type="number" min="10"
@input="setOptResultSize"></b-form-input>
<label>{{ $t("opt.queryMode") }}</label>
<b-form-select :options="queryModeOptions" :value="optQueryMode"
@input="setOptQueryMode"></b-form-select>
<label>{{ $t("opt.slideDuration") }}</label>
<b-form-input :value="optLightboxSlideDuration" type="number" min="1"
@input="setOptLightboxSlideDuration"></b-form-input>
<label>{{ $t("opt.vidPreviewInterval") }}</label>
<b-form-input :value="optVidPreviewInterval" type="number" min="50"
@input="setOptVidPreviewInterval"></b-form-input>
</b-card>
<h4 class="mt-3">{{ $t("mlOptions") }}</h4>
<b-card>
<label>{{ $t("opt.mlRepositories") }}</label>
<b-textarea rows="3" :value="optMlRepositories" @input="setOptMlRepositories"></b-textarea>
<br>
<b-form-checkbox :checked="optAutoAnalyze" @input="setOptAutoAnalyze">{{
$t("opt.autoAnalyze")
}}
</b-form-checkbox>
</b-card>
<h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
<b-card>
<label>{{ $t("opt.treemapType") }}</label>
<b-form-select :value="optTreemapType" :options="treemapTypeOptions"
@input="setOptTreemapType"></b-form-select>
<label>{{ $t("opt.treemapTiling") }}</label>
<b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
@input="setOptTreemapTiling"></b-form-select>
<label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
<b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
@input="setOptTreemapColorGroupingDepth"></b-form-input>
<label>{{ $t("opt.treemapSize") }}</label>
<b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
@input="setOptTreemapSize"></b-form-select>
<template v-if="$store.getters.optTreemapSize === 'custom'">
<!-- TODO Width/Height input -->
<b-form-input type="number" min="0" step="10"></b-form-input>
<b-form-input type="number" min="0" step="10"></b-form-input>
</template>
<label>{{ $t("opt.treemapColor") }}</label>
<b-form-select :value="optTreemapColor" :options="treemapColorOptions"
@input="setOptTreemapColor"></b-form-select>
</b-card>
<b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
</b-card-body>
</b-card> </b-card>
<br/> <b-card v-if="loading" class="mt-4">
<h4>{{ $t("searchOptions") }}</h4> <Preloader></Preloader>
<b-card>
<b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
$t("opt.hideDuplicates")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
<b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
$t("opt.tagOrOperator")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
<b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
$t("opt.searchInPath")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
$t("opt.suggestPath")
}}
</b-form-checkbox>
<br/>
<label>{{ $t("opt.fragmentSize") }}</label>
<b-form-input :value="optFragmentSize" step="10" type="number" min="0"
@input="setOptFragmentSize"></b-form-input>
<label>{{ $t("opt.resultSize") }}</label>
<b-form-input :value="optResultSize" type="number" min="10"
@input="setOptResultSize"></b-form-input>
<label>{{ $t("opt.queryMode") }}</label>
<b-form-select :options="queryModeOptions" :value="optQueryMode" @input="setOptQueryMode"></b-form-select>
<label>{{ $t("opt.slideDuration") }}</label>
<b-form-input :value="optLightboxSlideDuration" type="number" min="1"
@input="setOptLightboxSlideDuration"></b-form-input>
<label>{{ $t("opt.vidPreviewInterval") }}</label>
<b-form-input :value="optVidPreviewInterval" type="number" min="50"
@input="setOptVidPreviewInterval"></b-form-input>
</b-card> </b-card>
<DebugInfo v-else></DebugInfo>
<h4 class="mt-3">{{ $t("treemapOptions") }}</h4> </div>
<b-card>
<label>{{ $t("opt.treemapType") }}</label>
<b-form-select :value="optTreemapType" :options="treemapTypeOptions"
@input="setOptTreemapType"></b-form-select>
<label>{{ $t("opt.treemapTiling") }}</label>
<b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
@input="setOptTreemapTiling"></b-form-select>
<label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
<b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
@input="setOptTreemapColorGroupingDepth"></b-form-input>
<label>{{ $t("opt.treemapSize") }}</label>
<b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
@input="setOptTreemapSize"></b-form-select>
<template v-if="$store.getters.optTreemapSize === 'custom'">
<!-- TODO Width/Height input -->
<b-form-input type="number" min="0" step="10"></b-form-input>
<b-form-input type="number" min="0" step="10"></b-form-input>
</template>
<label>{{ $t("opt.treemapColor") }}</label>
<b-form-select :value="optTreemapColor" :options="treemapColorOptions"
@input="setOptTreemapColor"></b-form-select>
</b-card>
<b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
</b-card-body>
</b-card>
<b-card v-if="loading" class="mt-4">
<Preloader></Preloader>
</b-card>
<DebugInfo v-else></DebugInfo>
</div>
</template> </template>
<script> <script>
@ -208,164 +224,168 @@ import GearIcon from "@/components/icons/GearIcon.vue";
import LanguageIcon from "@/components/icons/LanguageIcon"; import LanguageIcon from "@/components/icons/LanguageIcon";
export default { export default {
components: {LanguageIcon, GearIcon, DebugInfo, Preloader}, components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
data() { data() {
return { return {
loading: false, loading: false,
configLoading: false, configLoading: false,
langOptions: [ langOptions: [
{value: "en", text: this.$t("lang.en")}, {value: "en", text: this.$t("lang.en")},
{value: "fr", text: this.$t("lang.fr")}, {value: "fr", text: this.$t("lang.fr")},
{value: "zh-CN", text: this.$t("lang.zh-CN")}, {value: "zh-CN", text: this.$t("lang.zh-CN")},
{value: "de", text: this.$t("lang.de")}, {value: "de", text: this.$t("lang.de")},
], ],
queryModeOptions: [ queryModeOptions: [
{value: "simple", text: this.$t("queryMode.simple")}, {value: "simple", text: this.$t("queryMode.simple")},
{value: "advanced", text: this.$t("queryMode.advanced")} {value: "advanced", text: this.$t("queryMode.advanced")}
], ],
displayModeOptions: [ displayModeOptions: [
{value: "grid", text: this.$t("displayMode.grid")}, {value: "grid", text: this.$t("displayMode.grid")},
{value: "list", text: this.$t("displayMode.list")} {value: "list", text: this.$t("displayMode.list")}
], ],
columnsOptions: [ columnsOptions: [
{value: "auto", text: this.$t("columns.auto")}, {value: "auto", text: this.$t("columns.auto")},
{value: 1, text: "1"}, {value: 1, text: "1"},
{value: 2, text: "2"}, {value: 2, text: "2"},
{value: 3, text: "3"}, {value: 3, text: "3"},
{value: 4, text: "4"}, {value: 4, text: "4"},
{value: 5, text: "5"}, {value: 5, text: "5"},
{value: 6, text: "6"}, {value: 6, text: "6"},
{value: 7, text: "7"}, {value: 7, text: "7"},
{value: 8, text: "8"}, {value: 8, text: "8"},
{value: 9, text: "9"}, {value: 9, text: "9"},
{value: 10, text: "10"}, {value: 10, text: "10"},
{value: 11, text: "11"}, {value: 11, text: "11"},
{value: 12, text: "12"}, {value: 12, text: "12"},
], ],
treemapTypeOptions: [ treemapTypeOptions: [
{value: "cascaded", text: this.$t("treemapType.cascaded")}, {value: "cascaded", text: this.$t("treemapType.cascaded")},
{value: "flat", text: this.$t("treemapType.flat")} {value: "flat", text: this.$t("treemapType.flat")}
], ],
treemapTilingOptions: [ treemapTilingOptions: [
{value: "binary", text: this.$t("treemapTiling.binary")}, {value: "binary", text: this.$t("treemapTiling.binary")},
{value: "squarify", text: this.$t("treemapTiling.squarify")}, {value: "squarify", text: this.$t("treemapTiling.squarify")},
{value: "slice", text: this.$t("treemapTiling.slice")}, {value: "slice", text: this.$t("treemapTiling.slice")},
{value: "dice", text: this.$t("treemapTiling.dice")}, {value: "dice", text: this.$t("treemapTiling.dice")},
{value: "sliceDice", text: this.$t("treemapTiling.sliceDice")}, {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
], ],
treemapSizeOptions: [ treemapSizeOptions: [
{value: "small", text: this.$t("treemapSize.small")}, {value: "small", text: this.$t("treemapSize.small")},
{value: "medium", text: this.$t("treemapSize.medium")}, {value: "medium", text: this.$t("treemapSize.medium")},
{value: "large", text: this.$t("treemapSize.large")}, {value: "large", text: this.$t("treemapSize.large")},
{value: "x-large", text: this.$t("treemapSize.xLarge")}, {value: "x-large", text: this.$t("treemapSize.xLarge")},
{value: "xx-large", text: this.$t("treemapSize.xxLarge")}, {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
// {value: "custom", text: this.$t("treemapSize.custom")}, // {value: "custom", text: this.$t("treemapSize.custom")},
], ],
treemapColorOptions: [ treemapColorOptions: [
{value: "PuBuGn", text: "Purple-Blue-Green"}, {value: "PuBuGn", text: "Purple-Blue-Green"},
{value: "PuRd", text: "Purple-Red"}, {value: "PuRd", text: "Purple-Red"},
{value: "PuBu", text: "Purple-Blue"}, {value: "PuBu", text: "Purple-Blue"},
{value: "YlOrBr", text: "Yellow-Orange-Brown"}, {value: "YlOrBr", text: "Yellow-Orange-Brown"},
{value: "YlOrRd", text: "Yellow-Orange-Red"}, {value: "YlOrRd", text: "Yellow-Orange-Red"},
{value: "YlGn", text: "Yellow-Green"}, {value: "YlGn", text: "Yellow-Green"},
{value: "YlGnBu", text: "Yellow-Green-Blue"}, {value: "YlGnBu", text: "Yellow-Green-Blue"},
{value: "Plasma", text: "Plasma"}, {value: "Plasma", text: "Plasma"},
{value: "Magma", text: "Magma"}, {value: "Magma", text: "Magma"},
{value: "Inferno", text: "Inferno"}, {value: "Inferno", text: "Inferno"},
{value: "Viridis", text: "Viridis"}, {value: "Viridis", text: "Viridis"},
{value: "Turbo", text: "Turbo"}, {value: "Turbo", text: "Turbo"},
], ],
themeOptions: [ themeOptions: [
{value: "light", text: this.$t("theme.light")}, {value: "light", text: this.$t("theme.light")},
{value: "black", text: this.$t("theme.black")} {value: "black", text: this.$t("theme.black")}
] ]
} }
}, },
computed: { computed: {
...mapGetters([ ...mapGetters([
"optTheme", "optTheme",
"optDisplay", "optDisplay",
"optColumns", "optColumns",
"optHighlight", "optHighlight",
"optFuzzy", "optFuzzy",
"optSearchInPath", "optSearchInPath",
"optSuggestPath", "optSuggestPath",
"optFragmentSize", "optFragmentSize",
"optQueryMode", "optQueryMode",
"optTreemapType", "optTreemapType",
"optTreemapTiling", "optTreemapTiling",
"optTreemapColorGroupingDepth", "optTreemapColorGroupingDepth",
"optTreemapColor", "optTreemapColor",
"optTreemapSize", "optTreemapSize",
"optLightboxLoadOnlyCurrent", "optLightboxLoadOnlyCurrent",
"optLightboxSlideDuration", "optLightboxSlideDuration",
"optResultSize", "optResultSize",
"optTagOrOperator", "optTagOrOperator",
"optLang", "optLang",
"optHideDuplicates", "optHideDuplicates",
"optHideLegacy", "optHideLegacy",
"optUpdateMimeMap", "optUpdateMimeMap",
"optUseDatePicker", "optUseDatePicker",
"optVidPreviewInterval", "optVidPreviewInterval",
"optSimpleLightbox", "optSimpleLightbox",
"optShowTagPickerFilter", "optShowTagPickerFilter",
"optFeaturedFields", "optFeaturedFields",
]), "optMlRepositories",
clientWidth() { "optAutoAnalyze",
return window.innerWidth; ]),
} clientWidth() {
}, return window.innerWidth;
mounted() { }
this.$store.subscribe((mutation) => { },
if (mutation.type.startsWith("setOpt")) { mounted() {
this.$store.dispatch("updateConfiguration"); this.$store.subscribe((mutation) => {
} if (mutation.type.startsWith("setOpt")) {
}); this.$store.dispatch("updateConfiguration");
}, }
methods: { });
...mapActions({ },
setSist2Info: "setSist2Info", methods: {
}), ...mapActions({
...mapMutations([ setSist2Info: "setSist2Info",
"setOptTheme", }),
"setOptDisplay", ...mapMutations([
"setOptColumns", "setOptTheme",
"setOptHighlight", "setOptDisplay",
"setOptFuzzy", "setOptColumns",
"setOptSearchInPath", "setOptHighlight",
"setOptSuggestPath", "setOptFuzzy",
"setOptFragmentSize", "setOptSearchInPath",
"setOptQueryMode", "setOptSuggestPath",
"setOptTreemapType", "setOptFragmentSize",
"setOptTreemapTiling", "setOptQueryMode",
"setOptTreemapColorGroupingDepth", "setOptTreemapType",
"setOptTreemapColor", "setOptTreemapTiling",
"setOptTreemapSize", "setOptTreemapColorGroupingDepth",
"setOptLightboxLoadOnlyCurrent", "setOptTreemapColor",
"setOptLightboxSlideDuration", "setOptTreemapSize",
"setOptResultSize", "setOptLightboxLoadOnlyCurrent",
"setOptTagOrOperator", "setOptLightboxSlideDuration",
"setOptLang", "setOptResultSize",
"setOptHideDuplicates", "setOptTagOrOperator",
"setOptHideLegacy", "setOptLang",
"setOptUpdateMimeMap", "setOptHideDuplicates",
"setOptUseDatePicker", "setOptHideLegacy",
"setOptVidPreviewInterval", "setOptUpdateMimeMap",
"setOptSimpleLightbox", "setOptUseDatePicker",
"setOptShowTagPickerFilter", "setOptVidPreviewInterval",
"setOptFeaturedFields", "setOptSimpleLightbox",
]), "setOptShowTagPickerFilter",
onResetClick() { "setOptFeaturedFields",
localStorage.removeItem("sist2_configuration"); "setOptMlRepositories",
window.location.reload(); "setOptAutoAnalyze",
} ]),
}, onResetClick() {
localStorage.removeItem("sist2_configuration");
window.location.reload();
}
},
} }
</script> </script>
<style> <style>
.shrink { .shrink {
flex-grow: inherit; flex-grow: inherit;
} }
</style> </style>

4
src/sist.h
View File

@ -51,11 +51,11 @@
#include <ctype.h> #include <ctype.h>
#include "git_hash.h" #include "git_hash.h"
#define VERSION "3.0.3" #define VERSION "3.0.4"
static const char *const Version = VERSION; static const char *const Version = VERSION;
static const int VersionMajor = 3; static const int VersionMajor = 3;
static const int VersionMinor = 0; static const int VersionMinor = 0;
static const int VersionPatch = 3; static const int VersionPatch = 4;
#ifndef SIST_PLATFORM #ifndef SIST_PLATFORM
#define SIST_PLATFORM unknown #define SIST_PLATFORM unknown