Compare commits

...

94 Commits

Author SHA1 Message Date
4dedd281f1 Push compiled vue changes 2022-01-09 09:30:31 -05:00
65c499e477 Merge pull request #231 from simon987/dev
v2.11.6
2022-01-09 09:28:24 -05:00
625f3d0d6e Option to update media type tab in real time, add media type table in details 2022-01-08 18:23:22 -05:00
64b8aab8bf Validate that all the tesseract data files are in the same folder 2022-01-08 15:04:07 -05:00
ad95684771 Update --ocr-* args, enable OCR'ing images 2022-01-08 14:24:50 -05:00
b37e5a4ad4 Fix some warnings in media.c 2022-01-08 11:06:14 -05:00
15ae2190cf Fix tesseract lang validation, update README.md, fix tesseract memory leak 2022-01-08 11:04:52 -05:00
255bc2d689 Tweak MIN_OCR_SIZE behavior, update gitignore 2022-01-08 10:33:02 -05:00
fe1aa6dd4c Merge pull request #227 from yatli/dev
refactor: split ocr_extract_text from ebook
2022-01-08 10:25:41 -05:00
cd2a44e016 Update ocr.h
Fix minimum image size validation in ocr_extract_text
2022-01-08 10:24:57 -05:00
ed2a3f342a Localize tag add/delete, fix some translations, add LanguageIcon, add --lang arg, fix lightbox slideshow time, fix gif hover 2022-01-08 10:03:38 -05:00
1107fe9a53 Remove libscan hash debug info 2022-01-08 10:00:34 -05:00
a96e65d039 Add zh-CN option in language dropdown 2022-01-07 17:44:49 -05:00
87936eecd4 Merge pull request #229 from yatli/master
add zh-CN translation
2022-01-07 13:55:14 -05:00
Yatao Li
d817a0e9dd add zh-CN translation 2022-01-08 01:39:50 +08:00
Yatao Li
94a5e0ac59 refactor: split ocr_extract_text from ebook 2022-01-07 23:20:35 +08:00
d40f5052f9 static link for libasan in debug build 2021-12-29 19:25:03 -05:00
ee9a8fa514 Add thread lock for incremental_mark_file_for_copy() 2021-12-29 19:18:10 -05:00
81008d8936 Add --list-file argument 2021-12-29 18:54:13 -05:00
52466d5d8a Update tesseract datapaths 2021-12-25 11:12:00 -05:00
5f73fc024b Version bump, update readme 2021-12-25 11:08:52 -05:00
f2fd7ccf41 Fix raw parsing maybe, fix index picker css 2021-12-25 11:08:52 -05:00
d87fee8e00 Merge pull request #214 from dpieski/patch-2
Update USAGE.md
2021-12-22 09:55:24 -05:00
Andrew
672d1344d7 Update USAGE.md
Get-WmiObject is deprecated in favor of Get-CimInstance
2021-12-15 15:00:36 -06:00
27e32db1ed Fix attempt for excludes 2021-11-17 20:18:48 -05:00
bb91139ffb console log fixes, version bump 2021-11-15 20:52:24 -05:00
70cfa8c37c Fix Dockerfile.arm64 2021-11-13 18:25:24 -05:00
7493dedc8c Merge pull request #208 from simon987/dev
v2.11.4
2021-11-13 17:37:47 -05:00
c786a31bb2 Merge remote-tracking branch 'origin/master' into dev
# Conflicts:
#	README.md
2021-11-13 17:36:55 -05:00
48d024e751 Update dockerfiles 2021-11-13 17:36:30 -05:00
08b2ca9d43 Update lcms -> lcms2 2021-11-12 11:29:50 -05:00
ed8b4f4fad Add natural sorting support 2021-11-12 10:33:51 -05:00
66de93a8bd Language & formatting 2021-11-12 10:17:32 -05:00
e3f78fb693 Shift click & select all/none in index picker 2021-11-12 10:12:25 -05:00
030643cee0 Move CI scripts to script folder 2021-11-12 09:05:37 -05:00
b17b9439df Print progress bar in index module 2021-11-07 13:20:05 -05:00
414f65346c Update docker command in README.md 2021-11-07 13:18:32 -05:00
be8eedc9c7 Skip subtree of excluded directories 2021-11-07 11:56:09 -05:00
5b62fe77f2 Update demo URL 2021-11-07 09:52:28 -05:00
61ab68ce15 Update argparse repo URL 2021-11-07 09:42:17 -05:00
82ecb8bb85 Update gitignore 2021-11-07 09:36:39 -05:00
a41b5dcc1f Remove libscan git submodule 2021-11-07 09:30:14 -05:00
06f21d5f0f Remove libscan submodule 2021-11-07 09:17:02 -05:00
e82a388d1e Don't show resolution badge on narrow images 2021-10-22 10:21:35 -04:00
bf02e571b3 Forgot to add that file two commits ago 2021-10-22 09:44:56 -04:00
750a392a61 Show reduced ResuldCard when there are no results 2021-10-22 09:32:17 -04:00
3d7b977a82 Read ES version, handle legacy versions, add notice & debug info 2021-10-21 19:14:43 -04:00
cd71551a22 Some documentation updates 2021-09-25 09:30:53 -04:00
58741058cf Merge pull request #200 from simon987/dev
v2.11.3
2021-09-24 20:56:00 -04:00
0a7e59b646 Some documentation updates 2021-09-24 20:55:08 -04:00
43a566fe2f Version bump 2021-09-24 20:33:19 -04:00
b2631a86c8 Rework index picker 2021-09-24 20:31:11 -04:00
d0a1deca30 Fix thumbnail in DocInfoModal.vue 2021-09-24 19:40:06 -04:00
b03ce90a05 Fix max_analyzed_offset error 2021-09-20 21:01:23 -04:00
a5eacb4950 Set list item color for sub-documents 2021-09-20 20:40:48 -04:00
0887046b41 Fix sidecar files, better error handling in store_write 2021-09-20 20:34:05 -04:00
17fda1e540 Support for rewind buffer 2021-09-11 20:46:40 -04:00
34b363bfd8 Add argument to calculate checksums 2021-09-11 14:31:48 -04:00
c9aa4bed72 Add argument to calculate checksums 2021-09-11 14:31:31 -04:00
7267d4bd2c Add basic JSON/NDJSON support 2021-09-07 08:14:32 -04:00
43470e9ce6 Add basic JSON/NDJSON support 2021-09-06 21:27:17 -04:00
0331d46fff Merge pull request #186 from simon987/dev
v2.11.2
2021-09-06 14:14:51 -04:00
bbf1aca936 Version bump 2021-09-06 14:14:00 -04:00
27560a82bb Basic support for WordPerfect files 2021-09-06 14:08:53 -04:00
f16ead1902 Parse page numbers from .docx files 2021-09-06 09:50:00 -04:00
e2e07e80c7 Install libasan5 in Dockerfile 2021-09-06 09:25:01 -04:00
9499c6b189 Add v prefix in version badge 2021-09-06 09:18:28 -04:00
c5cd00b76c Update USAGE.md 2021-09-05 20:26:09 -04:00
ec5f07cab8 Merge pull request #184 from simon987/dev
v2.11.1
2021-09-05 20:06:18 -04:00
f098f7916a Version bump 2021-09-05 20:05:46 -04:00
85d67a9393 null checks in sig_handler 2021-09-05 20:03:42 -04:00
c5ac89813f Fix UI bug when losing focus of tags/mime tree 2021-09-05 19:59:01 -04:00
ec5642a3df Fix docker build for arm64 2021-09-05 13:41:08 -04:00
c1de74e7eb Fix build_arm64.sh (again) 2021-09-05 12:58:49 -04:00
f31f138f2e Set default tagline when none is specified 2021-09-05 12:53:52 -04:00
6a48b219e6 Fix build_arm64.sh & update README 2021-09-05 12:19:44 -04:00
97e56b664c Merge pull request #182 from simon987/dev
Fix .dockerignore, add docker build for arm64
2021-09-05 11:30:03 -04:00
0acdaab31a Fix .dockerignore, add docker build for arm64 2021-09-05 11:06:04 -04:00
d0290b886b Update .gitattributes 2021-09-05 10:16:13 -04:00
4eea376869 Merge pull request #181 from simon987/dev
v2.11.0
2021-09-05 10:02:01 -04:00
f4e1d90a6b web UI rewrite, switch to ndjson.zst index format 2021-09-05 09:49:25 -04:00
391d8ed9d9 Add error handling for ES mappings/settings 2021-09-05 09:49:25 -04:00
89fd68be8e Update ES version in readme 2021-08-27 19:15:59 -04:00
9c0f3e0e31 Fix .docx segmentation fault 2021-08-16 17:50:54 -04:00
648e4817c4 Add Accept-Range header 2021-07-19 08:42:29 -04:00
bb23651087 update docker image version in readme 2021-07-10 13:05:13 -04:00
78f3c897e2 libscan version sync 2021-07-10 12:52:24 -04:00
a2209e91ca add fast-epub arg, tweak thread pool, error handling for nftw 2021-07-10 12:52:04 -04:00
ba31531d3a Return 404 when thumbnail is not found 2021-07-10 12:45:30 -04:00
d5a47b8dab Fix segfault in exec-script 2021-06-23 19:46:52 -04:00
d8c0b80524 Update README.md 2021-06-23 10:26:20 -04:00
142a4869e6 Bump version 2021-06-23 08:19:05 -04:00
ddb7f8d5d7 Fix some bugs in serve.c caused by mongoose upgrade 2021-06-23 08:18:11 -04:00
dfb8c67490 thread safety for debug info table 2021-06-14 15:04:08 -04:00
194 changed files with 41688 additions and 5753 deletions

View File

@@ -1,25 +1,30 @@
.idea
*/thumbs
*.cbp
CMakeCache.txt
CMakeFiles
cmake-build-debug
cmake_install.cmake
.idea/
*.sist2
docs/
test_i/
test_i_inc/
Testing/
.drone.yml
**/cmake_install.cmake
**/CMakeCache.txt
**/CMakeFiles/
LICENSE
Makefile
*.out
LOG
sist2*
index.sist2/
bundle*.css
bundle.js
**/*.a
**/vgcore.*
build/
**/*.md
**/*.cbp
VERSION
**/node_modules/
.git/
third-party/libscan/libscan-test-files/
sist2-*-linux-debug
sist2-*-linux
sist2_debug
sist2
**/libscan-test-files
**/scan_ub_test
**/scan_a_test
**/scan_test
**/ext_ffmpeg
**/ext_libmobi
**/scan_a_test
Dockerfile
*.idx/
VERSION
**/ext_libwpd
**/core
*.a

View File

@@ -10,22 +10,7 @@ steps:
- name: build
image: simon987/sist2-build
commands:
- ./ci/build.sh
- name: docker
image: plugins/docker
settings:
username:
from_secret: DOCKER_USER
password:
from_secret: DOCKER_PASSWORD
repo: simon987/sist2
context: ./
dockerfile: ./Dockerfile
auto_tag: true
auto_tag_suffix: x64-linux
when:
event:
- tag
- ./scripts/build.sh
- name: scp files
image: appleboy/drone-scp
settings:
@@ -42,6 +27,21 @@ steps:
- ./VERSION
- ./sist2-x64-linux
- ./sist2-x64-linux-debug
- name: docker
image: plugins/docker
settings:
username:
from_secret: DOCKER_USER
password:
from_secret: DOCKER_PASSWORD
repo: simon987/sist2
context: ./
dockerfile: ./Dockerfile
auto_tag: true
auto_tag_suffix: x64-linux
when:
event:
- tag
---
kind: pipeline
@@ -55,7 +55,7 @@ steps:
- name: build
image: simon987/sist2-build-arm64
commands:
- ./ci/build_arm64.sh
- ./scripts/build_arm64.sh
- name: scp files
image: appleboy/drone-scp
settings:
@@ -70,3 +70,19 @@ steps:
target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
source:
- ./sist2-arm64-linux
- ./sist2-arm64-linux-debug
- name: docker
image: plugins/docker
settings:
username:
from_secret: DOCKER_USER
password:
from_secret: DOCKER_PASSWORD
repo: simon987/sist2
context: ./
dockerfile: ./Dockerfile.arm64
auto_tag: true
auto_tag_suffix: arm64-linux
when:
event:
- tag

4
.gitattributes vendored
View File

@@ -1,3 +1,3 @@
CMakeModules/* linguist-vendored
web/js/*.min.js linguist-vendored
web/css/*.min.css linguist-vendored
**/*_generated.c linguist-vendored
**/*_generated.h linguist-vendored

14
.gitignore vendored
View File

@@ -9,12 +9,20 @@ Makefile
*.out
LOG
sist2*
index.sist2/
!sist2-vue/
*.sist2/
bundle*.css
bundle.js
*.a
vgcore.*
build/
third-party/
third-party/argparse
*.idx/
VERSION
VERSION
git_hash.h
Testing/
test_i
test_i_inc
node_modules/
.cmake/
i_inc/

11
.gitmodules vendored
View File

@@ -1,6 +1,9 @@
[submodule "third-party/libscan"]
path = third-party/libscan
url = https://github.com/simon987/libscan
[submodule "third-party/argparse"]
path = third-party/argparse
url = https://github.com/cofyc/argparse
url = https://github.com/simon987/argparse
[submodule "third-party/libscan/third-party/utf8.h"]
path = third-party/libscan/third-party/utf8.h
url = https://github.com/sheredom/utf8.h
[submodule "third-party/libscan/third-party/antiword"]
path = third-party/libscan/third-party/antiword
url = https://github.com/simon987/antiword

View File

@@ -4,17 +4,24 @@ set(CMAKE_C_STANDARD 11)
project(sist2 C)
option(SIST_DEBUG "Build a debug executable" on)
option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
add_compile_definitions(
"SIST_PLATFORM=${SIST_PLATFORM}"
)
if (SIST_DEBUG)
add_compile_definitions(
"SIST_DEBUG=${SIST_DEBUG}"
)
endif()
set(BUILD_TESTS on)
add_subdirectory(third-party/libscan)
set(ARGPARSE_SHARED off)
add_subdirectory(third-party/argparse)
add_executable(sist2
# argparse
third-party/argparse/argparse.h third-party/argparse/argparse.c
src/main.c
src/sist.h
src/io/walk.h src/io/walk.c
@@ -31,7 +38,11 @@ add_executable(sist2
src/log.c src/log.h
src/cli.c src/cli.h
src/stats.c src/stats.h src/ctx.c
src/parsing/sidecar.c src/parsing/sidecar.h)
src/parsing/sidecar.c src/parsing/sidecar.h
# argparse
third-party/argparse/argparse.h third-party/argparse/argparse.c
)
target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
@@ -76,6 +87,7 @@ if (SIST_DEBUG)
sist2
PRIVATE
-fsanitize=address
-static-libasan
)
set_target_properties(
sist2

View File

@@ -2,14 +2,14 @@ FROM simon987/sist2-build as build
MAINTAINER simon987 <me@simon987.net>
WORKDIR /build/
ADD . /build/
RUN cmake -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
COPY . .
RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
RUN make -j$(nproc)
RUN strip sist2
FROM ubuntu:20.10
FROM ubuntu:21.10
RUN apt update && apt install -y curl
RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
RUN mkdir -p /usr/share/tessdata && \
cd /usr/share/tessdata/ && \
@@ -20,9 +20,9 @@ RUN mkdir -p /usr/share/tessdata && \
curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
COPY --from=build /build/sist2 /root/sist2
ENTRYPOINT ["/root/sist2"]
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
ENTRYPOINT ["/root/sist2"]
COPY --from=build /build/sist2 /root/sist2

View File

@@ -3,13 +3,13 @@ MAINTAINER simon987 <me@simon987.net>
WORKDIR /build/
ADD . /build/
RUN cmake -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
RUN cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
RUN make -j$(nproc)
RUN strip sist2
FROM ubuntu:20.10
FROM --platform="linux/arm64/v8" ubuntu:21.10
RUN apt update && apt install -y curl
RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
RUN mkdir -p /usr/share/tessdata && \
cd /usr/share/tessdata/ && \
@@ -20,9 +20,9 @@ RUN mkdir -p /usr/share/tessdata && \
curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
COPY --from=build /build/sist2 /root/sist2
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
ENTRYPOINT ["/root/sist2"]
ENTRYPOINT ["/root/sist2"]
COPY --from=build /build/sist2 /root/sist2

View File

@@ -2,7 +2,7 @@
[![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
[![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
**Demo**: [sist2.simon987.net](https://sist2.simon987.net/?i=Demo%20files)
**Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
# sist2
@@ -10,7 +10,7 @@ sist2 (Simple incremental search tool)
*Warning: sist2 is in early development*
![sist2.png](docs/sist2.png)
![search panel](docs/sist2.png)
## Features
@@ -33,25 +33,25 @@ sist2 (Simple incremental search tool)
## Getting Started
1. Have an Elasticsearch (>= 6.X.X) instance running
1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
1. *(or)* Run using docker:
```bash
docker run -d --name es1 --net sist2_net -p 9200:9200 \
-e "discovery.type=single-node" elasticsearch:7.5.2
docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.14.0
```
1. *(or)* Run using docker-compose:
```yaml
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
environment:
- discovery.type=single-node
- "ES_JAVA_OPTS=-Xms1G -Xmx2G"
```
1. Download sist2 executable
1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not recommended!)*
1. *(or)* `docker pull simon987/sist2:2.10.1-x64-linux`
1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
recommended!)*
1. *(or)* `docker pull simon987/sist2:2.11.6-x64-linux`
1. See [Usage guide](docs/USAGE.md)
@@ -67,21 +67,23 @@ See [Usage guide](docs/USAGE.md) for more details
## Format support
File type | Library | Content | Thumbnail | Metadata
:---|:---|:---|:---|:---
pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
cbz,cbr | *(none)* | - | yes | - |
`audio/*` | ffmpeg | - | yes | ID3 tags |
`video/*` | ffmpeg | - | yes | title, comment, artist |
`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw | - | yes | Common EXIF tags, GPS tags |
ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
`text/plain` | *(none)* | yes | no | - |
html, xml | *(none)* | yes | no | - |
tar, zip, rar, 7z, ar ... | Libarchive | yes\* | - | no |
docx, xlsx, pptx | *(none)* | yes | if embedded | creator, modified_by, title |
doc (MS Word 97-2003) | antiword | yes | yes | author, title |
mobi, azw, azw3 | libmobi | yes | no | author, title |
| File type | Library | Content | Thumbnail | Metadata |
|:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
| pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
| cbz,cbr | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | - | yes | - |
| `audio/*` | ffmpeg | - | yes | ID3 tags |
| `video/*` | ffmpeg | - | yes | title, comment, artist |
| `image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
| raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw | - | yes | Common EXIF tags, GPS tags |
| ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
| `text/plain` | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | no | - |
| html, xml | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | no | - |
| tar, zip, rar, 7z, ar ... | Libarchive | yes\* | - | no |
| docx, xlsx, pptx | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | if embedded | creator, modified_by, title |
| doc (MS Word 97-2003) | antiword | yes | yes | author, title |
| mobi, azw, azw3 | libmobi | yes | no | author, title |
| wpd (WordPerfect) | libwpd | yes | no | *planned* |
| json, jsonl, ndjson | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | - | - |
\* *See [Archive files](#archive-files)*
@@ -100,18 +102,24 @@ scan is also supported.
### OCR
You can enable OCR support for pdf,xps,fb2,epub file types with the
`--ocr <lang>` option. Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with the
`--ocr-lang <lang>` option in combination with `--ocr-images` and/or `--ocr-ebooks`.
Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
The `simon987/sist2` image comes with common languages
(hin, jpn, eng, fra, rus, spa) pre-installed.
Examples
You can use the `+` separator to specify multiple languages. The language
name must be identical to the `*.traineddata` file installed on your system
(use `chi_sim` rather than `chi-sim`).
Examples:
```bash
sist2 scan --ocr jpn ~/Books/Manga/
sist2 scan --ocr eng ~/Books/Textbooks/
sist2 scan --ocr-ebooks --ocr-lang jpn ~/Books/Manga/
sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
```
## Build from source
@@ -124,7 +132,7 @@ You can compile **sist2** by yourself if you don't want to use the pre-compiled
git clone --recursive https://github.com/simon987/sist2/
cd sist2
docker build . -f ./Dockerfile -t my-sist2-image
docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
```
### On a linux computer
@@ -134,14 +142,14 @@ docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
```bash
apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git
```
1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
1. Install vcpkg dependencies
```bash
vcpkg install curl[core,openssl]
vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libuuid libmagic libraw jasper lcms gumbo
vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo
```
1. Build

View File

@@ -1,19 +0,0 @@
#!/usr/bin/env bash
VCPKG_ROOT="/vcpkg"
rm *.gz &>/dev/null
git submodule update --init --recursive
rm -rf CMakeFiles CMakeCache.txt
cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
make -j $(nproc)
strip sist2
./sist2 -v > VERSION
mv sist2 sist2-x64-linux
rm -rf CMakeFiles CMakeCache.txt
cmake -DSIST_DEBUG=on -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
make -j $(nproc)
mv sist2_debug sist2-x64-linux-debug

View File

@@ -1,13 +0,0 @@
#!/usr/bin/env bash
VCPKG_ROOT="/vcpkg"
rm *.gz &>/dev/null
git submodule update --init --recursive
rm -rf CMakeFiles CMakeCache.txt
cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
make -j $(nproc)
strip sist2
mv sist2 sist2-arm64-linux

View File

@@ -14,6 +14,7 @@
* [examples](#web-examples)
* [rewrite_url](#rewrite_url)
* [link to specific indices](#link-to-specific-indices)
* [elasticsearch](#elasticsearch)
* [exec-script](#exec-script)
* [tagging](#tagging)
* [sidecar files](#sidecar-files)
@@ -32,7 +33,7 @@ Lightning-fast file system indexer and search tool.
Scan options
-t, --threads=<int> Number of threads. DEFAULT=1
-q, --quality=<flt> Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5
-q, --quality=<flt> Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=3
--size=<int> Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
--content-size=<int> Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
--incremental=<str> Reuse an existing index and only scan modified files.
@@ -41,12 +42,15 @@ Scan options
--name=<str> Index display name. DEFAULT: (name of the directory)
--depth=<int> Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
--archive=<str> Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
--ocr=<str> Tesseract language (use tesseract --list-langs to see which are installed on your machine)
--archive-passphrase=<str> Passphrase for encrypted archive files
# TODO: add new --ocr-* options here
-e, --exclude=<str> Files that match this regex will not be scanned
--fast Only index file names & mime type
--treemap-threshold=<str> Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
--mem-buffer=<int> Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
--read-subtitles Read subtitles from media files
--read-subtitles Read subtitles from media files.
--fast-epub Faster but less accurate EPUB parsing (no thumbnails, metadata)
--checksums Calculate file checksums when scanning.
Index options
-t, --threads=<int> Number of threads. DEFAULT=1
@@ -66,13 +70,14 @@ Web options
--bind=<str> Listen on this address. DEFAULT=localhost:4090
--auth=<str> Basic auth in user:password format
--tag-auth=<str> Basic auth in user:password format for tagging
--tagline=<str> Tagline in navbar
--dev Serve html & js files from disk (for development)
Exec-script options
--es-url=<str> Elasticsearch url. DEFAULT=http://localhost:9200
--es-index=<str> Elasticsearch index name. DEFAULT=sist2
--script-file=<str> Path to user script.
--async-script Execute user script asynchronously.
Made by simon987 <me@simon987.net>. Released under GPL-3.0
```
## Scan
@@ -80,9 +85,9 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
### Scan options
* `-t, --threads`
Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-WmiObject Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
* `-q, --quality`
Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. *Does not affect PDF thumbnails quality*
Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best.
* `--size`
Thumbnail size in pixels.
* `--content-size`
@@ -125,6 +130,10 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
* `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
* `--checksums` Calculate file checksums (sha1) when scanning files. This option does not cause any additional read
operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
### Scan examples
@@ -145,15 +154,11 @@ sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
### Index format
A typical `binary` type index structure looks like this:
A typical `ndjson` type index structure looks like this:
```
documents.idx/
├── descriptor.json
├── _index_139965416830720
├── _index_139965425223424
├── _index_139965433616128
├── _index_139965442008832
├── _index_139965442008832
├── _index_main.ndjson.zst
├── treemap.csv
├── agg_mime.csv
├── agg_date.csv
@@ -169,9 +174,7 @@ documents.idx/
└── lock.mdb
```
The `_index_*` files contain the raw binary index data and are not meant to be
read by other applications. The format is generally compatible across different
sist2 versions.
The `_index_*.ndjson.zst` files contain the document data in JSON format, in a compressed newline-delemited file.
The `thumbs/` folder is a [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database)
database containing the thumbnails.
@@ -181,66 +184,6 @@ following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rew
The `.csv` are pre-computed aggregations necessary for the stats page.
*Advanced usage*
Instead of using the `scan` module, you can also import an index generated
by a third party application. The 'external' index must have the following format:
```
my_index/
├── descriptor.json
├── _index_0
└── thumbs/
| ├── data.mdb
| └── lock.mdb
└── meta/
└── <empty>
```
*descriptor.json*:
```json
{
"uuid": "<valid UUID4>",
"version": "_external_v1",
"root": "(optional)",
"name": "<name>",
"rewrite_url": "(optional)",
"type": "json",
"timestamp": 1578971024
}
```
*_index_0*: NDJSON format (One json object per line)
```json
{
"_id": "unique uuid for the file",
"index": "index uuid4 (same one as descriptor.json!)",
"mime": "application/x-cbz",
"size": 14341204,
"mtime": 1578882996,
"extension": "cbz",
"name": "my_book",
"path": "path/to/books",
"content": "text contents of the book",
"title": "Title of the book",
"tag": ["genre.fiction", "author.someguy", "etc..."],
"_keyword": [
{"k": "ISBN", "v": "ABCD34789231"}
],
"_text": [
{"k": "other", "v": "This will be indexed as text"}
]
}
```
You can find the full list of supported fields [here](../src/io/serialize.c#L90)
The `_keyword.*` items will be indexed and searchable as **keyword** fields (only full matches allowed).
The `_text.*` items will be indexed and searchable as **text** fields (fuzzy searching allowed)
*thumbs/*:
LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
@@ -248,9 +191,6 @@ and values are raw image bytes.
*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
Importing an external `binary` type index is technically possible but
it is currently unsupported and has no guaranties of back/forward compatibility.
## Index
### Index options
@@ -276,6 +216,7 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
down the process.
* `-f, --force-reset`
Reset Elasticsearch mappings and settings.
* `-t, --threads` Number of threads to use. Ideally, choose a number equal to the number of logical cores of the machine hosting Elasticsearch.
### Index examples
@@ -305,6 +246,8 @@ sist2 index --print ./my_index/ | jq | less
* `--auth=<str>` Basic auth in user:password format
* `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the
`--auth` argument, but authentication is only applied the `/tag/` endpoint.
* `--tagline=<str>` When specified, will replace the default tagline in the navbar.
* `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
### Web examples
@@ -324,14 +267,19 @@ sist2 web index1 index2 index3 index4
When the `rewrite_url` field is not empty, the web module ignores the `root`
field and will return a HTTP redirect to `<rewrite_url><path>/<name><extension>`
instead of serving the file from disk.
Both the `root` and `rewrite_url` fields are safe to manually modify from the
Both the `root` and `rewrite_url` fields are safe to manually modify from the
`descriptor.json` file.
### Link to specific indices
# Elasticsearch
To link to specific indices, you can add a list of comma-separated index name to
the URL: `?i=<name>,<name>`. By default, indices with `"(nsfw)"` in their name are
not displayed.
Elasticsearch versions >=6.8.0, <8.0.0 are supported by sist2.
Using a version >=7.14.0 is recommended to enable the following features:
- Bug fix for large documents (See #198)
When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
If you don't care about the features above, you can ignore it or disable it in the configuration page.
## exec-script
@@ -367,7 +315,7 @@ See [scripting](scripting.md) documentation.
# Sidecar files
When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the
original document's metadata. Sidecar metadata files will also work inside archives.
original document's indexed metadata (does not modify the actual file). Sidecar metadata files will also work inside archives.
Sidecar files themselves are not saved in the index.
This feature is useful to leverage third-party applications such as speech-to-text or

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.9 KiB

After

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 889 KiB

After

Width:  |  Height:  |  Size: 1011 KiB

View File

@@ -4,6 +4,10 @@
"type": "keyword",
"doc_values": true
},
"checksum": {
"type": "keyword",
"index": false
},
"_depth": {
"type": "integer"
},
@@ -47,7 +51,7 @@
"index": false
},
"duration": {
"type": "float",
"type": "integer",
"index": false
},
"width": {
@@ -74,6 +78,7 @@
"name": {
"analyzer": "content_analyzer",
"type": "text",
"fielddata": true,
"fields": {
"nGram": {
"type": "text",
@@ -134,7 +139,9 @@
}
},
"tag": {
"type": "keyword",
"type": "text",
"fielddata": true,
"analyzer": "tag_analyzer",
"copy_to": "suggest-tag"
},
"suggest-tag": {

View File

@@ -2,12 +2,18 @@
"index": {
"refresh_interval": "30s",
"codec": "best_compression",
"number_of_replicas": 0
"number_of_replicas": 0,
"highlight.max_analyzed_offset": 10000000
},
"analysis": {
"tokenizer": {
"path_tokenizer": {
"type": "path_hierarchy"
"type": "path_hierarchy",
"delimiter": "/"
},
"tag_tokenizer": {
"type": "path_hierarchy",
"delimiter": "."
},
"my_nGram_tokenizer": {
"type": "nGram",
@@ -22,6 +28,12 @@
"lowercase"
]
},
"tag_analyzer": {
"tokenizer": "tag_tokenizer",
"filter": [
"lowercase"
]
},
"case_insensitive_kw_analyzer": {
"tokenizer": "keyword",
"filter": [

View File

@@ -0,0 +1,58 @@
{
"index": {
"refresh_interval": "30s",
"codec": "best_compression",
"number_of_replicas": 0
},
"analysis": {
"tokenizer": {
"path_tokenizer": {
"type": "path_hierarchy",
"delimiter": "/"
},
"tag_tokenizer": {
"type": "path_hierarchy",
"delimiter": "."
},
"my_nGram_tokenizer": {
"type": "nGram",
"min_gram": 3,
"max_gram": 3
}
},
"analyzer": {
"path_analyzer": {
"tokenizer": "path_tokenizer",
"filter": [
"lowercase"
]
},
"tag_analyzer": {
"tokenizer": "tag_tokenizer",
"filter": [
"lowercase"
]
},
"case_insensitive_kw_analyzer": {
"tokenizer": "keyword",
"filter": [
"lowercase"
]
},
"my_nGram": {
"tokenizer": "my_nGram_tokenizer",
"filter": [
"lowercase",
"asciifolding"
]
},
"content_analyzer": {
"tokenizer": "standard",
"filter": [
"lowercase",
"asciifolding"
]
}
}
}
}

View File

@@ -2,16 +2,8 @@
rm -rf index.sist2/
rm src/static/js/bundle.js 2> /dev/null
cat `ls src/static/js/*.min.js` > src/static/js/bundle.js
cat src/static/js/{util,dom}.js >> src/static/js/bundle.js
rm src/static/css/bundle*.css 2> /dev/null
cat src/static/css/*.min.css > src/static/css/bundle.css
cat src/static/css/light.css >> src/static/css/bundle.css
cat src/static/css/*.min.css > src/static/css/bundle_dark.css
cat src/static/css/dark.css >> src/static/css/bundle_dark.css
python3 scripts/mime.py > src/parsing/mime_generated.c
python3 scripts/serve_static.py > src/web/static_generated.c
python3 scripts/index_static.py > src/index/static_generated.c
printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h

17
scripts/build.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/env bash
VCPKG_ROOT="/vcpkg"
git submodule update --init --recursive
rm -rf CMakeFiles CMakeCache.txt
cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
make -j $(nproc)
strip sist2
./sist2 -v > VERSION
mv sist2 sist2-x64-linux
rm -rf CMakeFiles CMakeCache.txt
cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
make -j $(nproc)
mv sist2_debug sist2-x64-linux-debug

17
scripts/build_arm64.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/env bash
VCPKG_ROOT="/vcpkg"
git submodule update --init --recursive
rm -rf CMakeFiles CMakeCache.txt
cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
make -j $(nproc)
strip sist2
mv sist2 sist2-arm64-linux
rm -rf CMakeFiles CMakeCache.txt
cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
make -j $(nproc)
strip sist2
mv sist2_debug sist2-arm64-linux-debug

View File

@@ -3,6 +3,7 @@ import json
files = [
"schema/mappings.json",
"schema/settings.json",
"schema/settings_legacy.json",
"schema/pipeline.json",
]

View File

@@ -22,6 +22,7 @@ application/java-archive, jar
application/java, class
application/javascript,
application/json, json
application/ndjson, jsonl|ndjson
application/marc, mrc
application/mbedlet, mbd
application/mime, aps
@@ -78,9 +79,7 @@ application/vocaltec-media-desc, vmd
application/vocaltec-media-file, vmf
application/warc, warc
application/winhelp, hlp
application/wordperfect6.0, w60
application/wordperfect6.1, w61
application/wordperfect, wp|wp5|wp6|wpd
application/wordperfect, wp|wp5|wp6|wpd|w60|w61
application/x-123, wk1
application/x-7z-compressed, 7z
application/x-aim, aim
@@ -157,7 +156,6 @@ application/x-livescreen, ivy
application/x-lotus, wq1
application/x-lz4+json, jsonlz4
application/x-lz4, lz4
application/x-lz4, lz4
application/x-lzh-compressed,
application/x-lzh, lzh
application/x-lzip, lz
1 application/arj arj
22 application/java class
23 application/javascript
24 application/json json
25 application/ndjson jsonl|ndjson
26 application/marc mrc
27 application/mbedlet mbd
28 application/mime aps
79 application/vocaltec-media-file vmf
80 application/warc warc
81 application/winhelp hlp
82 application/wordperfect6.0 application/wordperfect w60 wp|wp5|wp6|wpd|w60|w61
application/wordperfect6.1 w61
application/wordperfect wp|wp5|wp6|wpd
83 application/x-123 wk1
84 application/x-7z-compressed 7z
85 application/x-aim aim
156 application/x-lotus wq1
157 application/x-lz4+json jsonlz4
158 application/x-lz4 lz4
application/x-lz4 lz4
159 application/x-lzh-compressed
160 application/x-lzh lzh
161 application/x-lzip lz

View File

@@ -1,12 +1,10 @@
files = [
"src/static/css/bundle.css",
"src/static/css/bundle_dark.css",
"src/static/js/bundle.js",
"src/static/js/search.js",
"src/static/img/sprite-skin-flat.png",
"src/static/img/sprite-skin-flat-dark.png",
"src/static/search.html",
"src/static/stats.html",
"sist2-vue/src/assets/favicon.ico",
"sist2-vue/dist/css/chunk-vendors.css",
"sist2-vue/dist/css/index.css",
"sist2-vue/dist/js/chunk-vendors.js",
"sist2-vue/dist/js/index.js",
"sist2-vue/dist/index.html",
]
@@ -15,6 +13,10 @@ def clean(filepath):
for file in files:
with open(file, "rb") as f:
data = f.read()
try:
with open(file, "rb") as f:
data = f.read()
except:
data = bytes([])
print("char %s[%d] = {%s};" % (clean(file), len(data), ",".join(str(int(b)) for b in data)))

23
sist2-vue/.gitignore vendored Normal file
View File

@@ -0,0 +1,23 @@
.DS_Store
node_modules
# local env files
.env.local
.env.*.local
# Log files
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
# Editor directories and files
.idea
.vscode
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
*.iml

View File

@@ -0,0 +1,5 @@
module.exports = {
"presets": [
"@vue/cli-plugin-babel/preset"
]
}

9
sist2-vue/dist/css/chunk-vendors.css vendored Normal file

File diff suppressed because one or more lines are too long

1
sist2-vue/dist/css/index.css vendored Normal file

File diff suppressed because one or more lines are too long

3
sist2-vue/dist/index.html vendored Normal file
View File

@@ -0,0 +1,3 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>sist2</title><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="css/index.css" rel="preload" as="style"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
height: initial;
}</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/index.js"></script></body></html>

146
sist2-vue/dist/js/chunk-vendors.js vendored Normal file

File diff suppressed because one or more lines are too long

1
sist2-vue/dist/js/index.js vendored Normal file

File diff suppressed because one or more lines are too long

27250
sist2-vue/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

54
sist2-vue/package.json Normal file
View File

@@ -0,0 +1,54 @@
{
"name": "sist2",
"version": "2.11.0",
"private": true,
"scripts": {
"serve": "vue-cli-service serve",
"build": "vue-cli-service build --mode production"
},
"dependencies": {
"@egjs/vue-infinitegrid": "3.3.0",
"axios": "^0.21.1",
"bootstrap-vue": "^2.21.2",
"core-js": "^3.6.5",
"crypto-es": "^1.2.7",
"d3": "^5.16.0",
"date-fns": "^2.21.3",
"dom-to-image": "^2.6.0",
"fslightbox-vue": "file:../../../mnt/Hatchery/main/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
"nouislider": "^15.2.0",
"underscore": "^1.13.1",
"vue": "^2.6.12",
"vue-color": "^2.8.1",
"vue-i18n": "^8.24.4",
"vue-masonry-wall": "^0.3.2",
"vue-router": "^3.2.0",
"vue-simple-suggest": "^1.11.1",
"vuex": "^3.4.0"
},
"devDependencies": {
"@babel/polyfill": "^7.11.5",
"@vue/cli-plugin-babel": "~4.5.0",
"@vue/cli-plugin-router": "~4.5.0",
"@vue/cli-plugin-typescript": "~4.5.0",
"@vue/cli-plugin-vuex": "~4.5.0",
"@vue/cli-service": "~4.5.0",
"@vue/test-utils": "^1.0.3",
"bootstrap": "^4.5.2",
"inspire-tree": "^4.3.1",
"inspire-tree-dom": "^4.0.6",
"mutationobserver-shim": "^0.3.7",
"popper.js": "^1.16.1",
"portal-vue": "^2.1.7",
"sass": "^1.26.11",
"sass-loader": "^10.0.2",
"typescript": "~4.1.5",
"vue-cli-plugin-bootstrap-vue": "~0.7.0",
"vue-template-compiler": "^2.6.11"
},
"browserslist": [
"> 1%",
"last 2 versions",
"not dead"
]
}

View File

@@ -0,0 +1,32 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
<title><%= htmlWebpackPlugin.options.title %></title>
</head>
<body>
<noscript>
<style>
body {
height: initial;
}
</style>
<div style="text-align: center; margin-top: 100px">
<strong>
We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
Please enable it to continue.
</strong>
<br/>
<strong>
Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
si JavaScript est activé.
Veuillez l'activer pour continuer.
</strong>
</div>
</noscript>
<div id="app"></div>
</body>
</html>

312
sist2-vue/src/App.vue Normal file
View File

@@ -0,0 +1,312 @@
<template>
<div id="app" :class="getClass()">
<NavBar></NavBar>
<router-view v-if="!configLoading"/>
</div>
</template>
<script>
import NavBar from "@/components/NavBar";
import {mapGetters} from "vuex";
export default {
components: {NavBar},
data() {
return {
configLoading: false
}
},
computed: {
...mapGetters(["optTheme"]),
},
mounted() {
this.$store.dispatch("loadConfiguration").then(() => {
this.$root.$i18n.locale = this.$store.state.optLang;
});
this.$store.subscribe((mutation) => {
if (mutation.type === "setOptLang") {
this.$root.$i18n.locale = mutation.payload;
this.configLoading = true;
window.setTimeout(() => this.configLoading = false, 10);
}
});
},
methods: {
getClass() {
return {
"theme-light": this.optTheme === "light",
"theme-black": this.optTheme === "black",
}
}
}
,
}
</script>
<style>
html, body {
height: 100%;
}
#app {
/*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
/*text-align: center;*/
color: #2c3e50;
padding-bottom: 1em;
min-height: 100%;
}
/*Black theme*/
.theme-black {
background-color: #000;
}
.theme-black .card, .theme-black .modal-content {
background: #212121;
color: #e0e0e0;
border-radius: 1px;
border: none;
}
.theme-black .table {
color: #e0e0e0;
}
.theme-black .table td, .theme-black .table th {
border: none;
}
.theme-black .table thead th {
border-bottom: 1px solid #646464;
}
.theme-black .custom-select {
overflow: auto;
background-color: #37474F;
border: 1px solid #616161;
color: #bdbdbd;
}
.theme-black .custom-select:focus {
border-color: #757575;
outline: 0;
box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
}
.theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow {
background: none !important;
}
.theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before {
background-color: black !important;
}
.theme-black .inspire-tree .title {
color: #eee;
}
.theme-black .inspire-tree {
font-weight: 400;
font-size: 14px;
font-family: Helvetica, Nueue, Verdana, sans-serif;
max-height: 350px;
overflow: auto;
}
.inspire-tree [type=checkbox] {
left: 22px !important;
top: 7px !important;
}
.theme-black .form-control {
background-color: #37474F;
border: 1px solid #616161;
color: #dbdbdb !important;
}
.theme-black .form-control:focus {
background-color: #546E7A;
color: #fff;
}
.theme-black .input-group-text, .theme-black .default-input {
background: #37474F !important;
border: 1px solid #616161 !important;
color: #dbdbdb !important;
}
.theme-black ::placeholder {
color: #BDBDBD !important;
opacity: 1;
}
.theme-black .nav-tabs .nav-link {
color: #e0e0e0;
}
.theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
background-color: #212121;
border-color: #616161 #616161 #212121;
color: #e0e0e0;
}
.theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus {
border-color: #616161 #616161 #212121;
color: #e0e0e0;
}
.theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover {
border-color: #e0e0e0 #e0e0e0 #212121;
color: #e0e0e0;
}
.theme-black .nav-tabs {
border-bottom: #616161;
}
.theme-black a:hover, .theme-black .btn:hover {
color: #fff;
}
.theme-black .b-dropdown a:hover {
color: inherit;
}
.theme-black .btn {
color: #eee;
}
.theme-black .modal-header .close {
color: #e0e0e0;
text-shadow: none;
}
.theme-black .modal-header {
border-bottom: 1px solid #646464;
}
/* -------------------------- */
#nav {
padding: 30px;
}
#nav a {
font-weight: bold;
color: #2c3e50;
}
#nav a.router-link-exact-active {
color: #42b983;
}
.mobile {
display: none;
}
.container {
padding-top: 1em;
}
@media (max-width: 650px) {
.mobile {
display: initial;
}
.not-mobile {
display: none;
}
.grid-single-column .fit {
max-height: none !important;
}
.container {
padding-left: 0;
padding-right: 0;
padding-top: 0
}
.lightbox-caption {
display: none;
}
}
.info-icon {
width: 1rem;
margin-right: 0.2rem;
cursor: pointer;
line-height: 1rem;
height: 1rem;
background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
filter: brightness(45%);
display: block;
}
.tabs {
margin-top: 10px;
}
.modal-title {
text-overflow: ellipsis;
overflow: hidden;
white-space: nowrap;
}
@media screen and (min-width: 1500px) {
.container {
max-width: 1440px;
}
}
.noUi-connects {
border-radius: 1px !important;
}
mark {
background: #fff217;
border-radius: 0;
padding: 1px 0;
color: inherit;
}
.theme-black mark {
background: rgba(251, 191, 41, 0.25);
border-radius: 0;
padding: 1px 0;
color: inherit;
}
.theme-black .content-div mark {
background: rgba(251, 191, 41, 0.40);
color: white;
}
.content-div {
font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
font-size: 13px;
padding: 1em;
background-color: #f5f5f5;
border: 1px solid #ccc;
border-radius: 4px;
margin: 3px;
white-space: normal;
color: #000;
overflow: hidden;
}
.theme-black .content-div {
background-color: #37474F;
border: 1px solid #616161;
color: #E0E0E0FF;
}
.graph {
display: inline-block;
width: 40%;
}
</style>

406
sist2-vue/src/Sist2Api.ts Normal file
View File

@@ -0,0 +1,406 @@
import axios from "axios";
import {ext, strUnescape, lum} from "./util";
import CryptoES from 'crypto-es';
export interface EsTag {
id: string
count: number
color: string | undefined
isLeaf: boolean
}
export interface Tag {
style: string
text: string
rawText: string
fg: string
bg: string
userTag: boolean
}
export interface Index {
name: string
version: string
id: string
idPrefix: string
timestamp: number
}
export interface EsHit {
_index: string
_id: string
_score: number
_path_md5: string
_type: string
_tags: Tag[]
_seq: number
_source: {
path: string
size: number
mime: string
name: string
extension: string
index: string
_depth: number
mtime: number
videoc: string
audioc: string
parent: string
width: number
height: number
duration: number
tag: string[]
checksum: string
thumbnail: string
}
_props: {
isSubDocument: boolean
isImage: boolean
isGif: boolean
isVideo: boolean
isPlayableVideo: boolean
isPlayableImage: boolean
isAudio: boolean
hasThumbnail: boolean
tnW: number
tnH: number
}
highlight: {
name: string[] | undefined,
content: string[] | undefined,
}
}
function getIdPrefix(indices: Index[], id: string): string {
for (let i = 4; i < 32; i++) {
const prefix = id.slice(0, i);
if (indices.filter(idx => idx.id.slice(0, i) == prefix).length == 1) {
return prefix;
}
}
return id;
}
export interface EsResult {
took: number
hits: {
// TODO: ES 6.X ?
total: {
value: number
}
hits: EsHit[]
}
aggregations: any
}
class Sist2Api {
private baseUrl: string
constructor(baseUrl: string) {
this.baseUrl = baseUrl;
}
getSist2Info(): Promise<any> {
return axios.get(`${this.baseUrl}i`).then(resp => {
const indices = resp.data.indices as Index[];
resp.data.indices = indices.map(idx => {
return {
id: idx.id,
name: idx.name,
timestamp: idx.timestamp,
version: idx.version,
idPrefix: getIdPrefix(indices, idx.id)
} as Index;
});
return resp.data;
})
}
setHitProps(hit: EsHit): void {
hit["_props"] = {} as any;
const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
if ("parent" in hit._source) {
hit._props.isSubDocument = true;
}
if ("thumbnail" in hit._source) {
hit._props.hasThumbnail = true;
hit._props.tnW = Number(hit._source.thumbnail.split(",")[0]);
hit._props.tnH = Number(hit._source.thumbnail.split(",")[1]);
}
switch (mimeCategory) {
case "image":
if (hit._source.videoc === "gif") {
hit._props.isGif = true;
} else {
hit._props.isImage = true;
}
if ("width" in hit._source && !hit._props.isSubDocument && hit._source.videoc !== "tiff"
&& hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
hit._props.isPlayableImage = true;
}
break;
case "video":
if ("videoc" in hit._source) {
hit._props.isVideo = true;
}
if (hit._props.isVideo) {
const videoc = hit._source.videoc;
const mime = hit._source.mime;
hit._props.isPlayableVideo = mime != null &&
mime.startsWith("video/") &&
!hit._props.isSubDocument &&
hit._source.extension !== "mkv" &&
hit._source.extension !== "avi" &&
hit._source.extension !== "mov" &&
videoc !== "hevc" &&
videoc !== "mpeg1video" &&
videoc !== "mpeg2video" &&
videoc !== "wmv3";
}
break;
case "audio":
if ("audioc" in hit._source && !hit._props.isSubDocument) {
hit._props.isAudio = true;
}
break;
}
}
setHitTags(hit: EsHit): void {
const tags = [] as Tag[];
const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
switch (mimeCategory) {
case "image":
case "video":
if ("videoc" in hit._source && hit._source.videoc) {
tags.push({
style: "video",
text: hit._source.videoc.replace(" ", ""),
userTag: false
} as Tag);
}
break
case "audio":
if ("audioc" in hit._source && hit._source.audioc) {
tags.push({
style: "audio",
text: hit._source.audioc,
userTag: false
} as Tag);
}
break;
}
// User tags
if ("tag" in hit._source) {
hit._source.tag.forEach(tag => {
tags.push(this.createUserTag(tag));
})
}
hit._tags = tags;
}
createUserTag(tag: string): Tag {
const tokens = tag.split(".");
const colorToken = tokens.pop() as string;
const bg = colorToken;
const fg = lum(colorToken) > 50 ? "#000" : "#fff";
return {
style: "user",
fg: fg,
bg: bg,
text: tokens.join("."),
rawText: tag,
userTag: true,
} as Tag;
}
esQuery(query: any): Promise<EsResult> {
return axios.post(`${this.baseUrl}es`, query).then(resp => {
const res = resp.data as EsResult;
if (res.hits?.hits) {
res.hits.hits.forEach((hit: EsHit) => {
hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
hit["_path_md5"] = CryptoES.MD5(
hit["_source"]["path"] +
(hit["_source"]["path"] ? "/" : "") +
hit["_source"]["name"] + ext(hit)
).toString();
this.setHitProps(hit);
this.setHitTags(hit);
});
}
return res;
});
}
getMimeTypes(query = undefined) {
const AGGS = {
mimeTypes: {
terms: {
field: "mime",
size: 10000
}
}
};
if (!query) {
query = {
aggs: AGGS,
size: 0,
};
} else {
query.size = 0;
query.aggs = AGGS;
}
return this.esQuery(query).then(resp => {
const mimeMap: any[] = [];
const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
const tmp = bucket["key"].split("/");
const category = tmp[0];
const mime = tmp[1];
let category_exists = false;
const child = {
"id": bucket["key"],
"text": `${mime} (${bucket["doc_count"]})`
};
mimeMap.forEach(node => {
if (node.text === category) {
node.children.push(child);
category_exists = true;
}
});
if (!category_exists) {
mimeMap.push({text: category, children: [child], id: category});
}
})
mimeMap.forEach(node => {
if (node.children) {
node.children.sort((a, b) => a.id.localeCompare(b.id));
}
})
mimeMap.sort((a, b) => a.id.localeCompare(b.id))
return {buckets, mimeMap};
});
}
_createEsTag(tag: string, count: number): EsTag {
const tokens = tag.split(".");
if (/.*\.#[0-9a-f]{6}/.test(tag)) {
return {
id: tokens.slice(0, -1).join("."),
color: tokens.pop(),
isLeaf: true,
count: count
};
}
return {
id: tag,
count: count,
isLeaf: false,
color: undefined
};
}
getDocInfo(docId: string) {
return axios.get(`${this.baseUrl}d/${docId}`);
}
getTags() {
return this.esQuery({
aggs: {
tags: {
terms: {
field: "tag",
size: 10000
}
}
},
size: 0,
}).then(resp => {
const seen = new Set();
const tags = resp["aggregations"]["tags"]["buckets"]
.sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
.map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
// Remove duplicates (same tag with different color)
return tags.filter((t: EsTag) => {
if (seen.has(t.id)) {
return false;
}
seen.add(t.id);
return true;
});
});
}
saveTag(tag: string, hit: EsHit) {
return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
delete: false,
name: tag,
doc_id: hit["_id"],
path_md5: hit._path_md5
});
}
deleteTag(tag: string, hit: EsHit) {
return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
delete: true,
name: tag,
doc_id: hit["_id"],
path_md5: hit._path_md5
});
}
getTreemapCsvUrl(indexId: string) {
return `${this.baseUrl}s/${indexId}/1`;
}
getMimeCsvUrl(indexId: string) {
return `${this.baseUrl}s/${indexId}/2`;
}
getSizeCsv(indexId: string) {
return `${this.baseUrl}s/${indexId}/3`;
}
getDateCsv(indexId: string) {
return `${this.baseUrl}s/${indexId}/4`;
}
}
export default new Sist2Api("");

249
sist2-vue/src/Sist2Query.ts Normal file
View File

@@ -0,0 +1,249 @@
import store from "./store";
import {EsHit, Index} from "@/Sist2Api";
const SORT_MODES = {
score: {
mode: [
{_score: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._score
},
random: {
mode: [
{_score: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._score
},
dateAsc: {
mode: [
{mtime: {order: "asc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.mtime
},
dateDesc: {
mode: [
{mtime: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.mtime
},
sizeAsc: {
mode: [
{size: {order: "asc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.size
},
sizeDesc: {
mode: [
{size: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.size
},
nameAsc: {
mode: [
{name: {order: "asc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.name
},
nameDesc: {
mode: [
{name: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.name
}
} as any;
interface SortMode {
text: string
mode: any[]
key: (hit: EsHit) => any
}
class Sist2Query {
searchQuery(): any {
const getters = store.getters;
const searchText = getters.searchText;
const pathText = getters.pathText;
const empty = searchText === "";
const sizeMin = getters.sizeMin;
const sizeMax = getters.sizeMax;
const dateMin = getters.dateMin;
const dateMax = getters.dateMax;
const fuzzy = getters.fuzzy;
const size = getters.size;
const after = getters.lastDoc;
const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
const selectedMimeTypes = getters.selectedMimeTypes;
const selectedTags = getters.selectedTags;
const legacyES = store.state.sist2Info.esVersionLegacy;
const filters = [
{terms: {index: selectedIndexIds}}
] as any[];
if (sizeMin && sizeMax) {
filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
} else if (sizeMin) {
filters.push({range: {size: {gte: sizeMin}}})
} else if (sizeMax) {
filters.push({range: {size: {lte: sizeMax}}})
}
if (dateMin && dateMax) {
filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
} else if (dateMin) {
filters.push({range: {mtime: {gte: dateMin}}})
} else if (dateMax) {
filters.push({range: {mtime: {lte: dateMax}}})
}
const fields = [
"name^8",
"content^3",
"album^8", "artist^8", "title^8", "genre^2", "album_artist^8",
"font_name^6"
];
if (getters.optSearchInPath) {
fields.push("path.text^5");
}
if (fuzzy) {
fields.push("content.nGram");
if (getters.optSearchInPath) {
fields.push("path.nGram");
}
fields.push("name.nGram^3");
}
const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
if (path !== "") {
filters.push({term: {path: path}})
}
if (selectedMimeTypes.length > 0) {
filters.push({terms: {"mime": selectedMimeTypes}});
}
if (selectedTags.length > 0) {
if (getters.optTagOrOperator) {
filters.push({terms: {"tag": selectedTags}});
} else {
selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
}
}
let query;
if (getters.optQueryMode === "simple") {
query = {
simple_query_string: {
query: searchText,
fields: fields,
default_operator: "and"
}
}
} else {
query = {
query_string: {
query: searchText,
default_field: "name",
default_operator: "and"
}
}
}
const q = {
_source: {
excludes: ["content", "_tie"]
},
query: {
bool: {
filter: filters,
}
},
sort: SORT_MODES[getters.sortMode].mode,
aggs:
{
total_size: {"sum": {"field": "size"}},
total_count: {"value_count": {"field": "size"}}
},
size: size,
} as any;
if (!empty) {
q.query.bool.must = query;
}
if (after) {
q.search_after = [SORT_MODES[getters.sortMode].key(after), after["_id"]];
}
if (getters.optHighlight) {
q.highlight = {
pre_tags: ["<mark>"],
post_tags: ["</mark>"],
fragment_size: getters.optFragmentSize,
number_of_fragments: 1,
order: "score",
fields: {
content: {},
name: {},
"name.nGram": {},
"content.nGram": {},
font_name: {},
}
};
if (!legacyES) {
q.highlight.max_analyzed_offset = 9_999_999;
}
if (getters.optSearchInPath) {
q.highlight.fields["path.text"] = {};
q.highlight.fields["path.nGram"] = {};
}
}
if (getters.sortMode === "random") {
q.query = {
function_score: {
query: {
bool: {
must: filters,
}
},
functions: [
{
random_score: {
seed: getters.seed,
field: "_seq_no",
},
weight: 1000
}
],
boost_mode: "sum"
}
}
if (!empty) {
q.query.function_score.query.bool.must.push(query);
}
}
return q;
}
}
export default new Sist2Query();

View File

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

View File

@@ -0,0 +1,28 @@
<template>
<div class="content-div" v-html="content()" v-if="content()"></div>
</template>
<script>
export default {
name: "ContentDiv",
props: ["doc"],
methods: {
content() {
if (!this.doc.highlight) {
return null;
}
if (this.doc.highlight["content.nGram"]) {
return this.doc.highlight["content.nGram"][0];
}
if (this.doc.highlight.content) {
return this.doc.highlight.content[0];
}
}
}
}
</script>
<style scoped>
</style>

View File

@@ -0,0 +1,129 @@
<template>
<div class="graph">
<svg id="date-histogram"></svg>
</div>
</template>
<script>
import * as d3 from "d3";
import Sist2Api from "@/Sist2Api";
const formatSI = d3.format("~s");
function dateHistogram(data, svg, title) {
let bins = data.map(d => {
return {
length: Number(d.count),
x0: Number(d.bucket),
x1: Number(d.bucket) + 2629800
}
});
bins.sort((a, b) => a.length - b.length);
const margin = {
top: 50,
right: 20,
bottom: 70,
left: 40
};
const thresh = d3.quantile(bins, 0.9, d => d.length);
bins = bins.filter(d => d.length > thresh);
const width = 550;
const height = 450;
svg.selectAll("*").remove();
svg.attr("viewBox", [0, 0, width, height]);
const y = d3.scaleLinear()
.domain([0, d3.max(bins, d => d.length)]).nice()
.range([height - margin.bottom, margin.top]);
const x = d3.scaleLinear()
.domain(d3.extent(bins, d => d.x0)).nice()
.range([margin.left, width - margin.right]);
svg.append("g")
.attr("fill", "steelblue")
.selectAll("rect")
.data(bins)
.join("rect")
.attr("x", d => x(d.x0) + 1)
.attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
.attr("y", d => y(d.length))
.attr("height", d => y(0) - y(d.length))
.call(g => g
.append("title")
.text(d => d.length)
);
svg.append("g")
.attr("transform", `translate(0,${height - margin.bottom})`)
.call(
d3.axisBottom(x)
.ticks(width / 30)
.tickSizeOuter(0)
.tickFormat(t => d3.timeFormat("%Y-%m-%d")(d3.utcParse("%s")(t)))
)
.call(g => g
.selectAll("text")
.style("text-anchor", "end")
.attr("dx", "-.8em")
.attr("dy", ".15em")
.attr("transform", "rotate(-65)")
)
.call(g => g.append("text")
.attr("x", width - margin.right)
.attr("y", -4)
.attr("fill", "currentColor")
.attr("font-weight", "bold")
.attr("text-anchor", "end")
.text("mtime")
);
svg.append("g")
.attr("transform", `translate(${margin.left},0)`)
.call(
d3.axisLeft(y)
.ticks(height / 40)
.tickFormat(t => formatSI(t))
)
.call(g => g.select(".domain").remove())
.call(g => g.select(".tick:last-of-type text").clone()
.attr("x", 4)
.attr("text-anchor", "start")
.attr("font-weight", "bold")
.text("File count"));
svg.append("text")
.attr("x", (width / 2))
.attr("y", (margin.top / 2))
.attr("text-anchor", "middle")
.style("font-size", "16px")
.text(title);
}
export default {
name: "D3DateHistogram",
props: ["indexId"],
mounted() {
this.update(this.indexId);
},
watch: {
indexId: function () {
this.update(this.indexId);
},
},
methods: {
update(indexId) {
const svg = d3.select("#date-histogram");
d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
});
}
}
}
</script>

View File

@@ -0,0 +1,100 @@
<template>
<div class="graph">
<svg id="agg-mime-count"></svg>
</div>
</template>
<script>
import * as d3 from "d3";
import Sist2Api from "@/Sist2Api";
const formatSI = d3.format("~s");
const barHeight = 20;
const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
function mimeBarCount(data, svg, fillOpacity, title) {
const margin = {
top: 50,
right: 0,
bottom: 10,
left: Math.max(
d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
)
};
data.forEach(d => {
d.name = d.mime;
d.value = Number(d.count);
});
data = data.sort((a, b) => b.value - a.value).slice(0, 15);
const width = 550;
const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
svg.selectAll("*").remove();
svg.attr("viewBox", [0, 0, width, height]);
const y = d3.scaleBand()
.domain(d3.range(data.length))
.rangeRound([margin.top, height - margin.bottom]);
const x = d3.scaleLinear()
.domain([0, d3.max(data, d => d.value)])
.range([margin.left, width - margin.right]);
svg.append("g")
.attr("fill-opacity", fillOpacity)
.selectAll("rect")
.data(data)
.join("rect")
.attr("fill", d => ordinalColor(d.name))
.attr("x", x(0))
.attr("y", (d, i) => y(i))
.attr("width", d => x(d.value) - x(0))
.attr("height", y.bandwidth())
.append("title")
.text(d => d3.format(",")(d.value));
svg.append("g")
.attr("transform", `translate(0,${margin.top})`)
.call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
.call(g => g.select(".domain").remove());
svg.append("g")
.attr("transform", `translate(${margin.left},0)`)
.call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
svg.append("text")
.attr("x", (width / 2))
.attr("y", (margin.top / 2))
.attr("text-anchor", "middle")
.style("font-size", "16px")
.text(title);
}
export default {
name: "D3MimeBarSize",
props: ["indexId"],
mounted() {
this.update(this.indexId);
},
watch: {
indexId: function () {
this.update(this.indexId);
},
},
methods: {
update(indexId) {
const mimeSvgCount = d3.select("#agg-mime-count");
const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
});
}
}
}
</script>

View File

@@ -0,0 +1,99 @@
<template>
<div class="graph">
<svg id="agg-mime-size"></svg>
</div>
</template>
<script>
import * as d3 from "d3";
import Sist2Api from "@/Sist2Api";
const formatSI = d3.format("~s");
const barHeight = 20;
const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
function mimeBarSize(data, svg, fillOpacity, title) {
const margin = {
top: 50,
right: 0,
bottom: 10,
left: Math.max(
d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
)
};
data.forEach(d => {
d.name = d.mime;
d.value = Number(d.size);
});
data = data.sort((a, b) => b.value - a.value).slice(0, 15);
const width = 550;
const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
svg.selectAll("*").remove();
svg.attr("viewBox", [0, 0, width, height]);
const y = d3.scaleBand()
.domain(d3.range(data.length))
.rangeRound([margin.top, height - margin.bottom]);
const x = d3.scaleLinear()
.domain([0, d3.max(data, d => d.value)])
.range([margin.left, width - margin.right]);
svg.append("g")
.attr("fill-opacity", fillOpacity)
.selectAll("rect")
.data(data)
.join("rect")
.attr("fill", d => ordinalColor(d.name))
.attr("x", x(0))
.attr("y", (d, i) => y(i))
.attr("width", d => x(d.value) - x(0))
.attr("height", y.bandwidth())
.append("title")
.text(d => formatSI(d.value));
svg.append("g")
.attr("transform", `translate(0,${margin.top})`)
.call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
.call(g => g.select(".domain").remove());
svg.append("g")
.attr("transform", `translate(${margin.left},0)`)
.call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
svg.append("text")
.attr("x", (width / 2))
.attr("y", (margin.top / 2))
.attr("text-anchor", "middle")
.style("font-size", "16px")
.text(title);
}
export default {
name: "D3MimeBarSize",
props: ["indexId"],
mounted() {
this.update(this.indexId);
},
watch: {
indexId: function () {
this.update(this.indexId);
},
},
methods: {
update(indexId) {
const mimeSvgSize = d3.select("#agg-mime-size");
const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
});
}
}
}
</script>

View File

@@ -0,0 +1,126 @@
<template>
<div class="graph">
<svg id="size-histogram"></svg>
</div>
</template>
<script>
import * as d3 from "d3";
import Sist2Api from "@/Sist2Api";
const formatSI = d3.format("~s");
function sizeHistogram(data, svg, title) {
let bins = data.map(d => {
return {
length: Number(d.count),
x0: Number(d.bucket),
x1: Number(d.bucket) + (5 * 1024 * 1024)
}
});
bins = bins.sort((a, b) => b.length - a.length).slice(0, 25);
const margin = {
top: 50,
right: 20,
bottom: 70,
left: 40
};
const width = 550;
const height = 450;
svg.selectAll("*").remove();
svg.attr("viewBox", [0, 0, width, height]);
const y = d3.scaleLinear()
.domain([0, d3.max(bins, d => d.length)])
.range([height - margin.bottom, margin.top]);
const x = d3.scaleLinear()
.domain(d3.extent(bins, d => d.x0)).nice()
.range([margin.left, width - margin.right]);
svg.append("g")
.attr("fill", "steelblue")
.selectAll("rect")
.data(bins)
.join("rect")
.attr("x", d => x(d.x0) + 1)
.attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
.attr("y", d => y(d.length))
.attr("height", d => y(0) - y(d.length))
.call(g => g
.append("title")
.text(d => d.length)
);
svg.append("g")
.attr("transform", `translate(0,${height - margin.bottom})`)
.call(
d3.axisBottom(x)
.ticks(width / 30)
.tickSizeOuter(0)
.tickFormat(formatSI)
)
.call(g => g
.selectAll("text")
.style("text-anchor", "end")
.attr("dx", "-.8em")
.attr("dy", ".15em")
.attr("transform", "rotate(-65)")
)
.call(g => g.append("text")
.attr("x", width - margin.right)
.attr("y", -4)
.attr("fill", "currentColor")
.attr("font-weight", "bold")
.attr("text-anchor", "end")
.text("size (bytes)")
);
svg.append("g")
.attr("transform", `translate(${margin.left},0)`)
.call(
d3.axisLeft(y)
.ticks(height / 40)
.tickFormat(t => formatSI(t))
)
.call(g => g.select(".domain").remove())
.call(g => g.select(".tick:last-of-type text").clone()
.attr("x", 4)
.attr("text-anchor", "start")
.attr("font-weight", "bold")
.text("File count"));
svg.append("text")
.attr("x", (width / 2))
.attr("y", (margin.top / 2))
.attr("text-anchor", "middle")
.style("font-size", "16px")
.text(title);
}
export default {
name: "D3SizeHistogram",
props: ["indexId"],
mounted() {
this.update(this.indexId);
},
watch: {
indexId: function () {
this.update(this.indexId);
},
},
methods: {
update(indexId) {
const svg = d3.select("#size-histogram");
d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
});
}
}
}
</script>

View File

@@ -0,0 +1,267 @@
<template>
<div>
<b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
{{ $t("download") }}
</b-btn>
<svg id="treemap"></svg>
</div>
</template>
<script>
import * as d3 from "d3";
import {burrow} from "@/util-js"
import {humanFileSize} from "@/util";
import Sist2Api from "@/Sist2Api";
import domtoimage from "dom-to-image";
const TILING_MODES = {
"squarify": d3.treemapSquarify,
"binary": d3.treemapBinary,
"sliceDice": d3.treemapSliceDice,
"slice": d3.treemapSlice,
"dice": d3.treemapDice,
};
const COLORS = {
"PuBuGn": d3.interpolatePuBuGn,
"PuRd": d3.interpolatePuRd,
"PuBu": d3.interpolatePuBu,
"YlOrBr": d3.interpolateYlOrBr,
"YlOrRd": d3.interpolateYlOrRd,
"YlGn": d3.interpolateYlGn,
"YlGnBu": d3.interpolateYlGnBu,
"Plasma": d3.interpolatePlasma,
"Magma": d3.interpolateMagma,
"Inferno": d3.interpolateInferno,
"Viridis": d3.interpolateViridis,
"Turbo": d3.interpolateTurbo,
};
const SIZES = {
"small": [800, 600],
"medium": [1300, 750],
"large": [1900, 900],
"x-large": [2800, 1700],
"xx-large": [3600, 2000],
};
const uids = {};
function uid(name) {
let id = uids[name] || 0;
uids[name] = id + 1;
return name + id;
}
function cascade(root, offset) {
const x = new Map;
const y = new Map;
return root.eachAfter(d => {
if (d.children && d.children.length !== 0) {
x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
} else {
x.set(d, 0);
y.set(d, 0);
}
}).eachBefore(d => {
d.x1 -= 2 * offset * x.get(d);
d.y1 -= 2 * offset * y.get(d);
});
}
function cascadeTreemap(data, svg, width, height, tilingMode, treemapColor) {
const root = cascade(
d3.treemap()
.size([width, height])
.tile(TILING_MODES[tilingMode])
.paddingOuter(3)
.paddingTop(16)
.paddingInner(1)
.round(true)(
d3.hierarchy(data)
.sum(d => d.value)
.sort((a, b) => b.value - a.value)
),
3 // treemap.paddingOuter
);
const maxDepth = Math.max(...root.descendants().map(d => d.depth));
const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
svg.append("filter")
.attr("id", "shadow")
.append("feDropShadow")
.attr("flood-opacity", 0.3)
.attr("dx", 0)
.attr("stdDeviation", 3);
const node = svg.selectAll("g")
.data(
d3.nest()
.key(d => d.depth).sortKeys(d3.ascending)
.entries(root.descendants())
)
.join("g")
.attr("filter", "url(#shadow)")
.selectAll("g")
.data(d => d.values)
.join("g")
.attr("transform", d => `translate(${d.x0},${d.y0})`);
node.append("title")
.text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
node.append("rect")
.attr("id", d => (d.nodeUid = uid("node")))
.attr("fill", d => color(d.depth))
.attr("width", d => d.x1 - d.x0)
.attr("height", d => d.y1 - d.y0);
node.append("clipPath")
.attr("id", d => (d.clipUid = uid("clip")))
.append("use")
.attr("href", d => `#${d.nodeUid}`);
node.append("text")
.attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
.attr("clip-path", d => `url(#${d.clipUid})`)
.selectAll("tspan")
.data(d => [d.data.name, humanFileSize(d.value)])
.join("tspan")
.text(d => d);
node.filter(d => d.children).selectAll("tspan")
.attr("dx", 3)
.attr("y", 13);
node.filter(d => !d.children).selectAll("tspan")
.attr("x", 3)
.attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
}
function flatTreemap(data, svg, width, height, groupingDepth, tilingMode, fillOpacity) {
const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
const root = d3.treemap()
.tile(TILING_MODES[tilingMode])
.size([width, height])
.padding(1)
.round(true)(
d3.hierarchy(data)
.sum(d => d.value)
.sort((a, b) => b.value - a.value)
);
const leaf = svg.selectAll("g")
.data(root.leaves())
.join("g")
.attr("transform", d => `translate(${d.x0},${d.y0})`);
leaf.append("title")
.text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
leaf.append("rect")
.attr("id", d => (d.leafUid = uid("leaf")))
.attr("fill", d => {
while (d.depth > groupingDepth) d = d.parent;
return ordinalColor(d.data.name);
})
.attr("fill-opacity", fillOpacity)
.attr("width", d => d.x1 - d.x0)
.attr("height", d => d.y1 - d.y0);
leaf.append("clipPath")
.attr("id", d => (d.clipUid = uid("clip")))
.append("use")
.attr("href", d => `#${d.leafUid}`);
leaf.append("text")
.attr("clip-path", d => `url(#${d.clipUid})`)
.selectAll("tspan")
.data(d => {
if (d.data.name === ".") {
d = d.parent;
}
return [d.data.name, humanFileSize(d.value)]
})
.join("tspan")
.attr("x", 2)
.attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
.text(d => d);
}
function exportTreemap(indexName, width, height) {
domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
.then(function (blob) {
let a = document.createElement("a");
let url = URL.createObjectURL(blob);
a.href = url;
a.download = `${indexName}_treemap.png`;
document.body.appendChild(a);
a.click();
setTimeout(function () {
document.body.removeChild(a);
window.URL.revokeObjectURL(url);
}, 0);
});
}
export default {
name: "D3Treemap",
props: ["indexId"],
watch: {
indexId: function () {
this.update(this.indexId);
}
},
mounted() {
this.update(this.indexId);
},
methods: {
update(indexId) {
const width = SIZES[this.$store.state.optTreemapSize][0];
const height = SIZES[this.$store.state.optTreemapSize][1];
const tilingMode = this.$store.state.optTreemapTiling;
const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
const treemapColor = this.$store.state.optTreemapColor;
const treemapType = this.$store.state.optTreemapType;
const treemapSvg = d3.select("#treemap");
treemapSvg.selectAll("*").remove();
treemapSvg.attr("viewBox", [0, 0, width, height])
.attr("xmlns", "http://www.w3.org/2000/svg")
.attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
.attr("version", "1.1")
.style("overflow", "visible")
.style("font", "10px sans-serif");
d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
tabularData.forEach(row => {
row.taxonomy = row.path.split("/");
row.size = Number(row.size);
});
if (treemapType === "cascaded") {
const data = burrow(tabularData, false);
cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
} else {
const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
}
});
},
downloadTreemap() {
const width = SIZES[this.$store.state.optTreemapSize][0];
const height = SIZES[this.$store.state.optTreemapSize][1];
exportTreemap(this.indexId, width, height);
}
}
}
</script>

View File

@@ -0,0 +1,66 @@
<template>
<div id="dateSlider"></div>
</template>
<script>
import noUiSlider from 'nouislider';
import 'nouislider/dist/nouislider.css';
import {humanDate} from "@/util";
import {mergeTooltips} from "@/util-js";
export default {
name: "DateSlider",
mounted() {
this.$store.subscribe((mutation) => {
if (mutation.type === "setDateBoundsMax") {
const elem = document.getElementById("dateSlider");
if (elem.children.length > 0) {
return;
}
const dateMax = this.$store.state.dateBoundsMax;
const dateMin = this.$store.state.dateBoundsMin;
const slider = noUiSlider.create(elem, {
start: [
this.$store.state.dateMin ? this.$store.state.dateMin : dateMin,
this.$store.state.dateMax ? this.$store.state.dateMax : dateMax
],
tooltips: [true, true],
behaviour: "drag-tap",
connect: true,
range: {
"min": dateMin,
"max": dateMax,
},
format: {
to: x => humanDate(x),
from: x => x
}
});
mergeTooltips(elem, 10, " - ", true)
elem.querySelectorAll('.noUi-connect')[0].classList.add("slider-color0")
slider.on("set", (values, handle, unencoded) => {
if (handle === 0) {
this.$store.commit("setDateMin", unencoded[0] === dateMin ? undefined : Math.round(unencoded[0]));
} else {
this.$store.commit("setDateMax", unencoded[1] >= dateMax ? undefined : Math.round(unencoded[1]));
}
});
}
});
}
}
</script>
<style>
#dateSlider {
margin-top: 34px;
height: 12px;
}
</style>

View File

@@ -0,0 +1,40 @@
<template>
<b-card class="mb-4 mt-4">
<b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
<p v-html="$t('debugDescription')"></p>
<b-card-body>
<b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
<hr />
<IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx" class="mt-2"></IndexDebugInfo>
</b-card-body>
</b-card>
</template>
<script>
import IndexDebugInfo from "@/components/IndexDebugInfo";
import DebugIcon from "@/components/icons/DebugIcon";
export default {
name: "DebugInfo.vue",
components: {DebugIcon, IndexDebugInfo},
computed: {
tableItems() {
return [
{key: "version", value: this.$store.state.sist2Info.version},
{key: "platform", value: this.$store.state.sist2Info.platform},
{key: "debugBinary", value: this.$store.state.sist2Info.debug},
{key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
{key: "esIndex", value: this.$store.state.sist2Info.esIndex},
{key: "tagline", value: this.$store.state.sist2Info.tagline},
{key: "dev", value: this.$store.state.sist2Info.dev},
{key: "esVersion", value: this.$store.state.sist2Info.esVersion},
{key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
{key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
]
}
}
}
</script>

View File

@@ -0,0 +1,37 @@
<template>
<b-button-group>
<b-button variant="primary" :title="$t('displayMode.list')" :pressed="optDisplay==='list'"
@click="setOptDisplay('list')">
<svg width="20px" height="20px" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
<path fill="currentColor"
d="M80 368H16a16 16 0 0 0-16 16v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-64a16 16 0 0 0-16-16zm0-320H16A16 16 0 0 0 0 64v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16V64a16 16 0 0 0-16-16zm0 160H16a16 16 0 0 0-16 16v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-64a16 16 0 0 0-16-16zm416 176H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-320H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16zm0 160H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"></path>
</svg>
</b-button>
<b-button variant="primary" :title="$t('displayMode.grid')" :pressed="optDisplay==='grid'"
@click="setOptDisplay('grid')">
<svg width="20px" height="20px" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
<path fill="currentColor"
d="M149.333 56v80c0 13.255-10.745 24-24 24H24c-13.255 0-24-10.745-24-24V56c0-13.255 10.745-24 24-24h101.333c13.255 0 24 10.745 24 24zm181.334 240v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.256 0 24.001-10.745 24.001-24zm32-240v80c0 13.255 10.745 24 24 24H488c13.255 0 24-10.745 24-24V56c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24zm-32 80V56c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.256 0 24.001-10.745 24.001-24zm-205.334 56H24c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24zM0 376v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H24c-13.255 0-24 10.745-24 24zm386.667-56H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24zm0 160H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24zM181.333 376v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24z"></path>
</svg>
</b-button>
</b-button-group>
</template>
<script>
import {mapGetters, mapMutations} from "vuex";
export default {
name: "DisplayModeToggle",
computed: {
...mapGetters(["optDisplay"])
},
methods: {
...mapMutations(["setOptDisplay"])
}
}
</script>
<style scoped>
</style>

View File

@@ -0,0 +1,242 @@
<template>
<div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`">
<b-card
no-body
img-top
>
<!-- Info modal-->
<DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
<ContentDiv :doc="doc"></ContentDiv>
<!-- Thumbnail-->
<div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
<div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
</div>
<div
v-if="doc._props.isImage && !hover && doc._props.tnW / doc._props.tnH < 5"
class="card-img-overlay"
:class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
</div>
<div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0 && !hover"
class="card-img-overlay"
:class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
</div>
<div v-if="doc._props.isPlayableVideo" class="play">
<svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
<path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
</svg>
</div>
<img ref="tn"
v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
:src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
alt=""
:style="{height: (doc._props.isGif && hover) ? `${tnHeight()}px` : undefined}"
class="pointer fit card-img-top" @click="onThumbnailClick()">
<img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
class="fit card-img-top">
</div>
<!-- Audio player-->
<audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
:type="doc._source.mime"
:src="`f/${doc._id}`"
@play="onAudioPlay()"></audio>
<b-card-body class="padding-03">
<!-- Title line -->
<div style="display: flex">
<span class="info-icon" @click="onInfoClick()"></span>
<DocFileTitle :doc="doc"></DocFileTitle>
</div>
<!-- Tags -->
<div class="card-text">
<TagContainer :hit="doc"></TagContainer>
</div>
</b-card-body>
</b-card>
</div>
</template>
<script>
import {ext, humanFileSize, humanTime} from "@/util";
import TagContainer from "@/components/TagContainer.vue";
import DocFileTitle from "@/components/DocFileTitle.vue";
import DocInfoModal from "@/components/DocInfoModal.vue";
import ContentDiv from "@/components/ContentDiv.vue";
export default {
components: {ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
props: ["doc", "width"],
data() {
return {
ext: ext,
showInfo: false,
hover: false
}
},
computed: {
placeHolderStyle() {
const tokens = this.doc._source.thumbnail.split(",");
const w = Number(tokens[0]);
const h = Number(tokens[1]);
const MAX_HEIGHT = 400;
return {
height: `${Math.min((h / w) * this.width, MAX_HEIGHT)}px`,
}
},
smallBadge() {
return this.width < 150;
}
},
methods: {
humanFileSize: humanFileSize,
humanTime: humanTime,
onInfoClick() {
this.showInfo = true;
},
async onThumbnailClick() {
this.$store.commit("setUiLightboxSlide", this.doc._seq);
await this.$store.dispatch("showLightbox");
},
onAudioPlay() {
document.getElementsByTagName("audio").forEach((el) => {
if (el !== this.$refs["audio"]) {
el.pause();
}
});
},
onTnEnter() {
this.hover = true;
},
onTnLeave() {
this.hover = false;
},
tnHeight() {
return this.$refs.tn.height;
}
},
}
</script>
<style>
.img-wrapper {
position: relative;
}
.img-wrapper:hover svg {
fill: rgba(0, 0, 0, 1);
}
.pointer {
cursor: pointer;
}
.fit {
display: block;
min-width: 64px;
max-width: 100%;
/*max-height: 400px;*/
margin: 0 auto 0;
width: auto;
height: auto;
}
</style>
<style scoped>
.card-img-top {
border-top-left-radius: 0;
border-top-right-radius: 0;
}
.padding-03 {
padding: 0.3rem;
}
.card {
margin-top: 1em;
margin-left: 0;
margin-right: 0;
box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
border-radius: 0;
border: none;
}
.card-body {
padding: 0.3rem;
}
.thumbnail-placeholder {
}
.card-img-overlay {
pointer-events: none;
padding: 0.75rem;
bottom: unset;
top: 0;
left: unset;
right: unset;
}
.badge-resolution {
color: #212529;
background-color: #FFC107;
}
.play {
position: absolute;
width: 25px;
height: 25px;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
pointer-events: none;
}
.play svg {
fill: rgba(0, 0, 0, 0.7);
}
.doc-card {
padding-left: 3px;
padding-right: 3px;
}
.small-badge {
padding: 1px 3px;
font-size: 70%;
}
.audio-fit {
height: 39px;
vertical-align: bottom;
display: inline;
width: 100%;
}
.sub-document .card {
background: #AB47BC1F !important;
}
.theme-black .sub-document .card {
background: #37474F !important;
}
.sub-document .fit {
padding: 4px 4px 0 4px;
}
</style>

View File

@@ -0,0 +1,69 @@
<template>
<GridLayout
ref="grid-layout"
:options="gridOptions"
@append="append"
@layout-complete="$emit('layout-complete')"
>
<DocCard v-for="doc in docs" :key="doc._id" :doc="doc" :width="width"></DocCard>
</GridLayout>
</template>
<script>
import Vue from "vue";
import DocCard from "@/components/DocCard";
import VueInfiniteGrid from "@egjs/vue-infinitegrid";
Vue.use(VueInfiniteGrid);
export default Vue.extend({
components: {
DocCard,
},
props: ["docs", "append"],
data() {
return {
width: 0,
gridOptions: {
align: "center",
margin: 0,
transitionDuration: 0,
isOverflowScroll: false,
isConstantSize: false,
useFit: false,
// Indicates whether keep the number of DOMs is maintained. If the useRecycle value is 'true', keep the number
// of DOMs is maintained. If the useRecycle value is 'false', the number of DOMs will increase as card elements
// are added.
useRecycle: false
}
}
},
computed: {
colCount() {
const columns = this.$store.getters["optColumns"];
if (columns === "auto") {
return Math.round(this.$refs["grid-layout"].$el.scrollWidth / 300)
}
return columns;
},
},
mounted() {
this.width = this.$refs["grid-layout"].$el.scrollWidth / this.colCount;
if (this.colCount === 1) {
this.$refs["grid-layout"].$el.classList.add("grid-single-column");
}
this.$store.subscribe((mutation) => {
if (mutation.type === "busUpdateWallItems" && this.$refs["grid-layout"]) {
this.$refs["grid-layout"].updateItems();
}
});
},
});
</script>
<style>
</style>

View File

@@ -0,0 +1,63 @@
<template>
<a :href="`f/${doc._id}`" class="file-title-anchor" target="_blank">
<div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
v-html="fileName() + ext(doc)"></div>
</a>
</template>
<script>
import {ext} from "@/util";
export default {
name: "DocFileTitle",
props: ["doc"],
methods: {
ext: ext,
fileName() {
if (!this.doc.highlight) {
return this.doc._source.name;
}
if (this.doc.highlight["name.nGram"]) {
return this.doc.highlight["name.nGram"];
}
if (this.doc.highlight.name) {
return this.doc.highlight.name;
}
return this.doc._source.name;
}
}
}
</script>
<style scoped>
.file-title-anchor {
max-width: calc(100% - 1.2rem);
}
.file-title {
width: 100%;
line-height: 1rem;
height: 1.1rem;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
font-size: 16px;
font-family: "Source Sans Pro", sans-serif;
font-weight: bold;
}
.theme-black .file-title {
color: #ddd;
}
.theme-black .file-title:hover {
color: #fff;
}
.theme-light .file-title {
color: black;
}
.doc-card .file-title {
font-size: 12px;
}
</style>

View File

@@ -0,0 +1,33 @@
<template>
<b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
>
<template #modal-title>
<h5 class="modal-title" :title="doc._source.name + ext(doc)">{{ doc._source.name + ext(doc) }}</h5>
</template>
<img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
<InfoTable :doc="doc"></InfoTable>
<LazyContentDiv :doc-id="doc._id"></LazyContentDiv>
</b-modal>
</template>
<script>
import {ext} from "@/util";
import InfoTable from "@/components/InfoTable";
import LazyContentDiv from "@/components/LazyContentDiv";
export default {
name: "DocInfoModal",
components: {LazyContentDiv, InfoTable},
props: ["doc", "show"],
methods: {
ext: ext,
}
}
</script>
<style scoped>
</style>

View File

@@ -0,0 +1,45 @@
<template>
<b-list-group class="mt-3">
<DocListItem v-for="doc in docs" :key="doc._id" :doc="doc"></DocListItem>
</b-list-group>
</template>
<script lang="ts">
import DocListItem from "@/components/DocListItem.vue";
import Vue from "vue";
export default Vue.extend({
name: "DocList",
components: {DocListItem},
props: ["docs", "append"],
mounted() {
window.addEventListener("scroll", () => {
const threshold = 400;
const app = document.getElementById("app");
if ((window.innerHeight + window.scrollY) >= app.offsetHeight - threshold) {
this.append();
}
});
}
});
</script>
<style>
.theme-black .list-group-item {
background: #212121;
color: #e0e0e0;
border-bottom: none;
border-left: none;
border-right: none;
border-radius: 0;
padding: .25rem 0.5rem;
}
.theme-black .list-group-item:first-child {
border-top: none;
}
</style>

View File

@@ -0,0 +1,187 @@
<template>
<b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}"
@mouseenter="onTnEnter()" @mouseleave="onTnLeave()" >
<!-- Info modal-->
<DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
<div class="media ml-2">
<div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
<div class="img-wrapper">
<div v-if="doc._props.isPlayableVideo" class="play">
<svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
<path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
</svg>
</div>
<img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
:src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
alt=""
class="pointer fit-sm" @click="onThumbnailClick()">
<img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
class="fit-sm">
</div>
</div>
<div v-else class="file-icon-wrapper" style="">
<FileIcon></FileIcon>
</div>
<div class="doc-line ml-3">
<div style="display: flex">
<span class="info-icon" @click="showInfo = true"></span>
<DocFileTitle :doc="doc"></DocFileTitle>
</div>
<!-- Content highlight -->
<ContentDiv :doc="doc"></ContentDiv>
<div class="path-row">
<div class="path-line" v-html="path()"></div>
<TagContainer :hit="doc"></TagContainer>
</div>
<div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
<span v-if="doc._source.pages">{{ doc._source.pages }} {{
doc._source.pages > 1 ? $t("pages") : $t("page")
}}</span>
<span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
<span v-if="doc._source.author">{{ doc._source.author }}</span>
</div>
</div>
</div>
</b-list-group-item>
</template>
<script>
import TagContainer from "@/components/TagContainer";
import DocFileTitle from "@/components/DocFileTitle";
import DocInfoModal from "@/components/DocInfoModal";
import ContentDiv from "@/components/ContentDiv";
import FileIcon from "@/components/icons/FileIcon";
export default {
name: "DocListItem",
components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
props: ["doc"],
data() {
return {
hover: false,
showInfo: false
}
},
methods: {
async onThumbnailClick() {
this.$store.commit("setUiLightboxSlide", this.doc._seq);
await this.$store.dispatch("showLightbox");
},
path() {
if (!this.doc.highlight) {
return this.doc._source.path + "/"
}
if (this.doc.highlight["path.text"]) {
return this.doc.highlight["path.text"] + "/"
}
if (this.doc.highlight["path.nGram"]) {
return this.doc.highlight["path.nGram"] + "/"
}
return this.doc._source.path + "/"
},
onTnEnter() {
this.hover = true;
},
onTnLeave() {
this.hover = false;
},
}
}
</script>
<style scoped>
.sub-document {
background: #AB47BC1F !important;
}
.theme-black .sub-document {
background: #37474F !important;
}
.list-group {
margin-top: 1em;
}
.list-group-item {
padding: .25rem 0.5rem;
box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
border-radius: 0;
border: none;
}
.path-row {
display: -ms-flexbox;
display: flex;
-ms-flex-align: start;
align-items: flex-start;
}
.path-line {
color: #808080;
text-overflow: ellipsis;
overflow: hidden;
white-space: nowrap;
margin-right: 0.3em;
}
.theme-black .path-line {
color: #bbb;
}
.play {
position: absolute;
width: 18px;
height: 18px;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
pointer-events: none;
}
.play svg {
fill: rgba(0, 0, 0, 0.7);
}
.list-group-item .img-wrapper {
width: 88px;
height: 88px;
}
.fit-sm {
max-height: 100%;
max-width: 100%;
width: auto;
height: auto;
position: absolute;
top: 0;
bottom: 0;
left: 0;
right: 0;
margin: auto;
/*box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.12);*/
}
.doc-line {
max-width: calc(100% - 88px - 1.5rem);
flex: 1;
vertical-align: middle;
margin-top: auto;
margin-bottom: auto;
}
.file-icon-wrapper {
width: calc(88px + .5rem);
height: 88px;
position: relative;
}
</style>

View File

@@ -0,0 +1,72 @@
<template>
<b-modal :visible="show" size="lg" :hide-footer="true" static :title="$t('help.help')"
@close="$emit('close')"
@hide="$emit('close')"
>
<h2>{{$t("help.simpleSearch")}}</h2>
<table class="table">
<tbody>
<tr>
<td><code>+</code></td>
<td>{{$t("help.and")}}</td>
</tr>
<tr>
<td><code>|</code></td>
<td>{{$t("help.or")}}</td>
</tr>
<tr>
<td><code>-</code></td>
<td>{{$t("help.not")}}</td>
</tr>
<tr>
<td><code>""</code></td>
<td>{{$t("help.quotes")}}</td>
</tr>
<tr>
<td><code>{{$t("help.term")}}*</code></td>
<td>{{$t("help.prefix")}}</td>
</tr>
<tr>
<td><code>(</code> {{$t("and")}} <code>)</code></td>
<td>{{$t("help.parens")}}</td>
</tr>
<tr>
<td><code>{{$t("help.term")}}~N</code></td>
<td>{{$t("help.tildeTerm")}}</td>
</tr>
<tr>
<td><code>"..."~N</code></td>
<td>{{$t("help.tildePhrase")}}</td>
</tr>
</tbody>
</table>
<p v-html="$t('help.example1')"></p>
<p v-html="$t('help.defaultOperator')"></p>
<p v-html="$t('help.fuzzy')"></p>
<br>
<p v-html="$t('help.moreInfoSimple')"></p>
<p></p>
<h2>{{$t("help.advancedSearch")}}</h2>
<p v-html="$t('help.moreInfoAdvanced')"></p>
</b-modal>
</template>
<script>
export default {
name: "HelpDialog",
props: ["show"]
}
</script>
<style scoped>
</style>

View File

@@ -0,0 +1,30 @@
<template>
<div>
<h4>[{{ index.name }}]</h4>
<b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
</div>
</template>
<script>
import {humanDate} from "@/util";
export default {
name: "IndexDebugInfo",
props: ["index"],
computed: {
tableItems() {
return [
{key: this.$t("name"), value: this.index.name},
{key: this.$t("id"), value: this.index.id},
{key: this.$t("indexVersion"), value: this.index.version},
{key: this.$t("rewriteUrl"), value: this.index.rewriteUrl},
{key: this.$t("timestamp"), value: humanDate(this.index.timestamp)},
]
}
}
}
</script>
<style scoped>
</style>

View File

@@ -0,0 +1,171 @@
<template>
<div v-if="isMobile">
<b-form-select
:value="selectedIndicesIds"
@change="onSelect($event)"
:options="indices" multiple :select-size="6" text-field="name"
value-field="id"></b-form-select>
</div>
<div v-else>
<div class="d-flex justify-content-between align-content-center">
<span>
{{ selectedIndices.length }}
{{ selectedIndices.length === 1 ? $t("indexPicker.selectedIndex") : $t("indexPicker.selectedIndices") }}
</span>
<div>
<b-button variant="link" @click="selectAll()"> {{ $t("indexPicker.selectAll") }}</b-button>
<b-button variant="link" @click="selectNone()"> {{ $t("indexPicker.selectNone") }}</b-button>
</div>
</div>
<b-list-group id="index-picker-desktop" class="unselectable">
<b-list-group-item
v-for="idx in indices"
@click="toggleIndex(idx, $event)"
@click.shift="shiftClick(idx, $event)"
class="d-flex justify-content-between align-items-center list-group-item-action pointer"
:class="{active: lastClickIndex === idx}"
>
<div class="d-flex">
<b-checkbox @change="toggleIndex(idx)" :checked="isSelected(idx)"></b-checkbox>
{{ idx.name }}
<span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
</div>
<b-badge class="version-badge">v{{ idx.version }}</b-badge>
</b-list-group-item>
</b-list-group>
</div>
</template>
<script lang="ts">
import SmallBadge from "./SmallBadge.vue"
import {mapActions, mapGetters} from "vuex";
import Vue from "vue";
import {format} from "date-fns";
export default Vue.extend({
components: {
SmallBadge
},
data() {
return {
loading: true,
lastClickIndex: null
}
},
computed: {
...mapGetters([
"indices", "selectedIndices"
]),
selectedIndicesIds() {
return this.selectedIndices.map(idx => idx.id)
},
isMobile() {
return window.innerWidth <= 650;
}
},
methods: {
...mapActions({
setSelectedIndices: "setSelectedIndices"
}),
shiftClick(index, e) {
if (this.lastClickIndex === null) {
return;
}
const select = this.isSelected(this.lastClickIndex);
let leftBoundary = this.indices.indexOf(this.lastClickIndex);
let rightBoundary = this.indices.indexOf(index);
if (rightBoundary < leftBoundary) {
let tmp = leftBoundary;
leftBoundary = rightBoundary;
rightBoundary = tmp;
}
for (let i = leftBoundary; i <= rightBoundary; i++) {
if (select) {
if (!this.isSelected(this.indices[i])) {
this.setSelectedIndices([this.indices[i], ...this.selectedIndices]);
}
} else {
this.setSelectedIndices(this.selectedIndices.filter(idx => idx !== this.indices[i]));
}
}
},
selectAll() {
this.setSelectedIndices(this.indices);
},
selectNone() {
this.setSelectedIndices([]);
},
onSelect(value) {
this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
},
formatIdxDate(timestamp: number): string {
return format(new Date(timestamp * 1000), "yyyy-MM-dd");
},
toggleIndex(index, e) {
if (e.shiftKey) {
return;
}
this.lastClickIndex = index;
if (this.isSelected(index)) {
this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
} else {
this.setSelectedIndices([index, ...this.selectedIndices]);
}
},
isSelected(index) {
return this.selectedIndices.find(idx => idx.id == index.id) != null;
}
},
})
</script>
<style scoped>
.timestamp-text {
line-height: 24px;
font-size: 80%;
}
.theme-black .version-badge {
color: #eee !important;
background: none;
}
.version-badge {
color: #222 !important;
background: none;
}
.list-group-item {
padding: 0.2em 0.4em;
}
#index-picker-desktop {
overflow-y: auto;
max-height: 132px;
}
.btn-link:focus {
box-shadow: none;
}
.unselectable {
user-select: none;
-ms-user-select: none;
-moz-user-select: none;
-webkit-user-select: none;
}
.list-group-item.active {
z-index: 2;
background-color: inherit;
color: inherit;
}
</style>

View File

@@ -0,0 +1,94 @@
<template>
<b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0 mt-4">
<template #cell(value)="data">
<span v-if="'html' in data.item" v-html="data.item.html"></span>
<span v-else>{{ data.value }}</span>
</template>
</b-table>
</template>
<script>
import {humanDate, humanFileSize} from "@/util";
function makeGpsLink(latitude, longitude) {
if (isNaN(latitude) || isNaN(longitude)) {
return "";
}
return `<a target="_blank" href="https://maps.google.com/?q=${latitude},${longitude}&ll=${latitude},${longitude}&t=k&z=17">${latitude}, ${longitude}</a>`;
}
function dmsToDecimal(dms, ref) {
const tokens = dms.split(",")
const d = Number(tokens[0].trim().split(":")[0]) / Number(tokens[0].trim().split(":")[1])
const m = Number(tokens[1].trim().split(":")[0]) / Number(tokens[1].trim().split(":")[1])
const s = Number(tokens[2].trim().split(":")[0]) / Number(tokens[2].trim().split(":")[1])
return (d + (m / 60) + (s / 3600)) * (ref === "S" || ref === "W" ? -1 : 1)
}
export default {
name: "InfoTable",
props: ["doc"],
computed: {
tableItems() {
const src = this.doc._source;
const items = [
{key: "index", value: `[${this.$store.getters.indexMap[src.index].name}]`},
{key: "mtime", value: humanDate(src.mtime)},
{key: "mime", value: src.mime},
{key: "size", value: humanFileSize(src.size)},
{key: "path", value: src.path},
];
if ("width" in this.doc._source) {
items.push({
key: "image size",
value: `${src.width}x${src.height}`
})
}
const fields = [
"title", "duration", "audioc", "videoc",
"bitrate", "artist", "album", "album_artist", "genre", "font_name", "author",
"modified_by", "pages", "tag",
"exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
"exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",
"checksum"
];
fields.forEach(field => {
if (field in src) {
items.push({key: field, value: src[field]});
}
});
// Exif GPS
if ("exif_gps_longitude_dec" in src) {
items.push({
key: "Exif GPS",
html: makeGpsLink(src["exif_gps_latitude_dec"], src["exif_gps_longitude_dec"]),
});
} else if ("exif_gps_longitude_dms" in src) {
items.push({
key: "Exif GPS",
html: makeGpsLink(
dmsToDecimal(src["exif_gps_latitude_dms"], src["exif_gps_latitude_ref"]),
dmsToDecimal(src["exif_gps_longitude_dms"], src["exif_gps_longitude_ref"]),
),
});
}
return items;
}
}
}
</script>
<style scoped>
</style>

View File

@@ -0,0 +1,30 @@
<template>
<Preloader v-if="loading"></Preloader>
<div v-else-if="content" class="content-div">{{ content }}</div>
</template>
<script>
import Sist2Api from "@/Sist2Api";
import Preloader from "@/components/Preloader";
export default {
name: "LazyContentDiv",
components: {Preloader},
props: ["docId"],
data() {
return {
content: "",
loading: true
}
},
mounted() {
Sist2Api.getDocInfo(this.docId).then(src => {
this.content = src.data.content;
this.loading = false;
})
}
}
</script>
<style scoped>
</style>

View File

@@ -0,0 +1,128 @@
<template>
<div>
<FsLightbox
:key="lightboxKey"
:toggler="showLightbox"
:sources="lightboxSources"
:thumbs="lightboxThumbs"
:captions="lightboxCaptions"
:types="lightboxTypes"
:source-index="lightboxSlide"
:custom-toolbar-buttons="customButtons"
:slideshow-time="$store.getters.optLightboxSlideDuration * 1000"
:zoom-increment="0.5"
:load-only-current-source="$store.getters.optLightboxLoadOnlyCurrent"
:on-close="onClose"
:on-open="onShow"
:on-slide-change="onSlideChange"
></FsLightbox>
<a id="lightbox-download" style="display: none"></a>
</div>
</template>
<script>
import FsLightbox from "fslightbox-vue";
export default {
name: "Lightbox",
components: {FsLightbox},
data() {
return {
customButtons: [
{
viewBox: "0 0 384.928 384.928",
d: "M321.339,245.334c-4.74-4.692-12.439-4.704-17.179,0l-99.551,98.564V12.03 c0-6.641-5.438-12.03-12.151-12.03s-12.151,5.39-12.151,12.03v331.868l-99.551-98.552c-4.74-4.704-12.439-4.704-17.179,0 s-4.74,12.319,0,17.011l120.291,119.088c4.692,4.644,12.499,4.644,17.191,0l120.291-119.088 C326.091,257.653,326.091,250.038,321.339,245.334C316.599,240.642,326.091,250.038,321.339,245.334z",
width: "17px",
height: "17px",
title: "Download",
onClick: this.onDownloadClick
}
]
}
},
computed: {
showLightbox() {
return this.$store.getters["uiShowLightbox"];
},
lightboxSources() {
return this.$store.getters["uiLightboxSources"];
},
lightboxThumbs() {
return this.$store.getters["uiLightboxThumbs"];
},
lightboxKey() {
return this.$store.getters["uiLightboxKey"];
},
lightboxSlide() {
return this.$store.getters["uiLightboxSlide"];
},
lightboxCaptions() {
return this.$store.getters["uiLightboxCaptions"];
},
lightboxTypes() {
return this.$store.getters["uiLightboxTypes"];
}
},
methods: {
onDownloadClick() {
const url = this.lightboxSources[this.lightboxSlide];
const a = document.getElementById("lightbox-download");
a.setAttribute("href", url);
a.setAttribute("download", "");
a.click();
},
onShow() {
this.$store.commit("setUiLightboxIsOpen", true);
},
onClose() {
this.$store.commit("setUiLightboxIsOpen", false);
},
onSlideChange() {
// Pause all videos when changing slide
document.getElementsByTagName("video").forEach((el) => {
el.pause();
});
},
}
}
</script>
<style>
.fslightbox-toolbar-button:nth-child(2) {
order: 1;
}
.fslightbox-toolbar-button:nth-child(1) {
order: 2;
}
.fslightbox-toolbar-button:nth-child(3) {
order: 3;
}
.fslightbox-toolbar-button:nth-child(4) {
order: 4;
}
.fslightbox-toolbar-button:nth-child(5) {
order: 5;
}
@media (max-width: 650px) {
/* Disable fullscreen on mobile because it's buggy */
.fslightbox-toolbar-button:nth-child(6) {
display: none;
}
}
.fslightbox-toolbar-button:nth-child(6) {
order: 6;
}
.fslightbox-toolbar-button:nth-child(7) {
order: 7;
}
</style>

View File

@@ -0,0 +1,26 @@
<template>
<div class="lightbox-caption">
<p>
<b>{{
`[${$store.getters.indices.find(i => i.id === hit._source.index).name}]`
}}</b>{{ `/${hit._source.path}/${hit._source.name}${ext(hit)}` }}
</p>
<p style="margin-top: -1em">
<span v-if="hit._source.width">{{ `${hit._source.width}x${hit._source.height}`}}</span>
{{ ` (${humanFileSize(hit._source.size)})` }}
</p>
</div>
</template>
<script>
import {ext, humanFileSize} from "@/util";
export default {
name: "LightboxCaption",
props: ["hit"],
methods: {
humanFileSize: humanFileSize,
ext: ext
}
}
</script>

View File

@@ -0,0 +1,115 @@
<template>
<div id="mimeTree"></div>
</template>
<script>
import InspireTree from "inspire-tree";
import InspireTreeDOM from "inspire-tree-dom";
import "inspire-tree-dom/dist/inspire-tree-light.min.css";
import {getSelectedTreeNodes, getTreeNodeAttributes} from "@/util";
import Sist2Api from "@/Sist2Api";
import Sist2Query from "@/Sist2Query";
export default {
name: "MimePicker",
data() {
return {
mimeTree: null,
stashedMimeTreeAttributes: null
}
},
mounted() {
this.$store.subscribe((mutation) => {
if (mutation.type === "setUiMimeMap" && this.mimeTree === null) {
this.initializeTree();
} else if (mutation.type === "busSearch") {
this.updateTree();
}
});
},
methods: {
handleTreeClick(node, e) {
if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused") {
return;
}
this.$store.commit("setSelectedMimeTypes", getSelectedTreeNodes(this.mimeTree));
},
updateTree() {
if (this.$store.getters.optUpdateMimeMap === false) {
return;
}
if (this.stashedMimeTreeAttributes === null) {
this.stashedMimeTreeAttributes = getTreeNodeAttributes(this.mimeTree);
}
const query = Sist2Query.searchQuery();
Sist2Api.getMimeTypes(query).then(({buckets, mimeMap}) => {
this.$store.commit("setUiMimeMap", mimeMap);
this.$store.commit("setUiDetailsMimeAgg", buckets);
this.mimeTree.removeAll();
this.mimeTree.addNodes(mimeMap);
// Restore selected mimes
if (this.stashedMimeTreeAttributes === null) {
// NOTE: This happens when successive fast searches are triggered
this.stashedMimeTreeAttributes = {};
// Always add the selected mime types
this.$store.state.selectedMimeTypes.forEach(mime => {
this.stashedMimeTreeAttributes[mime] = {
checked: true
}
});
}
Object.entries(this.stashedMimeTreeAttributes).forEach(([mime, attributes]) => {
if (this.mimeTree.node(mime)) {
if (attributes.checked) {
this.mimeTree.node(mime).select();
}
if (attributes.collapsed === false) {
this.mimeTree.node(mime).expand();
}
}
});
this.stashedMimeTreeAttributes = null;
});
},
initializeTree() {
const mimeMap = this.$store.state.uiMimeMap;
this.mimeTree = new InspireTree({
selection: {
mode: "checkbox"
},
data: mimeMap
});
new InspireTreeDOM(this.mimeTree, {
target: "#mimeTree"
});
this.mimeTree.on("node.state.changed", this.handleTreeClick);
this.mimeTree.deselect();
if (this.$store.state._onLoadSelectedMimeTypes.length > 0) {
this.$store.state._onLoadSelectedMimeTypes.forEach(mime => {
this.mimeTree.node(mime).select();
});
}
}
}
}
</script>
<style scoped>
#mimeTree {
max-height: 350px;
overflow: auto;
}
</style>

View File

@@ -0,0 +1,109 @@
<template>
<b-navbar>
<b-navbar-brand v-if="$route.path !== '/'" to="/">
<Sist2Icon></Sist2Icon>
</b-navbar-brand>
<b-navbar-brand v-else href=".">
<Sist2Icon></Sist2Icon>
</b-navbar-brand>
<span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch" target="_blank">legacyES</a></span>
</span>
<span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>
<b-button class="ml-auto" to="stats" variant="link">{{ $t("stats") }}</b-button>
<b-button to="config" variant="link">{{ $t("config") }}</b-button>
</b-navbar>
</template>
<script>
import Sist2Icon from "@/components/icons/Sist2Icon";
export default {
name: "NavBar",
components: {Sist2Icon},
methods: {
tagline() {
return this.$store.state.sist2Info.tagline;
},
sist2Version() {
return this.$store.state.sist2Info.version;
},
isDebug() {
return this.$store.state.sist2Info.debug;
},
isLegacy() {
return this.$store.state.sist2Info.esVersionLegacy;
},
hideLegacy() {
return this.$store.state.optHideLegacy;
}
}
}
</script>
<style scoped>
.navbar {
box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
border-radius: 0;
}
.theme-black .navbar {
background: #546b7a30;
border-bottom: none;
}
.navbar-brand {
color: #222 !important;
font-size: 1.75rem;
padding: 0;
font-family: Hack;
}
.navbar-brand:hover {
color: #000 !important;
}
.version {
color: #222 !important;
margin-left: -18px;
margin-top: -14px;
font-size: 11px;
font-family: monospace;
}
.theme-black .version {
color: #f5f5f5 !important;
}
.theme-black .navbar-brand {
font-size: 1.75rem;
padding: 0;
color: #f5f5f5 !important;
}
.theme-black a:hover, .theme-black .btn:hover {
color: #fff;
}
.theme-black .navbar span {
color: #eee;
}
@media (max-width: 650px) {
.tagline {
display: none;
}
.version {
display: none;
}
}
.theme-light .btn-link {
color: #222;
}
</style>

View File

@@ -0,0 +1,245 @@
<template>
<div>
<div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
<div class="input-group-prepend">
<b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
<path
fill="currentColor"
d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
/>
</svg>
</b-button>
</div>
<VueSimpleSuggest
class="form-control-fix-flex"
@input="setPathText"
:value="getPathText"
:list="suggestPath"
:max-suggestions="0"
:placeholder="$t('pathBar.placeholder')"
>
<!-- Suggestion item template-->
<div slot="suggestion-item" slot-scope="{ suggestion, query }">
<div class="suggestion-line" :title="suggestion">
<strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
</div>
</div>
</VueSimpleSuggest>
</div>
<b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
<div id="pathTree"></div>
</b-modal>
</div>
</template>
<script>
import InspireTree from "inspire-tree";
import InspireTreeDOM from "inspire-tree-dom";
import "inspire-tree-dom/dist/inspire-tree-light.min.css";
import Sist2Api from "@/Sist2Api";
import {mapGetters, mapMutations} from "vuex";
import VueSimpleSuggest from 'vue-simple-suggest'
import 'vue-simple-suggest/dist/styles.css' // Optional CSS
export default {
name: "PathTree",
components: {
VueSimpleSuggest
},
data() {
return {
mimeTree: null,
pathItems: [],
tmpPath: ""
}
},
computed: {
...mapGetters(["getPathText"])
},
mounted() {
this.$store.subscribe((mutation) => {
// Wait until indices are loaded to get the root paths
if (mutation.type === "setIndices") {
let pathTree = new InspireTree({
data: (node, resolve, reject) => {
return this.getNextDepth(node);
},
sort: "text"
});
this.$store.state.indices.forEach(idx => {
pathTree.addNode({
id: "/" + idx.id,
values: ["/" + idx.id],
text: `/[${idx.name}]`,
index: idx.id,
depth: 0,
children: true
})
});
new InspireTreeDOM(pathTree, {
target: "#pathTree"
});
pathTree.on("node.click", this.handleTreeClick);
pathTree.expand();
}
});
},
methods: {
...mapMutations(["setPathText"]),
getSuggestionWithoutQueryPrefix(suggestion, query) {
return suggestion.slice(query.length)
},
async getPathChoices() {
return new Promise(getPaths => {
const q = {
suggest: {
path: {
prefix: this.getPathText,
completion: {
field: "suggest-path",
skip_duplicates: true,
size: 10000
}
}
}
};
Sist2Api.esQuery(q)
.then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
})
},
async suggestPath(term) {
if (!this.$store.state.optSuggestPath) {
return []
}
term = term.toLowerCase();
const choices = await this.getPathChoices();
let matches = [];
for (let i = 0; i < choices.length; i++) {
if (~choices[i].toLowerCase().indexOf(term)) {
matches.push(choices[i]);
}
}
return matches.sort((a, b) => a.length - b.length);
},
getNextDepth(node) {
const q = {
query: {
bool: {
filter: [
{term: {index: node.index}},
{range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
]
}
},
aggs: {
paths: {
terms: {
field: "path",
size: 10000
}
}
},
size: 0
};
if (node.depth > 0) {
q.query.bool.must = {
prefix: {
path: node.id,
}
};
}
return Sist2Api.esQuery(q).then(resp => {
const buckets = resp["aggregations"]["paths"]["buckets"];
if (!buckets) {
return false;
}
const paths = [];
return buckets
.filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
.sort((a, b) => a.key > b.key)
.map(bucket => {
if (paths.some(n => bucket.key.startsWith(n))) {
return null;
}
const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
paths.push(bucket.key);
return {
id: bucket.key,
text: `${name}/ (${bucket.doc_count})`,
depth: node.depth + 1,
index: node.index,
values: [bucket.key],
children: true,
}
}).filter(x => x !== null)
});
},
handleTreeClick(e, node, handler) {
if (node.depth !== 0) {
this.setPathText(node.id);
this.$refs['path-modal'].hide()
this.$emit("search");
}
handler();
},
},
}
</script>
<style scoped>
#mimeTree {
max-height: 350px;
overflow: auto;
}
.form-control-fix-flex {
flex: 1 1 auto;
width: 1%;
min-width: 0;
margin-bottom: 0;
}
.suggestion-line {
max-width: 100%;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
line-height: 1.1;
}
</style>
<style>
.suggestions {
max-height: 250px;
overflow-y: auto;
}
.theme-black .suggestions {
color: black
}
</style>

View File

@@ -0,0 +1,3 @@
<template>
<b-progress value="1" max="1" animated></b-progress>
</template>

View File

@@ -0,0 +1,134 @@
<template>
<b-card v-if="lastResultsLoaded" id="results">
<span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
<div style="float: right">
<b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
$t("details")
}}
</b-button>
<template v-if="hitCount !== 0">
<SortSelect class="ml-2"></SortSelect>
<DisplayModeToggle class="ml-2"></DisplayModeToggle>
</template>
</div>
<b-collapse id="collapse-1" class="pt-2" style="clear:both;">
<b-card>
<b-table :items="tableItems" small borderless bordered thead-class="hidden" class="mb-0"></b-table>
<br/>
<h4>
{{$t("mimeTypes")}}
<b-button size="sm" variant="primary" class="float-right" @click="onCopyClick"><ClipboardIcon/></b-button>
</h4>
<Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
<b-table
v-else
sort-by="doc_count"
:sort-desc="true"
thead-class="hidden"
:items="$store.state.uiDetailsMimeAgg" small bordered class="mb-0"
></b-table>
</b-card>
</b-collapse>
</b-card>
</template>
<script lang="ts">
import Sist2Api, {EsResult} from "@/Sist2Api";
import Vue from "vue";
import {humanFileSize} from "@/util";
import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
import SortSelect from "@/components/SortSelect.vue";
import Preloader from "@/components/Preloader.vue";
import Sist2Query from "@/Sist2Query";
import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
export default Vue.extend({
name: "ResultsCard",
components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
created() {
},
computed: {
lastResultsLoaded() {
return this.$store.state.lastQueryResults != null;
},
hitCount() {
return (this.$store.state.lastQueryResults as EsResult).aggregations.total_count.value;
},
tableItems() {
const items = [];
items.push({key: this.$t("queryTime"), value: this.took()});
items.push({key: this.$t("totalSize"), value: this.totalSize()});
return items;
}
},
methods: {
took() {
return (this.$store.state.lastQueryResults as EsResult).took + "ms";
},
totalSize() {
return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
},
onToggle() {
const show = !document.getElementById("collapse-1").classList.contains("show");
this.$store.commit("setUiShowDetails", show);
if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
// Mime aggs are not updated automatically, update now
this.forceUpdateMimeAgg();
}
},
onCopyClick() {
let tsvString = "";
this.$store.state.uiDetailsMimeAgg.slice().sort((a,b) => b["doc_count"] - a["doc_count"]).forEach(row => {
tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
});
navigator.clipboard.writeText(tsvString);
this.$bvToast.toast(
this.$t("toast.copiedToClipboard"),
{
title: null,
noAutoHide: false,
toaster: "b-toaster-bottom-right",
headerClass: "hidden",
bodyClass: "toast-body-info",
});
},
forceUpdateMimeAgg() {
const query = Sist2Query.searchQuery();
Sist2Api.getMimeTypes(query).then(({buckets}) => {
this.$store.commit("setUiDetailsMimeAgg", buckets);
});
}
},
});
</script>
<style>
#results {
margin-top: 1em;
box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
border-radius: 0;
border: none;
}
#results .card-body {
padding: 0.7em 1.25em;
}
.hidden {
display: none;
}
</style>

View File

@@ -0,0 +1,46 @@
<template>
<div>
<b-input-group>
<b-form-input :value="searchText"
:placeholder="advanced() ? $t('searchBar.advanced') : $t('searchBar.simple')"
@input="setSearchText($event)"></b-form-input>
<template #prepend>
<b-input-group-text>
<b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">
{{ $t("searchBar.fuzzy") }}
</b-form-checkbox>
</b-input-group-text>
</template>
<template #append>
<b-button variant="outline-secondary" @click="$emit('show-help')">{{$t("help.help")}}</b-button>
</template>
</b-input-group>
</div>
</template>
<script>
import {mapGetters, mapMutations} from "vuex";
export default {
computed: {
...mapGetters({
optQueryMode: "optQueryMode",
searchText: "searchText",
fuzzy: "fuzzy",
}),
},
methods: {
...mapMutations({
setSearchText: "setSearchText",
setFuzzy: "setFuzzy"
}),
advanced() {
return this.optQueryMode === "advanced"
}
}
}
</script>
<style>
</style>

View File

@@ -0,0 +1,135 @@
<template>
<div id="sizeSlider"></div>
</template>
<script>
import noUiSlider from 'nouislider';
import 'nouislider/dist/nouislider.css';
import {humanFileSize} from "@/util";
import {mergeTooltips} from "@/util-js";
export default {
name: "SizeSlider",
mounted() {
const elem = document.getElementById("sizeSlider");
const slider = noUiSlider.create(elem, {
start: [
this.$store.state.sizeMin ? this.$store.state.sizeMin : 0,
this.$store.state.sizeMax ? this.$store.state.sizeMax: 1000 * 1000 * 50000
],
tooltips: [true, true],
behaviour: "drag-tap",
connect: true,
range: {
'min': 0,
"10%": 1000 * 1000,
"20%": 1000 * 1000 * 10,
"50%": 1000 * 1000 * 5000,
"max": 1000 * 1000 * 50000,
},
format: {
to: x => x >= 1000 * 1000 * 50000 ? "50G+" : humanFileSize(Math.round(x)),
from: x => x
}
});
mergeTooltips(elem, 10, " - ")
elem.querySelectorAll('.noUi-connect')[0].classList.add("slider-color0")
slider.on("set", (values, handle, unencoded) => {
if (handle === 0) {
this.$store.commit("setSizeMin", unencoded[0] === 0 ? undefined : Math.round(unencoded[0]))
} else {
this.$store.commit("setSizeMax", unencoded[1] >= 1000 * 1000 * 50000 ? undefined : Math.round(unencoded[1]))
}
});
}
}
</script>
<style>
#sizeSlider {
margin-top: 34px;
height: 12px;
}
.slider-color0 {
background: #2196f3;
box-shadow: none;
}
.theme-black .slider-color0 {
background: #00bcd4;
}
.noUi-horizontal .noUi-handle {
width: 2px;
height: 18px;
right: -1px;
top: -3px;
border: none;
background-color: #1976d2;
box-shadow: none;
cursor: ew-resize;
border-radius: 0;
}
.theme-black .noUi-horizontal .noUi-handle {
background-color: #2168ac;
}
.noUi-handle:before {
top: -6px;
left: 3px;
width: 10px;
height: 28px;
background-color: transparent;
}
.noUi-handle:after {
top: -6px;
left: -10px;
width: 10px;
height: 28px;
background-color: transparent;
}
.noUi-draggable {
cursor: move;
}
.noUi-tooltip {
color: #fff;
font-size: 13px;
line-height: 1.333;
text-shadow: none;
padding: 0 2px;
background: #2196F3;
border-radius: 4px;
border: none;
}
.theme-black .noUi-tooltip {
background: #00bcd4;
}
.theme-black .noUi-connects {
background-color: #37474f;
}
.noUi-horizontal .noUi-origin > .noUi-tooltip {
bottom: 7px;
}
.noUi-target {
background-color: #e1e4e9;
border: none;
box-shadow: none;
}
</style>

View File

@@ -0,0 +1,21 @@
<template>
<b-badge variant="secondary" :pill="pill">{{ text }}</b-badge>
</template>
<script lang="ts">
import Vue from "vue";
export default Vue.extend({
props: {
text: String,
pill: Boolean
}
})
</script>
<style scoped>
.badge-pill {
padding: 0.3em .4em 0.1em;
border-radius: 6rem;
}
</style>

View File

@@ -0,0 +1,67 @@
<template>
<b-dropdown variant="primary">
<b-dropdown-item :class="{'dropdown-active': sort === 'score'}" @click="onSelect('score')">{{
$t("sort.relevance")
}}
</b-dropdown-item>
<b-dropdown-item :class="{'dropdown-active': sort === 'dateAsc'}" @click="onSelect('dateAsc')">{{
$t("sort.dateAsc")
}}
</b-dropdown-item>
<b-dropdown-item :class="{'dropdown-active': sort === 'dateDesc'}" @click="onSelect('dateDesc')">
{{ $t("sort.dateDesc") }}
</b-dropdown-item>
<b-dropdown-item :class="{'dropdown-active': sort === 'sizeAsc'}" @click="onSelect('sizeAsc')">{{
$t("sort.sizeAsc")
}}
</b-dropdown-item>
<b-dropdown-item :class="{'dropdown-active': sort === 'sizeDesc'}" @click="onSelect('sizeDesc')">
{{ $t("sort.sizeDesc") }}
</b-dropdown-item>
<b-dropdown-item :class="{'dropdown-active': sort === 'nameDesc'}" @click="onSelect('nameDesc')">
{{ $t("sort.nameDesc") }}
</b-dropdown-item>
<b-dropdown-item :class="{'dropdown-active': sort === 'nameAsc'}" @click="onSelect('nameAsc')">
{{ $t("sort.nameAsc") }}
</b-dropdown-item>
<b-dropdown-item :class="{'dropdown-active': sort === 'random'}" @click="onSelect('random')">
{{ $t("sort.random") }}
</b-dropdown-item>
<template #button-content>
<svg aria-hidden="true" width="20px" height="20px" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 320 512">
<path
fill="currentColor"
d="M41 288h238c21.4 0 32.1 25.9 17 41L177 448c-9.4 9.4-24.6 9.4-33.9 0L24 329c-15.1-15.1-4.4-41 17-41zm255-105L177 64c-9.4-9.4-24.6-9.4-33.9 0L24 183c-15.1 15.1-4.4 41 17 41h238c21.4 0 32.1-25.9 17-41z"></path>
</svg>
</template>
</b-dropdown>
</template>
<script>
export default {
name: "SortSelect",
computed: {
sort() {
return this.$store.state.sortMode;
}
},
methods: {
onSelect(sortMode) {
if (sortMode === "random") {
this.$store.commit("setSeed", Math.round(Math.random() * 100000));
}
this.$store.commit("setSortMode", sortMode);
}
},
}
</script>
<style>
.dropdown-active a {
font-weight: bold;
}
</style>

View File

@@ -0,0 +1,337 @@
<template>
<div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
<b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static lazy>
<b-row>
<b-col style="flex-grow: 2" sm>
<VueSimpleSuggest
ref="suggest"
:value="tagText"
@select="setTagText($event)"
@input="setTagText($event)"
class="form-control-fix-flex"
style="margin-top: 17px"
:list="suggestTag"
:max-suggestions="0"
:placeholder="$t('saveTagPlaceholder')"
>
<!-- Suggestion item template-->
<div slot="suggestion-item" slot-scope="{ suggestion, query}"
>
<div class="suggestion-line">
<span
class="badge badge-suggestion"
:style="{background: getBg(suggestion), color: getFg(suggestion)}"
>
<strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
</span>
</div>
</div>
</VueSimpleSuggest>
</b-col>
<b-col class="mt-4">
<TwitterColorPicker v-model="color" triangle="hide" :width="252" class="mr-auto ml-auto"></TwitterColorPicker>
</b-col>
</b-row>
<b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
</b-button>
</b-modal>
<template v-for="tag in hit._tags">
<div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
<span
:id="hit._id+tag.rawText"
:title="tag.text"
tabindex="-1"
class="badge pointer"
:style="badgeStyle(tag)" :class="badgeClass(tag)"
@click.right="onTagRightClick(tag, $event)"
>{{ tag.text.split(".").pop() }}</span>
<b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
<b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{$t("deleteTag")}}</b-button>
</b-popover>
</div>
<span
v-else :key="tag.text"
class="badge"
:style="badgeStyle(tag)" :class="badgeClass(tag)"
>{{ tag.text.split(".").pop() }}</span>
</template>
<!-- Add button -->
<small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{$t("addTag")}}</small>
<!-- Size tag-->
<small v-else class="text-muted badge-size">{{
humanFileSize(hit._source.size)
}}</small>
</div>
</template>
<script>
import {humanFileSize, lum} from "@/util";
import Vue from "vue";
import {Twitter} from 'vue-color'
import Sist2Api from "@/Sist2Api";
import VueSimpleSuggest from 'vue-simple-suggest'
export default Vue.extend({
components: {
"TwitterColorPicker": Twitter,
VueSimpleSuggest
},
props: ["hit"],
data() {
return {
showAddButton: false,
showModal: false,
tagText: null,
color: {
hex: "#e0e0e0",
},
}
},
computed: {
tagHover() {
return this.$store.getters["uiTagHover"];
}
},
methods: {
humanFileSize: humanFileSize,
getSuggestionWithoutQueryPrefix(suggestion, query) {
return suggestion.id.slice(query.length, -8)
},
getBg(suggestion) {
return suggestion.id.slice(-7);
},
getFg(suggestion) {
return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
},
setTagText(value) {
this.$refs.suggest.clearSuggestions();
if (typeof value === "string") {
this.tagText = {
id: value,
title: value
};
return;
}
this.color = {
hex: "#" + value.id.split("#")[1]
}
this.tagText = value;
},
badgeClass(tag) {
return `badge-${tag.style}`;
},
badgeStyle(tag) {
return {
background: tag.bg,
color: tag.fg,
};
},
onTagHover(tag) {
if (tag.userTag) {
this.$store.commit("setUiTagHover", tag);
}
},
onTagLeave() {
this.$store.commit("setUiTagHover", null);
},
onTagDeleteClick(tag, e) {
this.hit._tags = this.hit._tags.filter(t => t !== tag);
Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
//toast
this.$store.commit("busUpdateWallItems");
this.$store.commit("busUpdateTags");
});
},
tagAdd() {
this.showModal = true;
},
saveTag() {
if (this.tagText.id.includes("#")) {
this.$bvToast.toast(
this.$t("toast.invalidTag"),
{
title: this.$t("toast.invalidTagTitle"),
noAutoHide: true,
toaster: "b-toaster-bottom-right",
headerClass: "toast-header-error",
bodyClass: "toast-body-error",
});
return;
}
let tag = this.tagText.id + this.color.hex.replace("#", ".#");
const userTags = this.hit._tags.filter(t => t.userTag);
if (userTags.find(t => t.rawText === tag) != null) {
this.$bvToast.toast(
this.$t("toast.dupeTag"),
{
title: this.$t("toast.dupeTagTitle"),
noAutoHide: true,
toaster: "b-toaster-bottom-right",
headerClass: "toast-header-error",
bodyClass: "toast-body-error",
});
return;
}
this.hit._tags.push(Sist2Api.createUserTag(tag));
Sist2Api.saveTag(tag, this.hit).then(() => {
this.tagText = null;
this.showModal = false;
this.$store.commit("busUpdateWallItems");
this.$store.commit("busUpdateTags");
// TODO: toast
});
},
async suggestTag(term) {
term = term.toLowerCase();
const choices = await this.getTagChoices(term);
let matches = [];
for (let i = 0; i < choices.length; i++) {
if (~choices[i].toLowerCase().indexOf(term)) {
matches.push(choices[i]);
}
}
return matches.sort().map(match => {
return {
title: match.split(".").slice(0,-1).join("."),
id: match
}
});
},
getTagChoices(prefix) {
return new Promise(getPaths => {
Sist2Api.esQuery({
suggest: {
tag: {
prefix: prefix,
completion: {
field: "suggest-tag",
skip_duplicates: true,
size: 10000
}
}
}
}).then(resp => {
const result = [];
resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
tags.forEach(tag => {
const t = tag.slice(0, -8);
if (!result.find(x => x.slice(0, -8) === t)) {
result.push(tag);
}
});
});
getPaths(result);
});
});
}
},
});
</script>
<style scoped>
.badge-video {
color: #FFFFFF;
background-color: #F27761;
}
.badge-image {
color: #FFFFFF;
background-color: #AA99C9;
}
.badge-audio {
color: #FFFFFF;
background-color: #00ADEF;
}
.badge-user {
color: #212529;
background-color: #e0e0e0;
}
.badge-user:hover, .add-tag-button:hover {
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
}
.badge-text {
color: #FFFFFF;
background-color: #FAAB3C;
}
.badge {
margin-right: 3px;
}
.badge-delete {
margin-right: -2px;
margin-left: 2px;
margin-top: -1px;
font-family: monospace;
font-size: 90%;
background: rgba(0, 0, 0, 0.2);
padding: 0.1em 0.4em;
color: white;
cursor: pointer;
}
.badge-size {
width: 50px;
display: inline-block;
}
.add-tag-button {
cursor: pointer;
color: #212529;
background-color: #e0e0e0;
width: 50px;
}
.badge {
user-select: none;
}
.badge-suggestion {
font-size: 90%;
font-weight: normal;
}
</style>
<style>
.vc-twitter-body {
padding: 0 !important;
}
.vc-twitter {
box-shadow: none !important;
background: none !important;
}
.tooltip {
user-select: none;
}
.toast {
border: none;
}
</style>

View File

@@ -0,0 +1,186 @@
<template>
<div id="tagTree"></div>
</template>
<script>
import InspireTree from "inspire-tree";
import InspireTreeDOM from "inspire-tree-dom";
import "inspire-tree-dom/dist/inspire-tree-light.min.css";
import {getSelectedTreeNodes} from "@/util";
import Sist2Api from "@/Sist2Api";
function resetState(node) {
node._tree.defaultState.forEach(function (val, prop) {
node.state(prop, val);
});
return node;
}
function baseStateChange(prop, value, verb, node, deep) {
if (node.state(prop) !== value) {
node._tree.batch();
if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
resetState(node);
}
node.state(prop, value);
node._tree.emit('node.' + verb, node, false);
if (deep && node.hasChildren()) {
node.children.recurseDown(function (child) {
baseStateChange(prop, value, verb, child);
});
}
node.markDirty();
node._tree.end();
}
return node;
}
function addTag(map, tag, id, count) {
const tags = tag.split(".");
const child = {
id: id,
count: count,
text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
name: tags[0],
children: [],
// Overwrite base functions
blur: function () {
// noop
},
select: function () {
this.state("selected", true);
return this.check()
},
deselect: function () {
this.state("selected", false);
return this.uncheck()
},
uncheck: function () {
baseStateChange('checked', false, 'unchecked', this, false);
this.state('indeterminate', false);
if (this.hasParent()) {
this.getParent().refreshIndeterminateState();
}
this._tree.end();
return this;
},
check: function () {
baseStateChange('checked', true, 'checked', this, false);
if (this.hasParent()) {
this.getParent().refreshIndeterminateState();
}
this._tree.end();
return this;
}
};
let found = false;
map.forEach(node => {
if (node.name === child.name) {
found = true;
if (tags.length !== 1) {
addTag(node.children, tags.slice(1).join("."), id, count);
} else {
// Same name, different color
console.error("FIXME: Duplicate tag?")
console.trace(node)
}
}
});
if (!found) {
if (tags.length !== 1) {
addTag(child.children, tags.slice(1).join("."), id, count);
map.push(child);
} else {
map.push(child);
}
}
}
export default {
name: "TagPicker",
data() {
return {
tagTree: null,
loadedFromArgs: false,
}
},
mounted() {
this.$store.subscribe((mutation) => {
if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
this.initializeTree();
this.updateTree();
} else if (mutation.type === "busUpdateTags") {
window.setTimeout(this.updateTree, 2000);
}
});
},
methods: {
initializeTree() {
const tagMap = [];
this.tagTree = new InspireTree({
selection: {
mode: "checkbox",
autoDeselect: false,
},
checkbox: {
autoCheckChildren: false,
},
data: tagMap
});
new InspireTreeDOM(this.tagTree, {
target: '#tagTree'
});
this.tagTree.on("node.state.changed", this.handleTreeClick);
},
updateTree() {
// TODO: remember which tags are selected and restore?
const tagMap = [];
Sist2Api.getTags().then(tags => {
tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
this.tagTree.removeAll();
this.tagTree.addNodes(tagMap);
if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
this.$store.state._onLoadSelectedTags.forEach(mime => {
this.tagTree.node(mime).select();
this.loadedFromArgs = true;
});
}
});
},
handleTreeClick(node, e) {
if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused") {
return;
}
this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
},
}
}
</script>
<style scoped>
#mimeTree {
max-height: 350px;
overflow: auto;
}
</style>
<style>
.inspire-tree .focused>.wholerow {
border: none;
}
</style>

View File

@@ -0,0 +1,21 @@
<template>
<svg style="width:24px;height:24px" viewBox="0 0 24 24">
<path
fill="currentColor"
d="M17,9H7V7H17M17,13H7V11H17M14,17H7V15H14M12,3A1,1 0 0,1 13,4A1,1 0 0,1 12,5A1,1 0 0,1 11,4A1,1 0 0,1 12,3M19,3H14.82C14.4,1.84 13.3,1 12,1C10.7,1 9.6,1.84 9.18,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3Z"/>
</svg>
</template>
<script>
export default {
name: "ClipboardIcon"
}
</script>
<style scoped>
svg {
display: inline-block;
width: 20px;
height: 20px;
}
</style>

View File

@@ -0,0 +1,22 @@
<template>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 309.998 309.998" fill="currentColor">
<path
d="M294.998,155.03H250v-48.82l39.714-39.715c5.858-5.857,5.858-15.356,0-21.213c-5.857-5.857-15.355-5.857-21.213,0 l-23.7,23.701c-12.885-37.2-48.274-63.984-89.802-63.984c-41.528,0-76.913,26.787-89.797,63.989L41.497,45.282 c-5.856-5.859-15.354-5.857-21.213,0s-5.858,15.355,0,21.213L60,106.212v48.818H15c-8.284,0-15,6.716-15,15c0,8.284,6.716,15,15,15 h45.134c0.855,16.314,5.849,31.551,13.944,44.68l-49.685,49.683c-5.858,5.857-5.858,15.354,0,21.213 c2.929,2.93,6.768,4.394,10.607,4.394c3.838,0,7.678-1.465,10.606-4.394l48.095-48.093c16.558,14.018,37.957,22.486,61.301,22.486 c0.019,0,0.037-0.001,0.057-0.001c0.011,0,0.022,0.002,0.033,0.002c0.019,0,0.037-0.003,0.056-0.003 c23.285-0.035,44.629-8.494,61.15-22.483l48.094,48.092c2.929,2.929,6.768,4.394,10.606,4.394c3.839,0,7.678-1.465,10.607-4.394 c5.858-5.858,5.858-15.355,0-21.213l-49.683-49.681c8.096-13.131,13.089-28.366,13.944-44.682h45.132c8.284,0,15-6.716,15-15 C309.998,161.746,303.282,155.03,294.998,155.03z M154.999,34.999c30.681,0,56.465,21.365,63.254,50H91.747 C98.535,56.364,124.318,34.999,154.999,34.999z M90,179.999v-9.272c0.011-0.232,0.035-0.462,0.035-0.696 c0-0.234-0.024-0.464-0.035-0.695v-54.336h50.092v128.254C111.415,236.494,90,210.708,90,179.999z M170.092,243.212V114.999H220 v54.297c-0.012,0.244-0.037,0.486-0.037,0.734c0,0.248,0.025,0.49,0.037,0.734v9.234C220,210.645,198.676,236.388,170.092,243.212z"/>
</svg>
</template>
<script>
export default {
name: "DebugIcon"
}
</script>
<style scoped>
svg {
display: inline-block;
width: 20px;
height: 20px;
vertical-align: middle;
}
</style>

View File

@@ -0,0 +1,33 @@
<template>
<svg class="file-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 50 50">
<path
fill="currentColor"
d="M 7 2 L 7 48 L 43 48 L 43 14.59375 L 42.71875 14.28125 L 30.71875 2.28125 L 30.40625 2 Z M 9 4 L 29 4 L 29 16 L 41 16 L 41 46 L 9 46 Z M 31 5.4375 L 39.5625 14 L 31 14 Z"/>
</svg>
</template>
<script>
export default {
name: "FileIcon"
}
</script>
<style scoped>
.file-icon {
position: absolute;
max-height: 100%;
max-width: 100%;
top: 0;
bottom: 0;
left: 0;
right: 0;
}
.theme-black .file-icon {
color: #ffffff50;
}
.theme-light .file-icon {
color: #000000a0;
}
</style>

View File

@@ -0,0 +1,23 @@
<template>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 1000" fill="currentColor">
<g transform="translate(0.000000,512.000000) scale(0.100000,-0.100000)">
<path
d="M4568.5,5011c-73.2-7.7-154-25-177.1-36.6c-84.7-46.2-102-119.4-159.8-689.2s-65.5-610.3-159.8-670c-23.1-15.4-125.1-55.8-225.3-90.5c-100.1-32.7-290.7-111.7-423.6-175.2c-319.6-152.1-315.8-152.1-619.9,94.3c-718.1,583.3-650.7,535.2-747,535.2c-77,0-104-11.6-184.8-77c-157.9-127.1-410.1-375.4-567.9-558.3c-155.9-177.1-190.6-250.3-159.8-344.6c9.6-27,165.6-227.2,344.6-446.7c181-219.5,342.7-425.5,360-458.2c52-88.6,42.3-150.2-50.1-335c-73.2-148.3-144.4-325.4-252.2-623.8c-17.3-50-57.8-113.6-88.6-138.6c-63.5-53.9-59.7-53.9-695-117.4c-527.5-52-577.6-65.5-627.6-179c-46.2-105.9-46.2-1057,0-1162.9c50-113.6,98.2-127.1,646.9-181c271.5-25,523.7-52,560.2-57.8c111.7-17.3,179.1-107.8,259.9-344.6c38.5-115.5,119.4-310,177.1-431.3c57.8-119.4,104-240.7,104-269.5c0-78.9-42.4-140.5-394.7-568c-179-219.5-335-419.7-344.6-446.6c-30.8-94.3,3.9-167.5,159.8-344.6c157.9-181,410.1-429.3,564.1-554.5c96.3-78.9,188.7-105.9,265.7-75.1c26.9,11.6,234.9,173.3,462.1,360c227.2,188.7,433.2,348.5,458.2,358.1c82.8,30.8,136.7,17.3,354.3-86.6c119.4-57.8,308-136.7,419.7-175.2c111.7-38.5,221.4-82.8,244.5-98.2c94.3-59.7,102-100.1,159.8-670c61.6-606.5,73.2-648.8,188.7-700.8c105.9-46.2,1057-46.2,1162.9,0c115.5,52,127.1,94.3,188.7,700.8c57.8,569.9,65.4,610.3,159.8,670c23.1,15.4,132.9,59.7,244.5,98.2s300.3,117.4,417.8,175.2c219.5,104,273.4,117.5,356.2,86.6c25-9.6,231-169.4,458.2-358.1c227.2-186.8,435.1-348.5,462.1-360c77-28.9,169.4-3.9,265.7,75.1c152.1,121.3,442.8,410.1,583.4,577.6c140.6,163.6,173.3,242.6,136.7,333.1c-11.6,27-173.3,234.9-360,462.1c-188.7,227.2-348.5,433.2-358.1,458.2c-30.8,82.8-17.3,136.7,86.6,356.2c57.8,117.4,138.6,311.9,177.1,427.4c80.9,236.8,148.3,327.3,259.9,344.6c36.6,5.8,288.8,32.7,562.2,59.7c308,28.9,517.9,59.7,550.6,77c30.8,15.4,71.2,59.7,90.5,100.1c32.8,65.4,36.6,123.2,34.7,573.7c0,562.2-11.5,627.6-115.5,687.3c-46.2,27-188.7,48.1-612.2,90.5c-573.7,59.7-614.2,67.4-673.8,161.7c-15.4,23.1-59.7,132.9-98.2,244.5s-117.4,300.3-175.2,417.8c-57.8,119.4-104,240.7-104,271.5c0,80.9,40.4,138.6,394.7,569.9c181,219.5,335,419.7,344.6,446.7c30.8,94.3-3.9,167.5-159.8,344.6c-157.9,181-410.1,429.3-564.1,554.5c-96.3,78.9-188.7,104-265.7,75.1c-27-11.6-234.9-173.3-462.1-360c-227.2-188.7-433.2-348.5-458.2-358.1c-80.9-30.8-130.9-19.2-371.6,96.3c-130.9,61.6-325.4,142.5-431.3,177.1c-217.5,71.2-308,140.5-325.4,250.3c-5.8,36.6-32.7,288.8-57.8,560.3c-53.9,550.6-67.4,596.8-181,645C5502.3,5018.7,4807.3,5036,4568.5,5011z M5463.8,1897.8c502.5-127.1,954.9-494.8,1184-960.7c446.7-914.5,78.9-2011.9-824-2460.5c-1053.1-521.8-2308.4,52-2604.9,1189.8c-71.2,277.2-71.2,629.6,0,904.9c192.5,737.4,814.4,1284.2,1569.1,1376.6C4974.8,1971,5255.9,1949.8,5463.8,1897.8z"/>
</g>
</svg>
</template>
<script>
export default {
name: "GearIcon"
}
</script>
<style scoped>
svg {
display: inline-block;
width: 20px;
height: 20px;
margin-top: -4px;
}
</style>

View File

@@ -0,0 +1,21 @@
<template>
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24">
<path
fill="currentColor"
d="M12 0c-6.627 0-12 5.373-12 12s5.373 12 12 12 12-5.373 12-12-5.373-12-12-12zm1 16.057v-3.057h2.994c-.059 1.143-.212 2.24-.456 3.279-.823-.12-1.674-.188-2.538-.222zm1.957 2.162c-.499 1.33-1.159 2.497-1.957 3.456v-3.62c.666.028 1.319.081 1.957.164zm-1.957-7.219v-3.015c.868-.034 1.721-.103 2.548-.224.238 1.027.389 2.111.446 3.239h-2.994zm0-5.014v-3.661c.806.969 1.471 2.15 1.971 3.496-.642.084-1.3.137-1.971.165zm2.703-3.267c1.237.496 2.354 1.228 3.29 2.146-.642.234-1.311.442-2.019.607-.344-.992-.775-1.91-1.271-2.753zm-7.241 13.56c-.244-1.039-.398-2.136-.456-3.279h2.994v3.057c-.865.034-1.714.102-2.538.222zm2.538 1.776v3.62c-.798-.959-1.458-2.126-1.957-3.456.638-.083 1.291-.136 1.957-.164zm-2.994-7.055c.057-1.128.207-2.212.446-3.239.827.121 1.68.19 2.548.224v3.015h-2.994zm1.024-5.179c.5-1.346 1.165-2.527 1.97-3.496v3.661c-.671-.028-1.329-.081-1.97-.165zm-2.005-.35c-.708-.165-1.377-.373-2.018-.607.937-.918 2.053-1.65 3.29-2.146-.496.844-.927 1.762-1.272 2.753zm-.549 1.918c-.264 1.151-.434 2.36-.492 3.611h-3.933c.165-1.658.739-3.197 1.617-4.518.88.361 1.816.67 2.808.907zm.009 9.262c-.988.236-1.92.542-2.797.9-.89-1.328-1.471-2.879-1.637-4.551h3.934c.058 1.265.231 2.488.5 3.651zm.553 1.917c.342.976.768 1.881 1.257 2.712-1.223-.49-2.326-1.211-3.256-2.115.636-.229 1.299-.435 1.999-.597zm9.924 0c.7.163 1.362.367 1.999.597-.931.903-2.034 1.625-3.257 2.116.489-.832.915-1.737 1.258-2.713zm.553-1.917c.27-1.163.442-2.386.501-3.651h3.934c-.167 1.672-.748 3.223-1.638 4.551-.877-.358-1.81-.664-2.797-.9zm.501-5.651c-.058-1.251-.229-2.46-.492-3.611.992-.237 1.929-.546 2.809-.907.877 1.321 1.451 2.86 1.616 4.518h-3.933z"/>
</svg>
</template>
<script>
export default {
name: "LanguageIcon"
}
</script>
<style scoped>
svg {
display: inline-block;
width: 20px;
height: 20px;
}
</style>

View File

@@ -0,0 +1,40 @@
<template>
<svg
xmlns="http://www.w3.org/2000/svg"
width="27.868069mm"
height="7.6446671mm"
viewBox="0 0 27.868069 7.6446671"
>
<g transform="translate(-4.5018313,-4.1849793)">
<g
style="fill: currentColor;fill-opacity:1;stroke:none;stroke-width:0.26458332">
<path
d="m 6.3153296,11.829646 q -0.7717014,0 -1.8134983,-0.337619 v -0.916395 q 1.0128581,0.511252 1.803852,0.511252 0.5643067,0 0.901926,-0.236334 0.3376194,-0.236333 0.3376194,-0.63183 0,-0.3424428 -0.2845649,-0.5498376 Q 6.980922,9.4566645 6.3635609,9.3264399 L 5.9921796,9.2492698 Q 5.2301245,9.0949295 4.8732126,8.7428407 4.5211238,8.3859288 4.5211238,7.7733908 q 0,-0.7765245 0.5305447,-1.1961372 0.5305447,-0.4196126 1.5096409,-0.4196126 0.829579,0 1.6061036,0.3183268 V 7.3441319 Q 7.4101809,6.9004036 6.5854251,6.9004036 q -1.1671984,0 -1.1671984,0.7958171 0,0.2604492 0.1012858,0.4147895 0.1012858,0.1495171 0.3858507,0.2556261 0.2845649,0.1012858 0.8392253,0.2122179 l 0.3569119,0.067524 q 1.3408312,0.2652724 1.3408312,1.4614098 0,0.80064 -0.5691298,1.263661 -0.5691298,0.458197 -1.5578722,0.458197 z"
style="stroke-width:0.26458332"
/>
<path
d="m 11.943927,5.3087694 q -0.144694,0 -0.144694,-0.144694 V 4.3296733 q 0,-0.144694 0.144694,-0.144694 h 0.694531 q 0.144694,0 0.144694,0.144694 v 0.8344021 q 0,0.144694 -0.144694,0.144694 z M 13.5645,11.728361 q -0.795817,0 -1.234722,-0.511253 -0.434082,-0.516075 -0.434082,-1.4469398 V 6.9823969 H 10.714028 V 6.2878656 h 2.069124 v 3.4823026 q 0,0.5884228 0.221864,0.8971028 0.221865,0.308681 0.6463,0.308681 h 1.036974 v 0.752409 z"
style="stroke-width:0.26458332"
/>
<path
d="m 18.209178,11.829646 q -0.771701,0 -1.813498,-0.337619 v -0.916395 q 1.012858,0.511252 1.803852,0.511252 0.564306,0 0.901926,-0.236334 0.337619,-0.236333 0.337619,-0.63183 0,-0.3424428 -0.284565,-0.5498376 Q 18.87477,9.4566645 18.257409,9.3264399 l -0.371381,-0.07717 Q 17.123973,9.0949295 16.767061,8.7428407 16.414972,8.3859288 16.414972,7.7733908 q 0,-0.7765245 0.530545,-1.1961372 0.530545,-0.4196126 1.509641,-0.4196126 0.829579,0 1.606103,0.3183268 v 0.8681641 q -0.757232,-0.4437283 -1.581988,-0.4437283 -1.167198,0 -1.167198,0.7958171 0,0.2604492 0.101286,0.4147895 0.101286,0.1495171 0.385851,0.2556261 0.284565,0.1012858 0.839225,0.2122179 l 0.356912,0.067524 q 1.340831,0.2652724 1.340831,1.4614098 0,0.80064 -0.56913,1.263661 -0.56913,0.458197 -1.557872,0.458197 z"
style="stroke-width:0.26458332"
/>
<path
d="m 25.207545,11.709068 q -0.993565,0 -1.408355,-0.40032 -0.409966,-0.405143 -0.409966,-1.3794164 V 6.9775737 H 21.947107 V 6.2878656 h 1.442117 V 4.8746874 l 0.887457,-0.3858507 v 1.7990289 h 2.016069 v 0.6897081 h -2.016069 v 2.9517579 q 0,0.5932454 0.226687,0.8344024 0.226687,0.236333 0.790994,0.236333 h 0.998388 v 0.709001 z"
style="stroke-width:0.26458332"
/>
<path
d="m 27.995317,11.043476 q 0,-0.178456 0.120578,-0.299035 0.274919,-0.289388 0.651123,-0.684885 0.376205,-0.4003199 0.805464,-0.8681638 0.327973,-0.356912 0.491959,-0.5353679 0.16881,-0.1832791 0.255626,-0.2845649 0.09164,-0.1012858 0.178456,-0.2073948 0.255626,-0.3086805 0.405144,-0.5257215 0.15434,-0.2170411 0.250803,-0.4292589 0.168809,-0.3762045 0.168809,-0.7524089 0,-0.5980686 -0.352089,-0.935688 -0.356911,-0.3424425 -0.979096,-0.3424425 -0.863341,0 -1.938899,0.6414768 V 4.8361023 q 0.491959,-0.2363335 0.979096,-0.3569119 0.47749,-0.1205783 0.945334,-0.1205783 0.501606,0 0.940511,0.1350477 0.438905,0.1350478 0.766878,0.4244358 0.289388,0.2556261 0.463021,0.6270074 0.173633,0.3665582 0.173633,0.829579 0,0.4726671 -0.212218,0.9501574 -0.106109,0.2411567 -0.274919,0.4726671 -0.163986,0.2266873 -0.424435,0.540191 Q 31.270225,8.501684 31.077299,8.718725 30.884374,8.9357661 30.628748,9.2106847 30.445469,9.4084332 30.286305,9.5675966 30.131965,9.72676 29.958332,9.9003928 29.7847,10.069203 29.558012,10.300713 29.336148,10.5274 29.012998,10.869843 h 3.356901 v 0.819932 h -4.374582 z"
style="stroke-width:0.26458332"
/>
</g>
</g>
</svg>
</template>
<script>
export default {
name: "Sist2Icon"
}
</script>

View File

@@ -0,0 +1,487 @@
export default {
en: {
searchBar: {
simple: "Search",
advanced: "Advanced search",
fuzzy: "Fuzzy"
},
addTag: "Add",
deleteTag: "Delete",
download: "Download",
and: "and",
page: "page",
pages: "pages",
mimeTypes: "Media types",
tags: "Tags",
help: {
simpleSearch: "Simple search",
advancedSearch: "Advanced search",
help: "Help",
term: "<TERM>",
and: "AND operator",
or: "OR operator",
not: "negates a single term",
quotes: "will match the enclosed sequence of terms in that specific order",
prefix: "will match any term with a given prefix when used at the end of a word",
parens: "used to group expressions",
tildeTerm: "match a term with a given edit distance",
tildePhrase: "match a phrase with a given number of allowed intervening unmatched words",
example1:
"For example: <code>\"fried eggs\" +(eggplant | potato) -frittata</code> will match the " +
"phrase <i>fried eggs</i> and either <i>eggplant</i> or <i>potato</i>, but will ignore results " +
"containing <i>frittata</i>.",
defaultOperator:
"When neither <code>+</code> or <code>|</code> is specified, the default operator is " +
"<code>+</code> (and).",
fuzzy:
"When the <b>Fuzzy</b> option is checked, partial matches based on 3-grams are also returned.",
moreInfoSimple: "For more information, see <a target=\"_blank\" " +
"rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">Elasticsearch documentation</a>",
moreInfoAdvanced: "For documentation about the advanced search mode, see <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">Elasticsearch documentation</a>"
},
config: "Configuration",
configDescription: "Configuration is saved in real time for this browser.",
configReset: "Reset configuration",
searchOptions: "Search options",
treemapOptions: "Treemap options",
displayOptions: "Display options",
opt: {
lang: "Language",
highlight: "Enable highlighting",
fuzzy: "Set fuzzy search by default",
searchInPath: "Enable matching query against document path",
suggestPath: "Enable auto-complete in path filter bar",
fragmentSize: "Highlight context size in characters",
queryMode: "Search mode",
displayMode: "Display",
columns: "Column count",
treemapType: "Treemap type",
treemapTiling: "Treemap tiling",
treemapColorGroupingDepth: "Treemap color grouping depth (flat)",
treemapColor: "Treemap color (cascaded)",
treemapSize: "Treemap size",
theme: "Theme",
lightboxLoadOnlyCurrent: "Do not preload full-size images for adjacent slides in image viewer.",
slideDuration: "Slide duration",
resultSize: "Number of results per page",
tagOrOperator: "Use OR operator when specifying multiple tags.",
hideDuplicates: "Hide duplicate results based on checksum",
hideLegacy: "Hide the 'legacyES' Elasticsearch notice",
updateMimeMap: "Update the Media Types tree in real time"
},
queryMode: {
simple: "Simple",
advanced: "Advanced",
},
lang: {
en: "English",
fr: "Français",
"zh-CN": "简体中文",
},
displayMode: {
grid: "Grid",
list: "List",
},
columns: {
auto: "Auto"
},
treemapType: {
cascaded: "Cascaded",
flat: "Flat (compact)"
},
treemapSize: {
small: "Small",
medium: "Medium",
large: "Large",
xLarge: "xLarge",
xxLarge: "xxLarge",
custom: "Custom",
},
treemapTiling: {
binary: "Binary",
squarify: "Squarify",
slice: "Slice",
dice: "Dice",
sliceDice: "Slice & Dice",
},
theme: {
light: "Light",
black: "Black"
},
hit: "hit",
hits: "hits",
details: "Details",
stats: "Stats",
queryTime: "Query time",
totalSize: "Total size",
pathBar: {
placeholder: "Filter path",
modalTitle: "Select path"
},
debug: "Debug information",
debugDescription: "Information useful for debugging. If you encounter bugs or have suggestions for" +
" new features, please submit a new issue <a href='https://github.com/simon987/sist2/issues/new/choose'>here</a>.",
tagline: "Tagline",
toast: {
esConnErrTitle: "Elasticsearch connection error",
esConnErr: "sist2 web module encountered an error while connecting to Elasticsearch." +
" See server logs for more information.",
esQueryErrTitle: "Query error",
esQueryErr: "Could not parse or execute query, please check the Advanced search documentation. " +
"See server logs for more information.",
dupeTagTitle: "Duplicate tag",
dupeTag: "This tag already exists for this document.",
copiedToClipboard: "Copied to clipboard"
},
saveTagModalTitle: "Add tag",
saveTagPlaceholder: "Tag name",
confirm: "Confirm",
indexPickerPlaceholder: "Select an index",
sort: {
relevance: "Relevance",
dateAsc: "Date (Older first)",
dateDesc: "Date (Newer first)",
sizeAsc: "Size (Smaller first)",
sizeDesc: "Size (Larger first)",
nameAsc: "Name (A-z)",
nameDesc: "Name (Z-a)",
random: "Random",
},
d3: {
mimeCount: "File count distribution by media type",
mimeSize: "Size distribution by media type",
dateHistogram: "File modification time distribution",
sizeHistogram: "File size distribution",
},
indexPicker: {
selectNone: "Select None",
selectAll: "Select All",
selectedIndex: "selected index",
selectedIndices: "selected indices",
},
},
fr: {
searchBar: {
simple: "Recherche",
advanced: "Recherche avancée",
fuzzy: "Approximatif"
},
addTag: "Ajouter",
deleteTag: "Supprimer",
download: "Télécharger",
and: "et",
page: "page",
pages: "pages",
mimeTypes: "Types de médias",
tags: "Tags",
help: {
simpleSearch: "Recherche simple",
advancedSearch: "Recherche avancée",
help: "Aide",
term: "<TERME>",
and: "opérator ET",
or: "opérator OU",
not: "exclut un terme",
quotes: "recherche la séquence de termes dans cet ordre spécifique.",
prefix: "lorsqu'utilisé à la fin d'un mot, recherche tous les termes avec le préfixe donné.",
parens: "utilisé pour regrouper des expressions",
tildeTerm: "recherche un terme avec une distance d'édition donnée",
tildePhrase: "recherche une phrase avec un nombre donné de mots intermédiaires tolérés",
example1:
"Par exemple: <code>\"fried eggs\" +(eggplant | potato) -frittata</code> va rechercher la " +
"phrase <i>fried eggs</i> et soit <i>eggplant</i> ou <i>potato</i>, mais vas exlure les résultats " +
"qui contiennent <i>frittata</i>.",
defaultOperator:
"Lorsqu'aucun des opérateurs <code>+</code> ou <code>|</code> sont spécifiés, l'opérateur par défaut " +
"est <code>+</code> (ET).",
fuzzy:
"Lorsque l'option <b>Approximatif</b> est activée, les résultats partiels basés sur les trigrammes sont" +
" également inclus.",
moreInfoSimple: "Pour plus d'information, voir <a target=\"_blank\" " +
"rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">documentation Elasticsearch</a>",
moreInfoAdvanced: "Pour plus d'information sur la recherche avancée, voir <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">documentation Elasticsearch</a>"
},
config: "Configuration",
configDescription: "La configuration est enregistrée en temps réel pour ce navigateur.",
configReset: "Réinitialiser la configuration",
searchOptions: "Options de recherche",
treemapOptions: "Options du Treemap",
displayOptions: "Options d'affichage",
opt: {
lang: "Langue",
highlight: "Activer le surlignage",
fuzzy: "Activer la recherche approximative par défaut",
searchInPath: "Activer la recherche dans le chemin des documents",
suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
fragmentSize: "Longueur du contexte de surlignage, en nombre de caractères",
queryMode: "Mode de recherche",
displayMode: "Affichage",
columns: "Nombre de colonnes",
treemapType: "Type de Treemap",
treemapTiling: "Treemap tiling",
treemapColorGroupingDepth: "Groupage de couleur du Treemap (plat)",
treemapColor: "Couleur du Treemap (en cascade)",
treemapSize: "Taille du Treemap",
theme: "Thème",
lightboxLoadOnlyCurrent: "Désactiver le chargement des diapositives adjacentes pour le visualiseur d'images",
slideDuration: "Durée des diapositives",
resultSize: "Nombre de résultats par page",
tagOrOperator: "Utiliser l'opérateur OU lors de la spécification de plusieurs tags",
hideDuplicates: "Masquer les résultats en double",
hideLegacy: "Masquer la notice 'legacyES' Elasticsearch",
updateMimeMap: "Mettre à jour l'arbre de Types de médias en temps réel"
},
queryMode: {
simple: "Simple",
advanced: "Avancé",
},
lang: {
en: "English",
fr: "Français",
"zh-CN": "简体中文",
},
displayMode: {
grid: "Grille",
list: "Liste",
},
columns: {
auto: "Auto"
},
treemapType: {
cascaded: "En cascade",
flat: "Plat (compact)"
},
treemapSize: {
small: "Petit",
medium: "Moyen",
large: "Grand",
xLarge: "xGrand",
xxLarge: "xxGrand",
custom: "Personnalisé",
},
treemapTiling: {
binary: "Binary",
squarify: "Squarify",
slice: "Slice",
dice: "Dice",
sliceDice: "Slice & Dice",
},
theme: {
light: "Clair",
black: "Noir"
},
hit: "résultat",
hits: "résultats",
details: "Détails",
stats: "Stats",
queryTime: "Durée de la requête",
totalSize: "Taille totale",
pathBar: {
placeholder: "Filtrer le chemin",
modalTitle: "Sélectionner le chemin"
},
debug: "Information de débogage",
debugDescription: "Informations utiles pour le débogage\n" +
"Si vous rencontrez des bogues ou si vous avez des suggestions pour de nouvelles fonctionnalités," +
" veuillez soumettre un nouvel Issue <a href='https://github.com/simon987/sist2/issues/new/choose'>ici</a>.",
tagline: "Tagline",
toast: {
esConnErrTitle: "Erreur de connexion Elasticsearch",
esConnErr: "Le module web a rencontré une erreur lors de la connexion à Elasticsearch." +
" Consultez les journaux du serveur pour plus d'informations..",
esQueryErrTitle: "Erreur de requête",
esQueryErr: "Impossible d'analyser ou d'exécuter la requête, veuillez consulter la documentation sur la " +
"recherche avancée. Voir les journaux du serveur pour plus d'informations.",
dupeTagTitle: "Tag en double",
dupeTag: "Ce tag existe déjà pour ce document.",
copiedToClipboard: "Copié dans le presse-papier"
},
saveTagModalTitle: "Ajouter un tag",
saveTagPlaceholder: "Nom du tag",
confirm: "Confirmer",
indexPickerPlaceholder: "Sélectionner un index",
sort: {
relevance: "Pertinence",
dateAsc: "Date (Plus ancient)",
dateDesc: "Date (Plus récent)",
sizeAsc: "Taille (Plus petit)",
sizeDesc: "Taille (Plus grand)",
nameAsc: "Nom (A-z)",
nameDesc: "Nom (Z-a)",
random: "Aléatoire",
},
d3: {
mimeCount: "Distribution du nombre de fichiers par type de média",
mimeSize: "Distribution des tailles de fichiers par type de média",
dateHistogram: "Distribution des dates de modification",
sizeHistogram: "Distribution des tailles de fichier",
},
indexPicker: {
selectNone: "Sélectionner aucun",
selectAll: "Sélectionner tout",
selectedIndex: "indice sélectionné",
selectedIndices: "indices sélectionnés",
},
},
"zh-CN": {
searchBar: {
simple: "搜索",
advanced: "高级搜索",
fuzzy: "模糊搜索"
},
addTag: "添加",
deleteTag: "删除",
download: "下载",
and: "与",
page: "页",
pages: "页",
mimeTypes: "文件类型",
tags: "标签",
help: {
simpleSearch: "简易搜索",
advancedSearch: "高级搜索",
help: "帮助",
term: "<关键词>",
and: "与操作",
or: "或操作",
not: "反选单个关键词",
quotes: "括起来的部分视为一个关键词,保序",
prefix: "在词尾使用时,匹配前缀",
parens: "表达式编组",
tildeTerm: "匹配编辑距离以内的关键词",
tildePhrase: "匹配短语,容忍一些非匹配词",
example1:
"例如: <code>\"番茄\" +(炒蛋 | 牛腩) -饭</code> 将匹配" +
"短语 <i>番茄炒蛋</i>、<i>炒蛋</i> 或者 <i>牛腩</i>,而忽略任何带有" +
"<i>饭</i>的关键词.",
defaultOperator:
"表达式中无<code>+</code>或者<code>|</code>时,默认使用" +
"<code>+</code>(与操作)。",
fuzzy:
"选中<b>模糊搜索</b>选项时返回部分匹配的结果3-grams)。",
moreInfoSimple: "详细信息:<a target=\"_blank\" " +
"rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">Elasticsearch文档</a>",
moreInfoAdvanced: "高级搜索模式文档:<a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">Elasticsearch文档</a>"
},
config: "配置",
configDescription: "配置在此浏览器中实时保存。",
configReset: "重置所有设置",
searchOptions: "搜索选项",
treemapOptions: "树状图选项",
displayOptions: "显示选项",
opt: {
lang: "语言",
highlight: "启用高亮",
fuzzy: "默认使用模糊搜索",
searchInPath: "匹配文档路径",
suggestPath: "搜索框启用自动补全",
fragmentSize: "高亮上下文大小",
queryMode: "搜索模式",
displayMode: "显示",
columns: "列数",
treemapType: "树状图类属性",
treemapTiling: "树状图平铺",
treemapColorGroupingDepth: "树状图颜色编组深度(展开)",
treemapColor: "树状图颜色(折叠)",
treemapSize: "树状图大小",
theme: "主题",
lightboxLoadOnlyCurrent: "在图片查看器中,不要预读相邻的全图",
slideDuration: "幻灯片时长",
resultSize: "每页结果数",
tagOrOperator: "使用或操作OR匹配多个标签。",
hideDuplicates: "使用校验码隐藏重复结果",
hideLegacy: "隐藏'legacyES' Elasticsearch 通知",
updateMimeMap: "媒体类型树的实时更新"
},
queryMode: {
simple: "简单",
advanced: "高级",
},
lang: {
en: "English",
fr: "Français",
"zh-CN": "简体中文",
},
displayMode: {
grid: "网格",
list: "列表",
},
columns: {
auto: "自动"
},
treemapType: {
cascaded: "折叠",
flat: "平铺(紧凑)"
},
treemapSize: {
small: "小",
medium: "中",
large: "大",
xLarge: "加大",
xxLarge: "加加大",
custom: "自订",
},
treemapTiling: {
binary: "Binary",
squarify: "Squarify",
slice: "Slice",
dice: "Dice",
sliceDice: "Slice & Dice",
},
theme: {
light: "亮",
black: "暗"
},
hit: "命中",
hits: "命中",
details: "详细信息",
stats: "统计信息",
queryTime: "查询时间",
totalSize: "总大小",
pathBar: {
placeholder: "过滤路径",
modalTitle: "选择路径"
},
debug: "调试信息",
debugDescription: "对调试除错有用的信息。 若您遇到bug或者想建议新功能请提交新Issue到" +
"<a href='https://github.com/simon987/sist2/issues/new/choose'>这里</a>.",
tagline: "标签栏",
toast: {
esConnErrTitle: "Elasticsearch连接错误",
esConnErr: "sist2 web 模块连接Elasticsearch出错。" +
"查看服务日志以获取更多信息。",
esQueryErrTitle: "查询错误",
esQueryErr: "无法识别或执行查询,请查阅高级搜索文档。" +
"查看服务日志以获取更多信息。",
dupeTagTitle: "重复标签",
dupeTag: "该标签已存在于此文档。",
copiedToClipboard: "复制到剪贴板"
},
saveTagModalTitle: "增加标签",
saveTagPlaceholder: "标签名",
confirm: "确认",
indexPickerPlaceholder: "选择一个索引",
sort: {
relevance: "相关度",
dateAsc: "日期(由旧到新)",
dateDesc: "日期(由新到旧)",
sizeAsc: "大小(从小到大)",
sizeDesc: "大小(从大到小)",
nameAsc: "名字A-z",
nameDesc: "名字 Z-a",
random: "随机",
},
d3: {
mimeCount: "各类文件数量分布",
mimeSize: "各类文件大小分布",
dateHistogram: "文件修改时间分布",
sizeHistogram: "文件大小分布",
},
indexPicker: {
selectNone: "清空",
selectAll: "全选",
selectedIndex: "选中索引",
selectedIndices: "选中索引",
},
},
}

29
sist2-vue/src/main.js Normal file
View File

@@ -0,0 +1,29 @@
import '@babel/polyfill'
import 'mutationobserver-shim'
import Vue from 'vue'
import './plugins/bootstrap-vue'
import App from './App.vue'
import router from './router'
import store from './store'
import VueI18n from "vue-i18n";
import messages from "@/i18n/messages";
import VueRouter from "vue-router";
Vue.config.productionTip = false;
Vue.use(VueI18n);
Vue.use(VueRouter);
const i18n = new VueI18n({
locale: "en",
messages: messages
});
new Vue({
router,
store,
i18n,
render: h => h(App)
}).$mount("#app");

View File

@@ -0,0 +1,7 @@
import Vue from "vue"
import BootstrapVue from "bootstrap-vue"
import "bootstrap/dist/css/bootstrap.min.css"
import "bootstrap-vue/dist/bootstrap-vue.css"
Vue.use(BootstrapVue)

View File

@@ -0,0 +1,36 @@
import Vue from "vue"
import VueRouter, {RouteConfig} from "vue-router"
import StatsPage from "../views/StatsPage.vue"
import Configuration from "../views/Configuration.vue"
import SearchPage from "@/views/SearchPage.vue";
Vue.use(VueRouter)
const routes: Array<RouteConfig> = [
{
path: "/",
name: "SearchPage",
component: SearchPage
},
{
path: "/stats",
name: "Stats",
component: StatsPage
},
{
path: "/config",
name: "Configuration",
component: Configuration
}
]
const router = new VueRouter({
mode: "hash",
base: process.env.BASE_URL,
routes,
scrollBehavior (to, from, savedPosition) {
// return desired position
}
})
export default router

17
sist2-vue/src/shims-tsx.d.ts vendored Normal file
View File

@@ -0,0 +1,17 @@
import Vue, {VNode} from 'vue'
declare global {
namespace JSX {
// tslint:disable no-empty-interface
interface Element extends VNode {
}
// tslint:disable no-empty-interface
interface ElementClass extends Vue {
}
interface IntrinsicElements {
[elem: string]: any
}
}
}

4
sist2-vue/src/shims-vue.d.ts vendored Normal file
View File

@@ -0,0 +1,4 @@
declare module '*.vue' {
import Vue from 'vue'
export default Vue
}

View File

@@ -0,0 +1,370 @@
import Vue from "vue"
import Vuex from "vuex"
import VueRouter, {Route} from "vue-router";
import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
import {deserializeMimes, serializeMimes} from "@/util";
Vue.use(Vuex)
export default new Vuex.Store({
state: {
seed: 0,
indices: [] as Index[],
tags: [] as EsTag[],
sist2Info: null as any,
sizeMin: undefined,
sizeMax: undefined,
dateBoundsMin: null,
dateBoundsMax: null,
dateMin: undefined,
dateMax: undefined,
searchText: "",
pathText: "",
sortMode: "score",
fuzzy: false,
size: 60,
optLang: "en",
optLangIsDefault: true,
optHideDuplicates: true,
optTheme: "light",
optDisplay: "grid",
optHighlight: true,
optTagOrOperator: false,
optFuzzy: true,
optFragmentSize: 200,
optQueryMode: "simple",
optSearchInPath: false,
optColumns: "auto",
optSuggestPath: true,
optTreemapType: "cascaded",
optTreemapTiling: "squarify",
optTreemapColorGroupingDepth: 3,
optTreemapSize: "medium",
optTreemapColor: "PuBuGn",
optLightboxLoadOnlyCurrent: false,
optLightboxSlideDuration: 15,
optHideLegacy: false,
optUpdateMimeMap: true,
_onLoadSelectedIndices: [] as string[],
_onLoadSelectedMimeTypes: [] as string[],
_onLoadSelectedTags: [] as string[],
selectedIndices: [] as Index[],
selectedMimeTypes: [] as string[],
selectedTags: [] as string[],
lastQueryResults: null,
keySequence: 0,
querySequence: 0,
uiTagHover: null as Tag | null,
uiLightboxIsOpen: false,
uiShowLightbox: false,
uiLightboxSources: [] as string[],
uiLightboxThumbs: [] as string[],
uiLightboxCaptions: [] as any[],
uiLightboxTypes: [] as string[],
uiLightboxKey: 0,
uiLightboxSlide: 0,
uiReachedScrollEnd: false,
uiDetailsMimeAgg: null,
uiShowDetails: false,
uiMimeMap: [] as any[]
},
mutations: {
setUiShowDetails: (state, val) => state.uiShowDetails = val,
setUiDetailsMimeAgg: (state, val) => state.uiDetailsMimeAgg = val,
setUiReachedScrollEnd: (state, val) => state.uiReachedScrollEnd = val,
setTags: (state, val) => state.tags = val,
setPathText: (state, val) => state.pathText = val,
setSizeMin: (state, val) => state.sizeMin = val,
setSizeMax: (state, val) => state.sizeMax = val,
setSist2Info: (state, val) => state.sist2Info = val,
setSeed: (state, val) => state.seed = val,
setOptHideDuplicates: (state, val) => state.optHideDuplicates = val,
setOptLang: (state, val) => {
state.optLang = val;
state.optLangIsDefault = false;
},
setSortMode: (state, val) => state.sortMode = val,
setIndices: (state, val) => {
state.indices = val;
if (state._onLoadSelectedIndices.length > 0) {
state.selectedIndices = val.filter(
(idx: Index) => state._onLoadSelectedIndices.some(prefix => idx.id.startsWith(prefix))
);
} else {
state.selectedIndices = val;
}
},
setDateMin: (state, val) => state.dateMin = val,
setDateMax: (state, val) => state.dateMax = val,
setDateBoundsMin: (state, val) => state.dateBoundsMin = val,
setDateBoundsMax: (state, val) => state.dateBoundsMax = val,
setSearchText: (state, val) => state.searchText = val,
setFuzzy: (state, val) => state.fuzzy = val,
setLastQueryResult: (state, val) => state.lastQueryResults = val,
_setOnLoadSelectedIndices: (state, val) => state._onLoadSelectedIndices = val,
_setOnLoadSelectedMimeTypes: (state, val) => state._onLoadSelectedMimeTypes = val,
_setOnLoadSelectedTags: (state, val) => state._onLoadSelectedTags = val,
setSelectedIndices: (state, val) => state.selectedIndices = val,
setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
setSelectedTags: (state, val) => state.selectedTags = val,
setUiTagHover: (state, val: Tag | null) => state.uiTagHover = val,
setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
_setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
_setKeySequence: (state, val: number) => state.keySequence = val,
_setQuerySequence: (state, val: number) => state.querySequence = val,
addLightboxSource: (state, {source, thumbnail, caption, type}) => {
state.uiLightboxSources.push(source);
state.uiLightboxThumbs.push(thumbnail);
state.uiLightboxCaptions.push(caption);
state.uiLightboxTypes.push(type);
},
setUiLightboxSlide: (state, val: number) => state.uiLightboxSlide = val,
setUiLightboxSources: (state, val) => state.uiLightboxSources = val,
setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
setUiLightboxTypes: (state, val) => state.uiLightboxTypes = val,
setUiLightboxCaptions: (state, val) => state.uiLightboxCaptions = val,
setOptTheme: (state, val) => state.optTheme = val,
setOptDisplay: (state, val) => state.optDisplay = val,
setOptColumns: (state, val) => state.optColumns = val,
setOptHighlight: (state, val) => state.optHighlight = val,
setOptFuzzy: (state, val) => state.fuzzy = val,
setOptSearchInPath: (state, val) => state.optSearchInPath = val,
setOptSuggestPath: (state, val) => state.optSuggestPath = val,
setOptFragmentSize: (state, val) => state.optFragmentSize = val,
setOptQueryMode: (state, val) => state.optQueryMode = val,
setOptResultSize: (state, val) => state.size = val,
setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
setOptTreemapType: (state, val) => state.optTreemapType = val,
setOptTreemapTiling: (state, val) => state.optTreemapTiling = val,
setOptTreemapColorGroupingDepth: (state, val) => state.optTreemapColorGroupingDepth = val,
setOptTreemapSize: (state, val) => state.optTreemapSize = val,
setOptTreemapColor: (state, val) => state.optTreemapColor = val,
setOptHideLegacy: (state, val) => state.optHideLegacy = val,
setOptUpdateMimeMap: (state, val) => state.optUpdateMimeMap = val,
setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
setUiMimeMap: (state, val) => state.uiMimeMap = val,
busUpdateWallItems: () => {
// noop
},
busUpdateTags: () => {
// noop
},
busSearch: () => {
// noop
},
},
actions: {
setSist2Info: (store, val) => {
store.commit("setSist2Info", val);
if (store.state.optLangIsDefault) {
store.commit("setOptLang", val.lang);
}
},
loadFromArgs({commit}, route: Route) {
if (route.query.q) {
commit("setSearchText", route.query.q);
}
if (route.query.fuzzy !== undefined) {
commit("setFuzzy", true);
}
if (route.query.i) {
commit("_setOnLoadSelectedIndices", Array.isArray(route.query.i) ? route.query.i : [route.query.i,]);
}
if (route.query.dMin) {
commit("setDateMin", Number(route.query.dMin))
}
if (route.query.dMax) {
commit("setDateMax", Number(route.query.dMax))
}
if (route.query.sMin) {
commit("setSizeMin", Number(route.query.sMin))
}
if (route.query.sMax) {
commit("setSizeMax", Number(route.query.sMax))
}
if (route.query.path) {
commit("setPathText", route.query.path)
}
if (route.query.m) {
commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m as string));
}
if (route.query.t) {
commit("_setOnLoadSelectedTags", (route.query.t as string).split(","));
}
if (route.query.sort) {
commit("setSortMode", route.query.sort);
commit("setSeed", Number(route.query.seed));
}
},
async updateArgs({state}, router: VueRouter) {
await router.push({
query: {
q: state.searchText.trim() ? state.searchText.trim().replace(/\s+/g, " ") : undefined,
fuzzy: state.fuzzy ? null : undefined,
i: state.selectedIndices ? state.selectedIndices.map((idx: Index) => idx.idPrefix) : undefined,
dMin: state.dateMin,
dMax: state.dateMax,
sMin: state.sizeMin,
sMax: state.sizeMax,
path: state.pathText ? state.pathText : undefined,
m: serializeMimes(state.selectedMimeTypes),
t: state.selectedTags.length == 0 ? undefined : state.selectedTags.join(","),
sort: state.sortMode === "score" ? undefined : state.sortMode,
seed: state.sortMode === "random" ? state.seed.toString() : undefined
}
}).catch(() => {
// ignore
});
},
updateConfiguration({state}) {
const conf = {} as any;
Object.keys(state).forEach((key) => {
if (key.startsWith("opt")) {
conf[key] = (state as any)[key];
}
});
localStorage.setItem("sist2_configuration", JSON.stringify(conf));
},
loadConfiguration({state}) {
const confString = localStorage.getItem("sist2_configuration");
if (confString) {
const conf = JSON.parse(confString);
Object.keys(state).forEach((key) => {
if (key.startsWith("opt")) {
(state as any)[key] = conf[key];
}
});
}
},
setSelectedIndices: ({commit}, val) => commit("setSelectedIndices", val),
getKeySequence({commit, state}) {
const val = state.keySequence;
commit("_setKeySequence", val + 1);
return val
},
incrementQuerySequence({commit, state}) {
const val = state.querySequence;
commit("_setQuerySequence", val + 1);
return val
},
remountLightbox({commit, state}) {
// Change key to an arbitrary number to force the lightbox to remount
commit("setUiLightboxKey", state.uiLightboxKey + 1);
},
showLightbox({commit, state}) {
commit("_setUiShowLightbox", !state.uiShowLightbox);
},
clearResults({commit}) {
commit("setLastQueryResult", null);
commit("_setKeySequence", 0);
commit("_setUiShowLightbox", false);
commit("setUiLightboxSources", []);
commit("setUiLightboxThumbs", []);
commit("setUiLightboxTypes", []);
commit("setUiLightboxCaptions", []);
commit("setUiLightboxKey", 0);
commit("setUiDetailsMimeAgg", null);
}
},
modules: {},
getters: {
seed: (state) => state.seed,
getPathText: (state) => state.pathText,
indices: state => state.indices,
sist2Info: state => state.sist2Info,
indexMap: state => {
const map = {} as any;
state.indices.forEach(idx => map[idx.id] = idx);
return map;
},
selectedIndices: (state) => state.selectedIndices,
_onLoadSelectedIndices: (state) => state._onLoadSelectedIndices,
selectedMimeTypes: (state) => state.selectedMimeTypes,
selectedTags: (state) => state.selectedTags,
dateMin: state => state.dateMin,
dateMax: state => state.dateMax,
sizeMin: state => state.sizeMin,
sizeMax: state => state.sizeMax,
searchText: state => state.searchText,
pathText: state => state.pathText,
fuzzy: state => state.fuzzy,
size: state => state.size,
sortMode: state => state.sortMode,
lastQueryResult: state => state.lastQueryResults,
lastDoc: function (state): EsHit | null {
if (state.lastQueryResults == null) {
return null;
}
return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
},
uiTagHover: state => state.uiTagHover,
uiShowLightbox: state => state.uiShowLightbox,
uiLightboxSources: state => state.uiLightboxSources,
uiLightboxThumbs: state => state.uiLightboxThumbs,
uiLightboxCaptions: state => state.uiLightboxCaptions,
uiLightboxTypes: state => state.uiLightboxTypes,
uiLightboxKey: state => state.uiLightboxKey,
uiLightboxSlide: state => state.uiLightboxSlide,
optHideDuplicates: state => state.optHideDuplicates,
optLang: state => state.optLang,
optTheme: state => state.optTheme,
optDisplay: state => state.optDisplay,
optColumns: state => state.optColumns,
optHighlight: state => state.optHighlight,
optTagOrOperator: state => state.optTagOrOperator,
optFuzzy: state => state.optFuzzy,
optSearchInPath: state => state.optSearchInPath,
optSuggestPath: state => state.optSuggestPath,
optFragmentSize: state => state.optFragmentSize,
optQueryMode: state => state.optQueryMode,
optTreemapType: state => state.optTreemapType,
optTreemapTiling: state => state.optTreemapTiling,
optTreemapSize: state => state.optTreemapSize,
optTreemapColorGroupingDepth: state => state.optTreemapColorGroupingDepth,
optTreemapColor: state => state.optTreemapColor,
optLightboxLoadOnlyCurrent: state => state.optLightboxLoadOnlyCurrent,
optLightboxSlideDuration: state => state.optLightboxSlideDuration,
optResultSize: state => state.size,
optHideLegacy: state => state.optHideLegacy,
optUpdateMimeMap: state => state.optUpdateMimeMap,
}
})

139
sist2-vue/src/util-js.js Normal file
View File

@@ -0,0 +1,139 @@
export function mergeTooltips(slider, threshold, separator, fixTooltips) {
const isMobile = window.innerWidth <= 650;
if (isMobile) {
threshold = 25;
}
const textIsRtl = getComputedStyle(slider).direction === 'rtl';
const isRtl = slider.noUiSlider.options.direction === 'rtl';
const isVertical = slider.noUiSlider.options.orientation === 'vertical';
const tooltips = slider.noUiSlider.getTooltips();
const origins = slider.noUiSlider.getOrigins();
// Move tooltips into the origin element. The default stylesheet handles this.
tooltips.forEach(function (tooltip, index) {
if (tooltip) {
origins[index].appendChild(tooltip);
}
});
slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
const pools = [[]];
const poolPositions = [[]];
const poolValues = [[]];
let atPool = 0;
// Assign the first tooltip to the first pool, if the tooltip is configured
if (tooltips[0]) {
pools[0][0] = 0;
poolPositions[0][0] = positions[0];
poolValues[0][0] = values[0];
}
for (let i = 1; i < positions.length; i++) {
if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
atPool++;
pools[atPool] = [];
poolValues[atPool] = [];
poolPositions[atPool] = [];
}
if (tooltips[i]) {
pools[atPool].push(i);
poolValues[atPool].push(values[i]);
poolPositions[atPool].push(positions[i]);
}
}
pools.forEach(function (pool, poolIndex) {
const handlesInPool = pool.length;
for (let j = 0; j < handlesInPool; j++) {
const handleNumber = pool[j];
if (j === handlesInPool - 1) {
let offset = 0;
poolPositions[poolIndex].forEach(function (value) {
offset += 1000 - 10 * value;
});
const direction = isVertical ? 'bottom' : 'right';
const last = isRtl ? 0 : handlesInPool - 1;
const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
// Center this tooltip over the affected handles
tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
tooltips[handleNumber].style.display = 'block';
tooltips[handleNumber].style[direction] = offset + '%';
} else {
// Hide this tooltip
tooltips[handleNumber].style.display = 'none';
}
}
});
if (fixTooltips) {
const isMobile = window.innerWidth <= 650;
const len = isMobile ? 20 : 5;
if (positions[0] < len) {
tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
} else {
tooltips[0].style.right = "0"
}
if (positions[1] > (100 - len)) {
tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
} else {
tooltips[1].style.right = "0"
}
}
});
}
export function burrow(table, addSelfDir, rootName) {
const root = {};
table.forEach(row => {
let layer = root;
row.taxonomy.forEach(key => {
layer[key] = key in layer ? layer[key] : {};
layer = layer[key];
});
if (Object.keys(layer).length === 0) {
layer["$size$"] = row.size;
} else if (addSelfDir) {
layer["."] = {
"$size$": row.size,
};
}
});
const descend = function (obj, depth) {
return Object.keys(obj).filter(k => k !== "$size$").map(k => {
const child = {
name: k,
depth: depth,
value: 0,
children: descend(obj[k], depth + 1)
};
if ("$size$" in obj[k]) {
child.value = obj[k]["$size$"];
}
return child;
});
};
return {
name: rootName,
children: descend(root, 1),
value: 0,
depth: 0,
}
}

161
sist2-vue/src/util.ts Normal file
View File

@@ -0,0 +1,161 @@
import {EsHit} from "@/Sist2Api";
export function ext(hit: EsHit) {
return Object.prototype.hasOwnProperty.call(hit._source, "extension")
&& hit["_source"]["extension"] !== "" ? "." + hit["_source"]["extension"] : "";
}
export function strUnescape(str: string): string {
let result = "";
for (let i = 0; i < str.length; i++) {
const c = str[i];
const next = str[i + 1];
if (c === "]") {
if (next === "]") {
result += c;
i += 1;
} else {
result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
i += 2;
}
} else {
result += c;
}
}
return result;
}
const thresh = 1000;
const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
export function humanFileSize(bytes: number): string {
if (bytes === 0) {
return "0 B"
}
if (Math.abs(bytes) < thresh) {
return bytes + ' B';
}
let u = -1;
do {
bytes /= thresh;
++u;
} while (Math.abs(bytes) >= thresh && u < units.length - 1);
return bytes.toFixed(1) + units[u];
}
export function humanTime(sec_num: number): string {
sec_num = Math.floor(sec_num);
const hours = Math.floor(sec_num / 3600);
const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
const seconds = sec_num - (hours * 3600) - (minutes * 60);
return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
}
export function humanDate(numMilis: number): string {
const date = (new Date(numMilis * 1000));
return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
}
export function lum(c: string) {
c = c.substring(1);
const rgb = parseInt(c, 16);
const r = (rgb >> 16) & 0xff;
const g = (rgb >> 8) & 0xff;
const b = (rgb >> 0) & 0xff;
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
}
export function getSelectedTreeNodes(tree: any) {
const selectedNodes = new Set();
const selected = tree.selected();
for (let i = 0; i < selected.length; i++) {
if (selected[i].id === "any") {
return ["any"]
}
//Only get children
if (selected[i].text.indexOf("(") !== -1) {
if (selected[i].values) {
selectedNodes.add(selected[i].values.slice(-1)[0]);
} else {
selectedNodes.add(selected[i].id);
}
}
}
return Array.from(selectedNodes);
}
export function getTreeNodeAttributes(tree: any) {
const nodes = tree.selectable();
const attributes = {};
for (let i = 0; i < nodes.length; i++) {
let id = null;
if (nodes[i].text.indexOf("(") !== -1 && nodes[i].values) {
id = nodes[i].values.slice(-1)[0];
} else {
id = nodes[i].id
}
attributes[id] = {
checked: nodes[i].itree.state.checked,
collapsed: nodes[i].itree.state.collapsed,
}
}
return attributes;
}
export function serializeMimes(mimes: string[]): string | undefined {
if (mimes.length == 0) {
return undefined;
}
return mimes.map(mime => compressMime(mime)).join("");
}
export function deserializeMimes(mimeString: string): string[] {
return mimeString
.replaceAll(/([IVATUF])/g, "$$$&")
.split("$")
.map(mime => decompressMime(mime))
.slice(1) // Ignore the first (empty) token
}
export function compressMime(mime: string): string {
return mime
.replace("image/", "I")
.replace("video/", "V")
.replace("application/", "A")
.replace("text/", "T")
.replace("audio/", "U")
.replace("font/", "F")
.replace("+", ",")
.replace("x-", "X")
}
export function decompressMime(mime: string): string {
return mime
.replace("I", "image/")
.replace("V", "video/")
.replace("A", "application/")
.replace("T", "text/")
.replace("U", "audio/")
.replace("F", "font/")
.replace(",", "+")
.replace("X", "x-")
}

View File

@@ -0,0 +1,289 @@
<template>
<!-- <div :style="{width: `${$store.getters.optContainerWidth}px`}"-->
<div
v-if="!configLoading"
style="margin-left: auto; margin-right: auto;" class="container">
<b-card>
<b-card-title>
<GearIcon></GearIcon>
{{ $t("config") }}
</b-card-title>
<p>{{ $t("configDescription") }}</p>
<b-card-body>
<h4>{{ $t("displayOptions") }}</h4>
<b-card>
<label><LanguageIcon/><span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
<b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
<label>{{ $t("opt.theme") }}</label>
<b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
<label>{{ $t("opt.displayMode") }}</label>
<b-form-select :options="displayModeOptions" :value="optDisplay" @input="setOptDisplay"></b-form-select>
<label>{{ $t("opt.columns") }}</label>
<b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
<div style="height: 10px"></div>
<b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
{{ $t("opt.lightboxLoadOnlyCurrent") }}
</b-form-checkbox>
<b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
{{ $t("opt.hideLegacy") }}
</b-form-checkbox>
<b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
{{ $t("opt.updateMimeMap") }}
</b-form-checkbox>
</b-card>
<br/>
<h4>{{ $t("searchOptions") }}</h4>
<b-card>
<b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
$t("opt.hideDuplicates")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
<b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
$t("opt.tagOrOperator")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
<b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
$t("opt.searchInPath")
}}
</b-form-checkbox>
<b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
$t("opt.suggestPath")
}}
</b-form-checkbox>
<br/>
<label>{{ $t("opt.fragmentSize") }}</label>
<b-form-input :value="optFragmentSize" step="10" type="number" min="0"
@input="setOptFragmentSize"></b-form-input>
<label>{{ $t("opt.resultSize") }}</label>
<b-form-input :value="optResultSize" type="number" min="10"
@input="setOptResultSize"></b-form-input>
<label>{{ $t("opt.queryMode") }}</label>
<b-form-select :options="queryModeOptions" :value="optQueryMode" @input="setOptQueryMode"></b-form-select>
<label>{{ $t("opt.slideDuration") }}</label>
<b-form-input :value="optLightboxSlideDuration" type="number" min="1"
@input="setOptLightboxSlideDuration"></b-form-input>
</b-card>
<h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
<b-card>
<label>{{ $t("opt.treemapType") }}</label>
<b-form-select :value="optTreemapType" :options="treemapTypeOptions"
@input="setOptTreemapType"></b-form-select>
<label>{{ $t("opt.treemapTiling") }}</label>
<b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
@input="setOptTreemapTiling"></b-form-select>
<label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
<b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
@input="setOptTreemapColorGroupingDepth"></b-form-input>
<label>{{ $t("opt.treemapSize") }}</label>
<b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
@input="setOptTreemapSize"></b-form-select>
<template v-if="$store.getters.optTreemapSize === 'custom'">
<!-- TODO Width/Height input -->
<b-form-input type="number" min="0" step="10"></b-form-input>
<b-form-input type="number" min="0" step="10"></b-form-input>
</template>
<label>{{ $t("opt.treemapColor") }}</label>
<b-form-select :value="optTreemapColor" :options="treemapColorOptions"
@input="setOptTreemapColor"></b-form-select>
</b-card>
<b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
</b-card-body>
</b-card>
<b-card v-if="loading" class="mt-4">
<Preloader></Preloader>
</b-card>
<DebugInfo v-else></DebugInfo>
</div>
</template>
<script>
import {mapActions, mapGetters, mapMutations} from "vuex";
import DebugInfo from "@/components/DebugInfo.vue";
import Preloader from "@/components/Preloader.vue";
import sist2 from "@/Sist2Api";
import GearIcon from "@/components/icons/GearIcon.vue";
import LanguageIcon from "@/components/icons/LanguageIcon";
export default {
components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
data() {
return {
loading: true,
configLoading: false,
langOptions: [
{value: "en", text: this.$t("lang.en")},
{value: "fr", text: this.$t("lang.fr")},
{value: "zh-CN", text: this.$t("lang.zh-CN")},
],
queryModeOptions: [
{value: "simple", text: this.$t("queryMode.simple")},
{value: "advanced", text: this.$t("queryMode.advanced")}
],
displayModeOptions: [
{value: "grid", text: this.$t("displayMode.grid")},
{value: "list", text: this.$t("displayMode.list")}
],
columnsOptions: [
{value: "auto", text: this.$t("columns.auto")},
{value: 1, text: "1"},
{value: 2, text: "2"},
{value: 3, text: "3"},
{value: 4, text: "4"},
{value: 5, text: "5"},
{value: 6, text: "6"},
{value: 7, text: "7"},
{value: 8, text: "8"},
{value: 9, text: "9"},
{value: 10, text: "10"},
{value: 11, text: "11"},
{value: 12, text: "12"},
],
treemapTypeOptions: [
{value: "cascaded", text: this.$t("treemapType.cascaded")},
{value: "flat", text: this.$t("treemapType.flat")}
],
treemapTilingOptions: [
{value: "binary", text: this.$t("treemapTiling.binary")},
{value: "squarify", text: this.$t("treemapTiling.squarify")},
{value: "slice", text: this.$t("treemapTiling.slice")},
{value: "dice", text: this.$t("treemapTiling.dice")},
{value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
],
treemapSizeOptions: [
{value: "small", text: this.$t("treemapSize.small")},
{value: "medium", text: this.$t("treemapSize.medium")},
{value: "large", text: this.$t("treemapSize.large")},
{value: "x-large", text: this.$t("treemapSize.xLarge")},
{value: "xx-large", text: this.$t("treemapSize.xxLarge")},
// {value: "custom", text: this.$t("treemapSize.custom")},
],
treemapColorOptions: [
{value: "PuBuGn", text: "Purple-Blue-Green"},
{value: "PuRd", text: "Purple-Red"},
{value: "PuBu", text: "Purple-Blue"},
{value: "YlOrBr", text: "Yellow-Orange-Brown"},
{value: "YlOrRd", text: "Yellow-Orange-Red"},
{value: "YlGn", text: "Yellow-Green"},
{value: "YlGnBu", text: "Yellow-Green-Blue"},
{value: "Plasma", text: "Plasma"},
{value: "Magma", text: "Magma"},
{value: "Inferno", text: "Inferno"},
{value: "Viridis", text: "Viridis"},
{value: "Turbo", text: "Turbo"},
],
themeOptions: [
{value: "light", text: this.$t("theme.light")},
{value: "black", text: this.$t("theme.black")}
]
}
},
computed: {
...mapGetters([
"optTheme",
"optDisplay",
"optColumns",
"optHighlight",
"optFuzzy",
"optSearchInPath",
"optSuggestPath",
"optFragmentSize",
"optQueryMode",
"optTreemapType",
"optTreemapTiling",
"optTreemapColorGroupingDepth",
"optTreemapColor",
"optTreemapSize",
"optLightboxLoadOnlyCurrent",
"optLightboxSlideDuration",
"optResultSize",
"optTagOrOperator",
"optLang",
"optHideDuplicates",
"optHideLegacy",
"optUpdateMimeMap",
]),
clientWidth() {
return window.innerWidth;
}
},
mounted() {
sist2.getSist2Info().then(data => {
this.setSist2Info(data);
this.loading = false;
});
this.$store.subscribe((mutation) => {
if (mutation.type.startsWith("setOpt")) {
this.$store.dispatch("updateConfiguration");
}
});
},
methods: {
...mapActions({
setSist2Info: "setSist2Info",
}),
...mapMutations([
"setOptTheme",
"setOptDisplay",
"setOptColumns",
"setOptHighlight",
"setOptFuzzy",
"setOptSearchInPath",
"setOptSuggestPath",
"setOptFragmentSize",
"setOptQueryMode",
"setOptTreemapType",
"setOptTreemapTiling",
"setOptTreemapColorGroupingDepth",
"setOptTreemapColor",
"setOptTreemapSize",
"setOptLightboxLoadOnlyCurrent",
"setOptLightboxSlideDuration",
"setOptResultSize",
"setOptTagOrOperator",
"setOptLang",
"setOptHideDuplicates",
"setOptHideLegacy",
"setOptUpdateMimeMap"
]),
onResetClick() {
localStorage.removeItem("sist2_configuration");
window.location.reload();
}
},
}
</script>
<style>
.shrink {
flex-grow: inherit;
}
</style>

View File

@@ -0,0 +1,314 @@
<template>
<div class="container">
<Lightbox></Lightbox>
<HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
<b-card v-if="uiLoading">
<Preloader></Preloader>
</b-card>
<b-card v-show="!uiLoading" id="search-panel">
<SearchBar @show-help="showHelp=true"></SearchBar>
<b-row>
<b-col style="height: 70px;" sm="6">
<SizeSlider></SizeSlider>
</b-col>
<b-col>
<PathTree @search="search(true)"></PathTree>
</b-col>
</b-row>
<b-row>
<b-col sm="6">
<b-row>
<b-col style="height: 70px;">
<DateSlider></DateSlider>
</b-col>
</b-row>
<b-row>
<b-col>
<IndexPicker></IndexPicker>
</b-col>
</b-row>
</b-col>
<b-col>
<b-tabs justified>
<b-tab :title="$t('mimeTypes')">
<MimePicker></MimePicker>
</b-tab>
<b-tab :title="$t('tags')">
<TagPicker></TagPicker>
</b-tab>
</b-tabs>
</b-col>
</b-row>
</b-card>
<div v-show="docs.length === 0 && !uiLoading">
<Preloader v-if="searchBusy" class="mt-3"></Preloader>
<ResultsCard></ResultsCard>
</div>
<div v-if="docs.length > 0">
<ResultsCard></ResultsCard>
<DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
<DocList v-else :docs="docs" :append="appendFunc"></DocList>
</div>
</div>
</template>
<script lang="ts">
import Preloader from "@/components/Preloader.vue";
import {mapActions, mapGetters, mapMutations} from "vuex";
import sist2 from "../Sist2Api";
import Sist2Api, {EsHit, EsResult} from "../Sist2Api";
import SearchBar from "@/components/SearchBar.vue";
import IndexPicker from "@/components/IndexPicker.vue";
import Vue from "vue";
import Sist2Query from "@/Sist2Query";
import _debounce from "lodash/debounce";
import DocCardWall from "@/components/DocCardWall.vue";
import Lightbox from "@/components/Lightbox.vue";
import LightboxCaption from "@/components/LightboxCaption.vue";
import MimePicker from "../components/MimePicker.vue";
import ResultsCard from "@/components/ResultsCard.vue";
import PathTree from "@/components/PathTree.vue";
import SizeSlider from "@/components/SizeSlider.vue";
import DateSlider from "@/components/DateSlider.vue";
import TagPicker from "@/components/TagPicker.vue";
import DocList from "@/components/DocList.vue";
import HelpDialog from "@/components/HelpDialog.vue";
export default Vue.extend({
components: {
HelpDialog,
DocList,
TagPicker,
DateSlider,
SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
},
data: () => ({
loading: false,
uiLoading: true,
search: undefined as any,
docs: [] as EsHit[],
docIds: new Set(),
docChecksums: new Set(),
searchBusy: false,
Sist2Query: Sist2Query,
showHelp: false
}),
computed: {
...mapGetters(["indices", "optDisplay"]),
},
mounted() {
this.search = _debounce(async (clear: boolean) => {
if (clear) {
await this.clearResults();
}
await this.searchNow(Sist2Query.searchQuery());
}, 350, {leading: false});
this.$store.dispatch("loadFromArgs", this.$route).then(() => {
this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
this.$store.subscribe((mutation) => {
if ([
"setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
"setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
"setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
].includes(mutation.type)) {
if (this.searchBusy) {
return;
}
this.search(true);
}
});
});
this.getDateRange().then((range: { min: number, max: number }) => {
this.setDateBoundsMin(range.min);
this.setDateBoundsMax(range.max);
sist2.getSist2Info().then(data => {
this.setSist2Info(data);
this.setIndices(data.indices);
Sist2Api.getMimeTypes(Sist2Query.searchQuery()).then(({mimeMap}) => {
this.$store.commit("setUiMimeMap", mimeMap);
this.uiLoading = false;
this.search(true);
});
}).catch(() => {
this.showErrorToast();
});
});
},
methods: {
...mapActions({
setSist2Info: "setSist2Info",
}),
...mapMutations({
setIndices: "setIndices",
setDateBoundsMin: "setDateBoundsMin",
setDateBoundsMax: "setDateBoundsMax",
setTags: "setTags",
}),
showErrorToast() {
this.$bvToast.toast(
this.$t("toast.esConnErr"),
{
title: this.$t("toast.esConnErrTitle"),
noAutoHide: true,
toaster: "b-toaster-bottom-right",
headerClass: "toast-header-error",
bodyClass: "toast-body-error",
});
},
showSyntaxErrorToast: function (): void {
this.$bvToast.toast(
this.$t("toast.esQueryErr"),
{
title: this.$t("toast.esQueryErrTitle"),
noAutoHide: true,
toaster: "b-toaster-bottom-right",
headerClass: "toast-header-warning",
bodyClass: "toast-body-warning",
});
},
async searchNow(q: any) {
this.searchBusy = true;
await this.$store.dispatch("incrementQuerySequence");
this.$store.commit("busSearch");
Sist2Api.esQuery(q).then(async (resp: EsResult) => {
await this.handleSearch(resp);
this.searchBusy = false;
}).catch(err => {
if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
this.showSyntaxErrorToast();
} else {
this.showErrorToast();
}
});
},
async clearResults() {
this.docs = [];
this.docIds.clear();
this.docChecksums.clear();
await this.$store.dispatch("clearResults");
this.$store.commit("setUiReachedScrollEnd", false);
},
async handleSearch(resp: EsResult) {
if (resp.hits.hits.length == 0) {
this.$store.commit("setUiReachedScrollEnd", true);
}
resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
if (this.$store.state.optHideDuplicates) {
resp.hits.hits = resp.hits.hits.filter(hit => {
if (!("checksum" in hit._source)) {
return true;
}
const isDupe = !this.docChecksums.has(hit._source.checksum);
this.docChecksums.add(hit._source.checksum);
return isDupe;
});
}
for (const hit of resp.hits.hits) {
if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
hit._seq = await this.$store.dispatch("getKeySequence");
this.$store.commit("addLightboxSource", {
source: `f/${hit._id}`,
thumbnail: hit._props.hasThumbnail
? `t/${hit._source.index}/${hit._id}`
: null,
caption: {
component: LightboxCaption,
props: {hit: hit}
},
type: hit._props.isVideo ? "video" : "image"
});
}
}
await this.$store.dispatch("remountLightbox");
this.$store.commit("setLastQueryResult", resp);
this.docs.push(...resp.hits.hits);
},
getDateRange(): Promise<{ min: number, max: number }> {
return sist2.esQuery({
// TODO: filter current selected indices
aggs: {
dateMin: {min: {field: "mtime"}},
dateMax: {max: {field: "mtime"}},
},
size: 0
}).then(res => {
return {
min: res.aggregations.dateMin.value,
max: res.aggregations.dateMax.value,
}
})
},
appendFunc() {
if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
this.searchNow(Sist2Query.searchQuery());
}
}
},
beforeRouteUpdate(to, from, next) {
if (this.$store.state.uiLightboxIsOpen) {
this.$store.commit("_setUiShowLightbox", false);
next(false);
} else {
next();
}
},
})
</script>
<style>
#search-panel {
box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
border-radius: 0;
border: none;
}
.toast-header-info, .toast-body-info {
background: #2196f3;
color: #fff !important;
}
.toast-header-error, .toast-body-error {
background: #a94442;
color: #f2dede !important;
}
.toast-header-error {
color: #fff !important;
border-bottom: none;
margin-bottom: -1em;
}
.toast-header-error .close {
text-shadow: none;
}
.toast-header-warning, .toast-body-warning {
background: #FF8F00;
color: #FFF3E0 !important;
}
</style>

View File

@@ -0,0 +1,79 @@
<template>
<b-container>
<b-card v-if="loading">
<Preloader></Preloader>
</b-card>
<template v-else>
<b-card>
<b-card-body>
<b-select v-model="selectedIndex" :options="indexOptions">
<template #first>
<b-form-select-option :value="null" disabled>{{ $t("indexPickerPlaceholder") }}</b-form-select-option>
</template>
</b-select>
</b-card-body>
</b-card>
<b-card v-if="selectedIndex !== null" class="mt-3">
<b-card-body>
<D3Treemap :index-id="selectedIndex"></D3Treemap>
</b-card-body>
</b-card>
<b-card v-if="selectedIndex !== null" class="stats-card mt-3">
<D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
<D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
<D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
<D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
</b-card>
</template>
</b-container>
</template>
<script>
import D3Treemap from "@/components/D3Treemap";
import Sist2Api from "@/Sist2Api";
import Preloader from "@/components/Preloader.vue";
import D3MimeBarCount from "@/components/D3MimeBarCount";
import D3MimeBarSize from "@/components/D3MimeBarSize";
import D3DateHistogram from "@/components/D3DateHistogram";
import D3SizeHistogram from "@/components/D3SizeHistogram";
export default {
components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
data() {
return {
loading: true,
selectedIndex: null,
indices: []
}
},
computed: {
indexOptions() {
return this.indices.map(idx => {
return {
text: idx.name,
value: idx.id
};
})
}
},
mounted() {
Sist2Api.getSist2Info().then(data => {
this.indices = data.indices;
this.loading = false;
})
}
}
</script>
<style>
.stats-card {
text-align: center;
padding: 1em;
}
</style>

40
sist2-vue/tsconfig.json Normal file
View File

@@ -0,0 +1,40 @@
{
"compilerOptions": {
"target": "esnext",
"module": "esnext",
"strict": false,
"jsx": "preserve",
"importHelpers": true,
"moduleResolution": "node",
"experimentalDecorators": true,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"sourceMap": false,
"baseUrl": ".",
"types": [
"webpack-env",
],
"paths": {
"@/*": [
"src/*"
]
},
"lib": [
"esnext",
"dom",
"dom.iterable",
"scripthost"
]
},
"include": [
"src/**/*.ts",
"src/**/*.tsx",
"src/**/*.vue",
"tests/**/*.ts",
"tests/**/*.tsx"
],
"exclude": [
"node_modules"
]
}

10
sist2-vue/vue.config.js Normal file
View File

@@ -0,0 +1,10 @@
module.exports = {
filenameHashing: false,
productionSourceMap: false,
publicPath: "./",
pages: {
index: {
entry: "src/main.js"
}
}
}

3
sist2-vue/watch.sh Executable file
View File

@@ -0,0 +1,3 @@
#!/usr/bin/env bash
./node_modules/@vue/cli-service/bin/vue-cli-service.js build --watch

View File

@@ -4,13 +4,15 @@
#define DEFAULT_OUTPUT "index.sist2/"
#define DEFAULT_CONTENT_SIZE 32768
#define DEFAULT_QUALITY 5
#define DEFAULT_SIZE 500
#define DEFAULT_QUALITY 1
#define DEFAULT_SIZE 300
#define DEFAULT_REWRITE_URL ""
#define DEFAULT_ES_URL "http://localhost:9200"
#define DEFAULT_ES_INDEX "sist2"
#define DEFAULT_BATCH_SIZE 100
#define DEFAULT_TAGLINE "Lightning-fast file system indexer and search tool"
#define DEFAULT_LANG "en"
#define DEFAULT_LISTEN_ADDRESS "localhost:4090"
#define DEFAULT_TREEMAP_THRESHOLD 0.0005
@@ -20,6 +22,7 @@
const char *TESS_DATAPATHS[] = {
"/usr/share/tessdata/",
"/usr/share/tesseract-ocr/tessdata/",
"/usr/share/tesseract-ocr/4.00/tessdata/",
"./",
NULL
};
@@ -91,7 +94,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
if (args->incremental != NULL) {
args->incremental = abspath(args->incremental);
if (abs_path == NULL) {
sist_log("main.c", SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
sist_log("main.c", LOG_SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
args->incremental = NULL;
}
}
@@ -143,7 +146,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
if (args->name == NULL) {
args->name = g_path_get_basename(args->output);
} else {
char* tmp = malloc(strlen(args->name) + 1);
char *tmp = malloc(strlen(args->name) + 1);
strcpy(tmp, args->name);
args->name = tmp;
}
@@ -165,17 +168,50 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
return 1;
}
if (args->tesseract_lang != NULL) {
TessBaseAPI *api = TessBaseAPICreate();
if (args->ocr_images && args->tesseract_lang == NULL) {
fprintf(stderr, "You must specify --ocr-lang <LANG> to use --ocr-images");
return 1;
}
char filename[128];
sprintf(filename, "%s.traineddata", args->tesseract_lang);
const char *path = find_file_in_paths(TESS_DATAPATHS, filename);
if (path == NULL) {
LOG_FATAL("cli.c", "Could not find tesseract language file!");
if (args->ocr_ebooks && args->tesseract_lang == NULL) {
fprintf(stderr, "You must specify --ocr-lang <LANG> to use --ocr-ebooks");
return 1;
}
if (args->tesseract_lang != NULL) {
if (!args->ocr_ebooks && !args->ocr_images) {
fprintf(stderr, "You must specify at least one of --ocr-ebooks, --ocr-images");
return 1;
}
ret = TessBaseAPIInit3(api, path, args->tesseract_lang);
TessBaseAPI *api = TessBaseAPICreate();
const char *trained_data_path = NULL;
char *lang = malloc(strlen(args->tesseract_lang) + 1);
strcpy(lang, args->tesseract_lang);
lang = strtok(lang, "+");
while (lang != NULL) {
char filename[128];
sprintf(filename, "%s.traineddata", lang);
const char *path = find_file_in_paths(TESS_DATAPATHS, filename);
if (path == NULL) {
LOG_FATALF("cli.c", "Could not find tesseract language file: %s!", filename);
}
if (trained_data_path != NULL && path != trained_data_path) {
LOG_FATAL("cli.c", "When specifying more than one tesseract language, all the traineddata "
"files must be in the same folder")
}
trained_data_path = path;
lang = strtok(NULL, "+");
}
free(lang);
ret = TessBaseAPIInit3(api, trained_data_path, args->tesseract_lang);
if (ret != 0) {
fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang);
return 1;
@@ -183,7 +219,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
TessBaseAPIEnd(api);
TessBaseAPIDelete(api);
args->tesseract_path = path;
args->tesseract_path = trained_data_path;
}
if (args->exclude_regex != NULL) {
@@ -216,6 +252,19 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
args->max_memory_buffer = DEFAULT_MAX_MEM_BUFFER;
}
if (args->list_path != NULL) {
if (strcmp(args->list_path, "-") == 0) {
args->list_file = stdin;
LOG_DEBUG("cli.c", "Using stdin as list file")
} else {
args->list_file = fopen(args->list_path, "r");
if (args->list_file == NULL) {
LOG_FATALF("main.c", "List file could not be opened: %s (%s)", args->list_path, errno);
}
}
}
LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
LOG_DEBUGF("cli.c", "arg size=%d", args->size)
LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
@@ -232,8 +281,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)
LOG_DEBUGF("cli.c", "arg fast=%d", args->fast)
LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub)
LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold)
LOG_DEBUGF("cli.c", "arg max_memory_buffer=%d", args->max_memory_buffer)
LOG_DEBUGF("cli.c", "arg list_path=%s", args->list_path)
return 0;
}
@@ -359,6 +410,19 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
args->es_index = DEFAULT_ES_INDEX;
}
if (args->tagline == NULL) {
args->tagline = DEFAULT_TAGLINE;
}
if (args->lang == NULL) {
args->lang = DEFAULT_LANG;
}
if (strlen(args->lang) != 2 && strlen(args->lang) != 5) {
fprintf(stderr, "Invalid --lang value, see usage\n");
return 1;
}
if (args->credentials != NULL) {
char *ptr = strstr(args->credentials, ":");
if (ptr == NULL) {
@@ -417,6 +481,8 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline)
LOG_DEBUGF("cli.c", "arg dev=%d", args->dev)
LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)
LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials)

View File

@@ -21,12 +21,18 @@ typedef struct scan_args {
char *archive_passphrase;
char *tesseract_lang;
const char *tesseract_path;
int ocr_images;
int ocr_ebooks;
char *exclude_regex;
int fast;
const char* treemap_threshold_str;
double treemap_threshold;
int max_memory_buffer;
int read_subtitles;
int fast_epub;
int calculate_checksums;
char *list_path;
FILE *list_file;
} scan_args_t;
scan_args_t *scan_args_create();
@@ -58,11 +64,14 @@ typedef struct web_args {
char *listen_address;
char *credentials;
char *tag_credentials;
char *tagline;
char *lang;
char auth_user[256];
char auth_pass[256];
int auth_enabled;
int tag_auth_enabled;
int index_count;
int dev;
const char **indices;
} web_args_t;

View File

@@ -1,6 +1,11 @@
#include "ctx.h"
ScanCtx_t ScanCtx;
ScanCtx_t ScanCtx = {
.stat_index_size = 0,
.stat_tn_size = 0,
.dbg_current_files = NULL,
.pool = NULL
};
WebCtx_t WebCtx;
IndexCtx_t IndexCtx;
LogCtx_t LogCtx;

View File

@@ -14,7 +14,10 @@
#include "libscan/mobi/scan_mobi.h"
#include "libscan/raw/raw.h"
#include "libscan/msdoc/msdoc.h"
#include "libscan/wpd/wpd.h"
#include "libscan/json/json.h"
#include "src/io/store.h"
#include "src/index/elastic.h"
#include <glib.h>
#include <pcre.h>
@@ -27,20 +30,30 @@ typedef struct {
tpool_t *pool;
tpool_t *writer_pool;
int threads;
int depth;
int calculate_checksums;
size_t stat_tn_size;
size_t stat_index_size;
GHashTable *original_table;
GHashTable *copy_table;
pthread_mutex_t copy_table_mu;
pcre *exclude;
pcre_extra *exclude_extra;
int fast;
GHashTable *dbg_current_files;
pthread_mutex_t dbg_current_files_mu;
int dbg_failed_files_count;
int dbg_skipped_files_count;
int dbg_excluded_files_count;
pthread_mutex_t dbg_file_counts_mu;
scan_arc_ctx_t arc_ctx;
scan_comic_ctx_t comic_ctx;
@@ -52,6 +65,8 @@ typedef struct {
scan_mobi_ctx_t mobi_ctx;
scan_raw_ctx_t raw_ctx;
scan_msdoc_ctx_t msdoc_ctx;
scan_wpd_ctx_t wpd_ctx;
scan_json_ctx_t json_ctx;
} ScanCtx_t;
typedef struct {
@@ -62,6 +77,7 @@ typedef struct {
typedef struct {
char *es_url;
es_version_t *es_version;
char *es_index;
int batch_size;
tpool_t *pool;
@@ -73,13 +89,17 @@ typedef struct {
typedef struct {
char *es_url;
es_version_t *es_version;
char *es_index;
int index_count;
char *auth_user;
char *auth_pass;
int auth_enabled;
int tag_auth_enabled;
struct index_t indices[64];
char *tagline;
struct index_t indices[256];
char lang[10];
int dev;
} WebCtx_t;
extern ScanCtx_t ScanCtx;

View File

@@ -253,7 +253,7 @@ void _elastic_flush(int max) {
} else {
print_errors(r);
LOG_INFOF("elastic.c", "Indexed %d documents (%zukB) <%d>", count, buf_len / 1024, r->status_code);
LOG_DEBUGF("elastic.c", "Indexed %d documents (%zukB) <%d>", count, buf_len / 1024, r->status_code);
delete_queue(max);
if (Indexer->queued != 0) {
@@ -356,7 +356,65 @@ void finish_indexer(char *script, int async_script, char *index_id) {
free_response(r);
}
void elastic_init(int force_reset, const char* user_mappings, const char* user_settings) {
es_version_t *elastic_get_version(const char *es_url) {
response_t *r = web_get(es_url, 30);
char *tmp = malloc(r->size + 1);
memcpy(tmp, r->body, r->size);
*(tmp + r->size) = '\0';
cJSON *response = cJSON_Parse(tmp);
free(tmp);
free_response(r);
if (response == NULL) {
return NULL;
}
if (cJSON_GetObjectItem(response, "version") == NULL ||
cJSON_GetObjectItem(cJSON_GetObjectItem(response, "version"), "number") == NULL) {
cJSON_Delete(response);
return NULL;
}
char *version_str = cJSON_GetObjectItem(cJSON_GetObjectItem(response, "version"), "number")->valuestring;
es_version_t *version = malloc(sizeof(es_version_t));
const char *tok = strtok(version_str, ".");
version->major = atoi(tok);
tok = strtok(NULL, ".");
version->minor = atoi(tok);
tok = strtok(NULL, ".");
version->patch = atoi(tok);
cJSON_Delete(response);
return version;
}
void elastic_init(int force_reset, const char *user_mappings, const char *user_settings) {
es_version_t *es_version = elastic_get_version(IndexCtx.es_url);
IndexCtx.es_version = es_version;
if (es_version == NULL) {
LOG_FATAL("elastic.c", "Could not get ES version")
}
LOG_INFOF("elastic.c",
"Elasticsearch version is %s (supported=%d, legacy=%d)",
format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), USE_LEGACY_ES_SETTINGS(es_version));
if (!IS_SUPPORTED_ES_VERSION(es_version)) {
LOG_FATAL("elastic.c", "sist2 only supports Elasticsearch v6.8 or newer")
}
char *settings = NULL;
if (USE_LEGACY_ES_SETTINGS(es_version)) {
settings = settings_json;
} else {
settings = settings_legacy_json;
}
// Check if index exists
char url[4096];
@@ -392,13 +450,21 @@ void elastic_init(int force_reset, const char* user_mappings, const char* user_s
free_response(r);
snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
r = web_put(url, user_settings ? user_settings : settings_json);
LOG_INFOF("elastic.c", "Update user_settings <%d>", r->status_code);
r = web_put(url, user_settings ? user_settings : settings);
LOG_INFOF("elastic.c", "Update ES settings <%d>", r->status_code);
if (r->status_code != 200) {
print_error(r);
LOG_FATAL("elastic.c", "Could not update user settings")
}
free_response(r);
snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index);
r = web_put(url, user_mappings ? user_mappings : mappings_json);
LOG_INFOF("elastic.c", "Update user_mappings <%d>", r->status_code);
LOG_INFOF("elastic.c", "Update ES mappings <%d>", r->status_code);
if (r->status_code != 200) {
print_error(r);
LOG_FATAL("elastic.c", "Could not update user mappings")
}
free_response(r);
snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index);

View File

@@ -9,6 +9,26 @@ typedef struct es_bulk_line {
char line[0];
} es_bulk_line_t;
typedef struct {
int major;
int minor;
int patch;
} es_version_t;
#define VERSION_GE(version, maj, min) ((version)->major > (maj) || ((version)->major == (maj) && (version)->minor >= (min)))
#define IS_SUPPORTED_ES_VERSION(es_version) VERSION_GE((es_version), 6, 8)
#define USE_LEGACY_ES_SETTINGS(es_version) (!VERSION_GE((es_version), 7, 14))
__always_inline
static const char *format_es_version(es_version_t *version) {
static char buf[64];
snprintf(buf, sizeof(buf), "%d.%d.%d", version->major, version->minor, version->patch);
return buf;
}
/**
* Note: indexer is *not* thread safe
*/
@@ -31,6 +51,8 @@ cJSON *elastic_get_document(const char *id_str);
char *elastic_get_status();
es_version_t *elastic_get_version(const char *es_url);
void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]);
#endif

File diff suppressed because one or more lines are too long

View File

@@ -3,35 +3,282 @@
#include "src/parsing/parse.h"
#include "src/parsing/mime.h"
static __thread int index_fd = -1;
#include <zstd.h>
typedef struct {
unsigned char path_md5[MD5_DIGEST_LENGTH];
unsigned long size;
unsigned int mime;
int mtime;
short base;
short ext;
char has_parent;
} line_t;
char *get_meta_key_text(enum metakey meta_key) {
#define META_NEXT 0xFFFF
switch (meta_key) {
case MetaContent:
return "content";
case MetaWidth:
return "width";
case MetaHeight:
return "height";
case MetaMediaDuration:
return "duration";
case MetaMediaAudioCodec:
return "audioc";
case MetaMediaVideoCodec:
return "videoc";
case MetaMediaBitrate:
return "bitrate";
case MetaArtist:
return "artist";
case MetaAlbum:
return "album";
case MetaAlbumArtist:
return "album_artist";
case MetaGenre:
return "genre";
case MetaTitle:
return "title";
case MetaFontName:
return "font_name";
case MetaParent:
return "parent";
case MetaExifMake:
return "exif_make";
case MetaExifDescription:
return "exif_description";
case MetaExifSoftware:
return "exif_software";
case MetaExifExposureTime:
return "exif_exposure_time";
case MetaExifFNumber:
return "exif_fnumber";
case MetaExifFocalLength:
return "exif_focal_length";
case MetaExifUserComment:
return "exif_user_comment";
case MetaExifIsoSpeedRatings:
return "exif_iso_speed_ratings";
case MetaExifModel:
return "exif_model";
case MetaExifDateTime:
return "exif_datetime";
case MetaAuthor:
return "author";
case MetaModifiedBy:
return "modified_by";
case MetaThumbnail:
return "thumbnail";
case MetaPages:
return "pages";
case MetaExifGpsLongitudeRef:
return "exif_gps_longitude_ref";
case MetaExifGpsLongitudeDMS:
return "exif_gps_longitude_dms";
case MetaExifGpsLongitudeDec:
return "exif_gps_longitude_dec";
case MetaExifGpsLatitudeRef:
return "exif_gps_latitude_ref";
case MetaExifGpsLatitudeDMS:
return "exif_gps_latitude_dms";
case MetaExifGpsLatitudeDec:
return "exif_gps_latitude_dec";
case MetaChecksum:
return "checksum";
default:
LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key)
}
}
void skip_meta(FILE *file) {
enum metakey key = 0;
fread(&key, sizeof(uint16_t), 1, file);
char *build_json_string(document_t *doc) {
cJSON *json = cJSON_CreateObject();
int buffer_size_guess = 8192;
while (key != META_NEXT) {
if (IS_META_INT(key)) {
fseek(file, sizeof(int), SEEK_CUR);
} else if (IS_META_LONG(key)) {
fseek(file, sizeof(long), SEEK_CUR);
} else {
while ((getc(file))) {}
const char *mime_text = mime_get_mime_text(doc->mime);
if (mime_text == NULL) {
cJSON_AddNullToObject(json, "mime");
} else {
cJSON_AddStringToObject(json, "mime", mime_text);
}
cJSON_AddNumberToObject(json, "size", (double) doc->size);
cJSON_AddNumberToObject(json, "mtime", doc->mtime);
// Ignore root directory in the file path
doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
doc->base = (short) (doc->base - ScanCtx.index.desc.root_len);
char *filepath = doc->filepath + ScanCtx.index.desc.root_len;
cJSON_AddStringToObject(json, "extension", filepath + doc->ext);
// Remove extension
if (*(filepath + doc->ext - 1) == '.') {
*(filepath + doc->ext - 1) = '\0';
} else {
*(filepath + doc->ext) = '\0';
}
char filepath_escaped[PATH_MAX * 3];
str_escape(filepath_escaped, filepath + doc->base);
cJSON_AddStringToObject(json, "name", filepath_escaped);
if (doc->base > 0) {
*(filepath + doc->base - 1) = '\0';
str_escape(filepath_escaped, filepath);
cJSON_AddStringToObject(json, "path", filepath_escaped);
} else {
cJSON_AddStringToObject(json, "path", "");
}
char md5_str[MD5_STR_LENGTH];
buf2hex(doc->path_md5, MD5_DIGEST_LENGTH, md5_str);
cJSON_AddStringToObject(json, "_id", md5_str);
// Metadata
meta_line_t *meta = doc->meta_head;
while (meta != NULL) {
switch (meta->key) {
case MetaPages:
case MetaWidth:
case MetaHeight:
case MetaMediaDuration:
case MetaMediaBitrate: {
cJSON_AddNumberToObject(json, get_meta_key_text(meta->key), (double) meta->long_val);
buffer_size_guess += 20;
break;
}
case MetaMediaAudioCodec:
case MetaMediaVideoCodec:
case MetaContent:
case MetaArtist:
case MetaAlbum:
case MetaAlbumArtist:
case MetaGenre:
case MetaFontName:
case MetaParent:
case MetaExifMake:
case MetaExifDescription:
case MetaExifSoftware:
case MetaExifExposureTime:
case MetaExifFNumber:
case MetaExifFocalLength:
case MetaExifUserComment:
case MetaExifIsoSpeedRatings:
case MetaExifDateTime:
case MetaExifModel:
case MetaAuthor:
case MetaModifiedBy:
case MetaThumbnail:
case MetaExifGpsLongitudeDMS:
case MetaExifGpsLongitudeDec:
case MetaExifGpsLongitudeRef:
case MetaExifGpsLatitudeDMS:
case MetaExifGpsLatitudeDec:
case MetaExifGpsLatitudeRef:
case MetaChecksum:
case MetaTitle: {
cJSON_AddStringToObject(json, get_meta_key_text(meta->key), meta->str_val);
buffer_size_guess += (int) strlen(meta->str_val);
break;
}
default:
LOG_FATALF("serialize.c", "Invalid meta key: %x %s", meta->key, get_meta_key_text(meta->key))
}
fread(&key, sizeof(uint16_t), 1, file);
meta_line_t *tmp = meta;
meta = meta->next;
free(tmp);
}
char *json_str = cJSON_PrintBuffered(json, buffer_size_guess, FALSE);
cJSON_Delete(json);
return json_str;
}
static struct {
FILE *out_file;
size_t buf_out_size;
void *buf_out;
ZSTD_CCtx *cctx;
} WriterCtx = {
.out_file = NULL
};
#define ZSTD_COMPRESSION_LEVEL 10
void initialize_writer_ctx(const char *file_path) {
WriterCtx.out_file = fopen(file_path, "wb");
WriterCtx.buf_out_size = ZSTD_CStreamOutSize();
WriterCtx.buf_out = malloc(WriterCtx.buf_out_size);
WriterCtx.cctx = ZSTD_createCCtx();
ZSTD_CCtx_setParameter(WriterCtx.cctx, ZSTD_c_compressionLevel, ZSTD_COMPRESSION_LEVEL);
ZSTD_CCtx_setParameter(WriterCtx.cctx, ZSTD_c_checksumFlag, FALSE);
LOG_DEBUGF("serialize.c", "Open index file for writing %s", file_path)
}
void zstd_write_string(const char *string, const size_t len) {
ZSTD_inBuffer input = {string, len, 0};
do {
ZSTD_outBuffer output = {WriterCtx.buf_out, WriterCtx.buf_out_size, 0};
ZSTD_compressStream2(WriterCtx.cctx, &output, &input, ZSTD_e_continue);
if (output.pos > 0) {
ScanCtx.stat_index_size += fwrite(WriterCtx.buf_out, 1, output.pos, WriterCtx.out_file);
}
} while (input.pos != input.size);
}
void write_document_func(void *arg) {
if (WriterCtx.out_file == NULL) {
char dstfile[PATH_MAX];
snprintf(dstfile, PATH_MAX, "%s_index_main.ndjson.zst", ScanCtx.index.path);
initialize_writer_ctx(dstfile);
}
document_t *doc = arg;
char *json_str = build_json_string(doc);
const size_t json_str_len = strlen(json_str);
json_str = realloc(json_str, json_str_len + 1);
*(json_str + json_str_len) = '\n';
zstd_write_string(json_str, json_str_len + 1);
free(json_str);
free(doc->filepath);
}
void zstd_close() {
if (WriterCtx.out_file == NULL) {
LOG_DEBUG("serialize.c", "No zstd stream to close, skipping cleanup")
return;
}
size_t remaining;
do {
ZSTD_outBuffer output = {WriterCtx.buf_out, WriterCtx.buf_out_size, 0};
remaining = ZSTD_endStream(WriterCtx.cctx, &output);
if (output.pos > 0) {
ScanCtx.stat_index_size += fwrite(WriterCtx.buf_out, 1, output.pos, WriterCtx.out_file);
}
} while (remaining != 0);
ZSTD_freeCCtx(WriterCtx.cctx);
free(WriterCtx.buf_out);
fclose(WriterCtx.out_file);
LOG_DEBUG("serialize.c", "End zstd stream & close index file")
}
void writer_cleanup() {
zstd_close();
WriterCtx.out_file = NULL;
}
void write_index_descriptor(char *path, index_descriptor_t *desc) {
@@ -49,7 +296,7 @@ void write_index_descriptor(char *path, index_descriptor_t *desc) {
LOG_FATALF("serialize.c", "Could not open index descriptor: %s", strerror(errno));
}
char *str = cJSON_Print(json);
int ret = write(fd, str, strlen(str));
size_t ret = write(fd, str, strlen(str));
if (ret == -1) {
LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno));
}
@@ -88,7 +335,7 @@ index_descriptor_t read_index_descriptor(char *path) {
strcpy(descriptor.version, cJSON_GetObjectItem(json, "version")->valuestring);
strcpy(descriptor.id, cJSON_GetObjectItem(json, "id")->valuestring);
if (cJSON_GetObjectItem(json, "type") == NULL) {
strcpy(descriptor.type, INDEX_TYPE_BIN);
strcpy(descriptor.type, INDEX_TYPE_NDJSON);
} else {
strcpy(descriptor.type, cJSON_GetObjectItem(json, "type")->valuestring);
}
@@ -99,392 +346,165 @@ index_descriptor_t read_index_descriptor(char *path) {
return descriptor;
}
char *get_meta_key_text(enum metakey meta_key) {
switch (meta_key) {
case MetaContent:
return "content";
case MetaWidth:
return "width";
case MetaHeight:
return "height";
case MetaMediaDuration:
return "duration";
case MetaMediaAudioCodec:
return "audioc";
case MetaMediaVideoCodec:
return "videoc";
case MetaMediaBitrate:
return "bitrate";
case MetaArtist:
return "artist";
case MetaAlbum:
return "album";
case MetaAlbumArtist:
return "album_artist";
case MetaGenre:
return "genre";
case MetaTitle:
return "title";
case MetaFontName:
return "font_name";
case MetaParent:
return "parent";
case MetaExifMake:
return "exif_make";
case MetaExifSoftware:
return "exif_software";
case MetaExifExposureTime:
return "exif_exposure_time";
case MetaExifFNumber:
return "exif_fnumber";
case MetaExifFocalLength:
return "exif_focal_length";
case MetaExifUserComment:
return "exif_user_comment";
case MetaExifIsoSpeedRatings:
return "exif_iso_speed_ratings";
case MetaExifModel:
return "exif_model";
case MetaExifDateTime:
return "exif_datetime";
case MetaAuthor:
return "author";
case MetaModifiedBy:
return "modified_by";
case MetaThumbnail:
return "thumbnail";
case MetaPages:
return "pages";
case MetaExifGpsLongitudeRef:
return "exif_gps_longitude_ref";
case MetaExifGpsLongitudeDMS:
return "exif_gps_longitude_dms";
case MetaExifGpsLongitudeDec:
return "exif_gps_longitude_dec";
case MetaExifGpsLatitudeRef:
return "exif_gps_latitude_ref";
case MetaExifGpsLatitudeDMS:
return "exif_gps_latitude_dms";
case MetaExifGpsLatitudeDec:
return "exif_gps_latitude_dec";
default:
LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key)
}
}
void write_document(document_t *doc) {
if (index_fd == -1) {
char dstfile[PATH_MAX];
pthread_t self = pthread_self();
snprintf(dstfile, PATH_MAX, "%s_index_%lu", ScanCtx.index.path, self);
index_fd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR);
if (index_fd == -1) {
perror("open");
}
}
dyn_buffer_t buf = dyn_buffer_create();
// Ignore root directory in the file path
doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
doc->base = (short) (doc->base - ScanCtx.index.desc.root_len);
doc->filepath += ScanCtx.index.desc.root_len;
dyn_buffer_write(&buf, doc, sizeof(line_t));
dyn_buffer_write_str(&buf, doc->filepath);
meta_line_t *meta = doc->meta_head;
while (meta != NULL) {
dyn_buffer_write_short(&buf, (uint16_t) meta->key);
if (IS_META_INT(meta->key)) {
dyn_buffer_write_int(&buf, meta->int_val);
} else if (IS_META_LONG(meta->key)) {
dyn_buffer_write_long(&buf, meta->long_val);
} else {
dyn_buffer_write_str(&buf, meta->str_val);
}
meta_line_t *tmp = meta;
meta = meta->next;
free(tmp);
}
dyn_buffer_write_short(&buf, META_NEXT);
int res = write(index_fd, buf.buf, buf.cur);
if (res == -1) {
LOG_FATALF("serialize.c", "Could not write document: %s", strerror(errno))
}
ScanCtx.stat_index_size += buf.cur;
dyn_buffer_destroy(&buf);
tpool_add_work(ScanCtx.writer_pool, write_document_func, doc);
}
void thread_cleanup() {
close(index_fd);
cleanup_parse();
cleanup_font();
}
void read_index_bin_handle_line(const char *line, const char *index_id, index_func func) {
void read_index_bin(const char *path, const char *index_id, index_func func) {
line_t line;
dyn_buffer_t buf = dyn_buffer_create();
cJSON *document = cJSON_Parse(line);
const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring;
FILE *file = fopen(path, "rb");
while (TRUE) {
buf.cur = 0;
size_t _ = fread((void *) &line, sizeof(line_t), 1, file);
if (feof(file)) {
break;
}
cJSON_AddStringToObject(document, "index", index_id);
cJSON *document = cJSON_CreateObject();
cJSON_AddStringToObject(document, "index", index_id);
// Load meta from sidecar files
cJSON *meta_obj = NULL;
if (IndexCtx.meta != NULL) {
const char *meta_string = g_hash_table_lookup(IndexCtx.meta, path_md5_str);
if (meta_string != NULL) {
meta_obj = cJSON_Parse(meta_string);
char path_md5_str[MD5_STR_LENGTH];
buf2hex(line.path_md5, sizeof(line.path_md5), path_md5_str);
const char *mime_text = mime_get_mime_text(line.mime);
if (mime_text == NULL) {
cJSON_AddNullToObject(document, "mime");
} else {
cJSON_AddStringToObject(document, "mime", mime_get_mime_text(line.mime));
}
cJSON_AddNumberToObject(document, "size", (double) line.size);
cJSON_AddNumberToObject(document, "mtime", line.mtime);
int c = 0;
while ((c = getc(file)) != 0) {
dyn_buffer_write_char(&buf, (char) c);
}
dyn_buffer_write_char(&buf, '\0');
cJSON_AddStringToObject(document, "extension", buf.buf + line.ext);
if (*(buf.buf + line.ext - 1) == '.') {
*(buf.buf + line.ext - 1) = '\0';
} else {
*(buf.buf + line.ext) = '\0';
}
char tmp[PATH_MAX * 3];
str_escape(tmp, buf.buf + line.base);
cJSON_AddStringToObject(document, "name", tmp);
if (line.base > 0) {
*(buf.buf + line.base - 1) = '\0';
str_escape(tmp, buf.buf);
cJSON_AddStringToObject(document, "path", tmp);
} else {
cJSON_AddStringToObject(document, "path", "");
}
enum metakey key = 0;
fread(&key, sizeof(uint16_t), 1, file);
size_t ret;
while (key != META_NEXT) {
switch (key) {
case MetaPages:
case MetaWidth:
case MetaHeight: {
int value;
ret = fread(&value, sizeof(int), 1, file);
cJSON_AddNumberToObject(document, get_meta_key_text(key), value);
break;
}
case MetaMediaDuration:
case MetaMediaBitrate: {
long value;
ret = fread(&value, sizeof(long), 1, file);
cJSON_AddNumberToObject(document, get_meta_key_text(key), (double) value);
break;
}
case MetaMediaAudioCodec:
case MetaMediaVideoCodec:
case MetaContent:
case MetaArtist:
case MetaAlbum:
case MetaAlbumArtist:
case MetaGenre:
case MetaFontName:
case MetaParent:
case MetaExifMake:
case MetaExifSoftware:
case MetaExifExposureTime:
case MetaExifFNumber:
case MetaExifFocalLength:
case MetaExifUserComment:
case MetaExifIsoSpeedRatings:
case MetaExifDateTime:
case MetaExifModel:
case MetaAuthor:
case MetaModifiedBy:
case MetaThumbnail:
case MetaExifGpsLongitudeDMS:
case MetaExifGpsLongitudeDec:
case MetaExifGpsLongitudeRef:
case MetaExifGpsLatitudeDMS:
case MetaExifGpsLatitudeDec:
case MetaExifGpsLatitudeRef:
case MetaTitle: {
buf.cur = 0;
while ((c = getc(file)) != 0) {
if (SHOULD_KEEP_CHAR(c) || c == ' ') {
dyn_buffer_write_char(&buf, (char) c);
}
}
dyn_buffer_write_char(&buf, '\0');
cJSON_AddStringToObject(document, get_meta_key_text(key), buf.buf);
break;
}
default:
LOG_FATALF("serialize.c", "Invalid meta key (corrupt index): %x", key)
cJSON *child;
for (child = meta_obj->child; child != NULL; child = child->next) {
char meta_key[4096];
strcpy(meta_key, child->string);
cJSON_DeleteItemFromObject(document, meta_key);
cJSON_AddItemReferenceToObject(document, meta_key, child);
}
fread(&key, sizeof(uint16_t), 1, file);
}
cJSON *meta_obj = NULL;
if (IndexCtx.meta != NULL) {
const char *meta_string = g_hash_table_lookup(IndexCtx.meta, path_md5_str);
if (meta_string != NULL) {
meta_obj = cJSON_Parse(meta_string);
cJSON *child;
for (child = meta_obj->child; child != NULL; child = child->next) {
char meta_key[4096];
strcpy(meta_key, child->string);
cJSON_DeleteItemFromObject(document, meta_key);
cJSON_AddItemReferenceToObject(document, meta_key, child);
}
}
}
if (IndexCtx.tags != NULL) {
const char *tags_string = g_hash_table_lookup(IndexCtx.tags, path_md5_str);
if (tags_string != NULL) {
cJSON *tags_arr = cJSON_Parse(tags_string);
cJSON_DeleteItemFromObject(document, "tag");
cJSON_AddItemToObject(document, "tag", tags_arr);
}
}
func(document, path_md5_str);
cJSON_Delete(document);
if (meta_obj) {
cJSON_Delete(meta_obj);
}
}
dyn_buffer_destroy(&buf);
fclose(file);
// Load tags from tags DB
if (IndexCtx.tags != NULL) {
const char *tags_string = g_hash_table_lookup(IndexCtx.tags, path_md5_str);
if (tags_string != NULL) {
cJSON *tags_arr = cJSON_Parse(tags_string);
cJSON_DeleteItemFromObject(document, "tag");
cJSON_AddItemToObject(document, "tag", tags_arr);
}
}
func(document, path_md5_str);
cJSON_DeleteItemFromObject(document, "_id");
cJSON_Delete(document);
if (meta_obj) {
cJSON_Delete(meta_obj);
}
}
const char *json_type_copy_fields[] = {
"mime", "name", "path", "extension", "index", "size", "mtime", "parent",
void read_index_ndjson(const char *path, const char *index_id, index_func func) {
dyn_buffer_t buf = dyn_buffer_create();
// Meta
"title", "content", "width", "height", "duration", "audioc", "videoc",
"bitrate", "artist", "album", "album_artist", "genre", "title", "font_name",
// Initialize zstd things
FILE *file = fopen(path, "rb");
// Special
"tag", "_url"
};
size_t const buf_in_size = ZSTD_DStreamInSize();
void *const buf_in = malloc(buf_in_size);
const char *json_type_array_fields[] = {
"_keyword", "_text"
};
size_t const buf_out_size = ZSTD_DStreamOutSize();
void *const buf_out = malloc(buf_out_size);
void read_index_json(const char *path, UNUSED(const char *index_id), index_func func) {
ZSTD_DCtx *const dctx = ZSTD_createDCtx();
FILE *file = fopen(path, "r");
while (TRUE) {
char *line = NULL;
size_t len;
size_t read = getline(&line, &len, file);
if (read < 0) {
if (line) {
free(line);
}
break;
}
size_t read;
size_t last_ret = 0;
while ((read = fread(buf_in, 1, buf_in_size, file))) {
ZSTD_inBuffer input = {buf_in, read, 0};
cJSON *input = cJSON_Parse(line);
if (input == NULL) {
LOG_FATALF("serialize.c", "Could not parse JSON line: \n%s", line)
}
if (line) {
free(line);
}
while (input.pos < input.size) {
ZSTD_outBuffer output = {buf_out, buf_out_size, 0};
cJSON *document = cJSON_CreateObject();
const char *id_str = cJSON_GetObjectItem(input, "_id")->valuestring;
size_t const ret = ZSTD_decompressStream(dctx, &output, &input);
for (int i = 0; i < (sizeof(json_type_copy_fields) / sizeof(json_type_copy_fields[0])); i++) {
cJSON *value = cJSON_GetObjectItem(input, json_type_copy_fields[i]);
if (value != NULL) {
cJSON_AddItemReferenceToObject(document, json_type_copy_fields[i], value);
}
}
for (int i = 0; i < output.pos; i++) {
char c = ((char *) output.dst)[i];
for (int i = 0; i < (sizeof(json_type_array_fields) / sizeof(json_type_array_fields[0])); i++) {
cJSON *arr = cJSON_GetObjectItem(input, json_type_array_fields[i]);
if (arr != NULL) {
cJSON *obj;
cJSON_ArrayForEach(obj, arr) {
char key[1024];
cJSON *k = cJSON_GetObjectItem(obj, "k");
cJSON *v = cJSON_GetObjectItem(obj, "v");
if (k == NULL || v == NULL || !cJSON_IsString(k) || !cJSON_IsString(v)) {
char *str = cJSON_Print(obj);
LOG_FATALF("serialize.c", "Invalid %s member: must contain .k and .v string fields: \n%s",
json_type_array_fields[i], str)
}
snprintf(key, sizeof(key), "%s.%s", json_type_array_fields[i], k->valuestring);
cJSON_AddStringToObject(document, key, v->valuestring);
if (c == '\n') {
dyn_buffer_write_char(&buf, '\0');
read_index_bin_handle_line(buf.buf, index_id, func);
buf.cur = 0;
} else {
dyn_buffer_write_char(&buf, c);
}
}
last_ret = ret;
}
func(document, id_str);
cJSON_Delete(document);
cJSON_Delete(input);
}
if (last_ret != 0) {
/* The last return value from ZSTD_decompressStream did not end on a
* frame, but we reached the end of the file! We assume this is an
* error, and the input was truncated.
*/
LOG_FATALF("serialize.c", "EOF before end of stream: %zu", last_ret)
}
ZSTD_freeDCtx(dctx);
free(buf_in);
free(buf_out);
dyn_buffer_destroy(&buf);
fclose(file);
}
void read_index(const char *path, const char index_id[MD5_STR_LENGTH], const char *type, index_func func) {
if (strcmp(type, INDEX_TYPE_BIN) == 0) {
read_index_bin(path, index_id, func);
} else if (strcmp(type, INDEX_TYPE_JSON) == 0) {
read_index_json(path, index_id, func);
if (strcmp(type, INDEX_TYPE_NDJSON) == 0) {
read_index_ndjson(path, index_id, func);
}
}
void incremental_read(GHashTable *table, const char *filepath) {
FILE *file = fopen(filepath, "rb");
line_t line;
static __thread GHashTable *IncrementalReadTable = NULL;
LOG_DEBUGF("serialize.c", "Incremental read %s", filepath)
void json_put_incremental(cJSON *document, UNUSED(const char id_str[MD5_STR_LENGTH])) {
const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring;
const int mtime = cJSON_GetObjectItem(document, "mtime")->valueint;
while (1) {
size_t ret = fread((void *) &line, sizeof(line_t), 1, file);
if (ret != 1 || feof(file)) {
break;
incremental_put_str(IncrementalReadTable, path_md5_str, mtime);
}
void incremental_read(GHashTable *table, const char *filepath, index_descriptor_t *desc) {
IncrementalReadTable = table;
read_index(filepath, desc->id, desc->type, json_put_incremental);
}
static __thread GHashTable *IncrementalCopyTable = NULL;
static __thread store_t *IncrementalCopySourceStore = NULL;
static __thread store_t *IncrementalCopyDestinationStore = NULL;
void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[MD5_STR_LENGTH])) {
const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring;
unsigned char path_md5[MD5_DIGEST_LENGTH];
hex2buf(path_md5_str, MD5_STR_LENGTH - 1, path_md5);
if (cJSON_GetObjectItem(document, "parent") != NULL || incremental_get_str(IncrementalCopyTable, path_md5_str)) {
// Copy index line
cJSON_DeleteItemFromObject(document, "index");
char *json_str = cJSON_PrintUnformatted(document);
const size_t json_str_len = strlen(json_str);
json_str = realloc(json_str, json_str_len + 1);
*(json_str + json_str_len) = '\n';
zstd_write_string(json_str, json_str_len + 1);
free(json_str);
// Copy tn store contents
size_t buf_len;
char *buf = store_read(IncrementalCopySourceStore, (char *) path_md5, sizeof(path_md5), &buf_len);
if (buf_len != 0) {
store_write(IncrementalCopyDestinationStore, (char *) path_md5, sizeof(path_md5), buf, buf_len);
free(buf);
}
incremental_put(table, line.path_md5, line.mtime);
while ((getc(file)) != 0) {}
skip_meta(file);
}
fclose(file);
}
/**
@@ -493,72 +513,14 @@ void incremental_read(GHashTable *table, const char *filepath) {
*/
void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
const char *dst_filepath, GHashTable *copy_table) {
FILE *file = fopen(filepath, "rb");
FILE *dst_file = fopen(dst_filepath, "ab");
line_t line;
LOG_DEBUGF("serialize.c", "Incremental copy %s", filepath)
while (TRUE) {
size_t ret = fread((void *) &line, sizeof(line_t), 1, file);
if (ret != 1 || feof(file)) {
break;
}
// Assume that files with parents still exist.
// One way to "fix" this would be to check if the parent is marked for copy but it would consistently
// delete files with grandparents, which is a side-effect worse than having orphaned files
if (line.has_parent || incremental_get(copy_table, line.path_md5)) {
fwrite(&line, sizeof(line), 1, dst_file);
// Copy filepath
char filepath_buf[PATH_MAX];
char c;
char *ptr = filepath_buf;
while ((c = (char) getc(file))) {
*ptr++ = c;
}
*ptr = '\0';
fwrite(filepath_buf, (ptr - filepath_buf) + 1, 1, dst_file);
// Copy tn store contents
size_t buf_len;
char path_md5[MD5_DIGEST_LENGTH];
MD5((unsigned char *) filepath_buf, (ptr - filepath_buf), (unsigned char *) path_md5);
char *buf = store_read(store, path_md5, sizeof(path_md5), &buf_len);
if (buf_len != 0) {
store_write(dst_store, path_md5, sizeof(path_md5), buf, buf_len);
free(buf);
}
enum metakey key = 0;
while (1) {
fread(&key, sizeof(uint16_t), 1, file);
fwrite(&key, sizeof(uint16_t), 1, dst_file);
if (key == META_NEXT) {
break;
}
if (IS_META_INT(key)) {
int val;
ret = fread(&val, sizeof(val), 1, file);
fwrite(&val, sizeof(val), 1, dst_file);
} else if (IS_META_LONG(key)) {
long val;
ret = fread(&val, sizeof(val), 1, file);
fwrite(&val, sizeof(val), 1, dst_file);
} else {
while ((c = (char) getc(file))) {
fwrite(&c, sizeof(c), 1, dst_file);
}
fwrite("\0", sizeof(c), 1, dst_file);
}
}
} else {
while ((getc(file))) {}
skip_meta(file);
}
if (WriterCtx.out_file == NULL) {
initialize_writer_ctx(dst_filepath);
}
fclose(file);
fclose(dst_file);
IncrementalCopyTable = copy_table;
IncrementalCopySourceStore = store;
IncrementalCopyDestinationStore = dst_store;
read_index(filepath, "", INDEX_TYPE_NDJSON, incremental_copy_handle_doc);
}

Some files were not shown because too many files have changed in this diff Show More