mirror of
https://github.com/simon987/sist2.git
synced 2025-09-16 02:56:53 +00:00
Compare commits
No commits in common. "master" and "3.4.0" have entirely different histories.
88
.drone.yml
88
.drone.yml
@ -7,36 +7,11 @@ platform:
|
||||
arch: amd64
|
||||
|
||||
steps:
|
||||
- name: submodules
|
||||
image: alpine/git
|
||||
commands:
|
||||
- git submodule update --init --recursive
|
||||
- name: docker
|
||||
image: plugins/docker
|
||||
depends_on:
|
||||
- submodules
|
||||
settings:
|
||||
username:
|
||||
from_secret: DOCKER_USER
|
||||
password:
|
||||
from_secret: DOCKER_PASSWORD
|
||||
repo: sist2app/sist2
|
||||
context: ./
|
||||
dockerfile: ./Dockerfile
|
||||
auto_tag: true
|
||||
auto_tag_suffix: x64-linux
|
||||
when:
|
||||
event:
|
||||
- tag
|
||||
- name: build
|
||||
image: sist2app/sist2-build
|
||||
depends_on:
|
||||
- submodules
|
||||
image: simon987/sist2-build
|
||||
commands:
|
||||
- ./scripts/build.sh
|
||||
- name: scp files
|
||||
depends_on:
|
||||
- build
|
||||
image: appleboy/drone-scp
|
||||
settings:
|
||||
host:
|
||||
@ -47,11 +22,26 @@ steps:
|
||||
from_secret: SSH_USER
|
||||
key:
|
||||
from_secret: SSH_KEY
|
||||
target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
|
||||
target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
|
||||
source:
|
||||
- ./VERSION
|
||||
- ./sist2-x64-linux
|
||||
- ./sist2-x64-linux-debug
|
||||
- name: docker
|
||||
image: plugins/docker
|
||||
settings:
|
||||
username:
|
||||
from_secret: DOCKER_USER
|
||||
password:
|
||||
from_secret: DOCKER_PASSWORD
|
||||
repo: simon987/sist2
|
||||
context: ./
|
||||
dockerfile: ./Dockerfile
|
||||
auto_tag: true
|
||||
auto_tag_suffix: x64-linux
|
||||
when:
|
||||
event:
|
||||
- tag
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
@ -62,36 +52,11 @@ platform:
|
||||
arch: arm64
|
||||
|
||||
steps:
|
||||
- name: submodules
|
||||
image: alpine/git
|
||||
commands:
|
||||
- git submodule update --init --recursive
|
||||
- name: docker
|
||||
image: plugins/docker
|
||||
depends_on:
|
||||
- submodules
|
||||
settings:
|
||||
username:
|
||||
from_secret: DOCKER_USER
|
||||
password:
|
||||
from_secret: DOCKER_PASSWORD
|
||||
repo: sist2app/sist2
|
||||
context: ./
|
||||
dockerfile: ./Dockerfile.arm64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: arm64-linux
|
||||
when:
|
||||
event:
|
||||
- tag
|
||||
- name: build
|
||||
image: sist2app/sist2-build-arm64
|
||||
depends_on:
|
||||
- submodules
|
||||
image: simon987/sist2-build-arm64
|
||||
commands:
|
||||
- ./scripts/build_arm64.sh
|
||||
- name: scp files
|
||||
depends_on:
|
||||
- build
|
||||
image: appleboy/drone-scp
|
||||
settings:
|
||||
host:
|
||||
@ -102,7 +67,22 @@ steps:
|
||||
from_secret: SSH_USER
|
||||
key:
|
||||
from_secret: SSH_KEY
|
||||
target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
|
||||
target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
|
||||
source:
|
||||
- ./sist2-arm64-linux
|
||||
- ./sist2-arm64-linux-debug
|
||||
- name: docker
|
||||
image: plugins/docker
|
||||
settings:
|
||||
username:
|
||||
from_secret: DOCKER_USER
|
||||
password:
|
||||
from_secret: DOCKER_PASSWORD
|
||||
repo: simon987/sist2
|
||||
context: ./
|
||||
dockerfile: ./Dockerfile.arm64
|
||||
auto_tag: true
|
||||
auto_tag_suffix: arm64-linux
|
||||
when:
|
||||
event:
|
||||
- tag
|
||||
|
||||
@ -62,9 +62,7 @@ add_executable(
|
||||
src/database/database_schema.c
|
||||
src/database/database_fts.c
|
||||
src/web/web_fts.c
|
||||
src/database/database_embeddings.c
|
||||
src/ignorelist.c
|
||||
src/ignorelist.h)
|
||||
src/database/database_embeddings.c)
|
||||
set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
|
||||
|
||||
target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
|
||||
@ -78,7 +76,6 @@ find_package(CURL CONFIG REQUIRED)
|
||||
find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
|
||||
find_package(unofficial-sqlite3 CONFIG REQUIRED)
|
||||
find_package(OpenBLAS CONFIG REQUIRED)
|
||||
find_package(libgit2 CONFIG REQUIRED)
|
||||
|
||||
|
||||
target_include_directories(
|
||||
@ -92,7 +89,7 @@ target_include_directories(
|
||||
target_compile_options(
|
||||
sist2
|
||||
PRIVATE
|
||||
# -fPIC
|
||||
-fPIC
|
||||
)
|
||||
|
||||
if (SIST_DEBUG)
|
||||
@ -150,9 +147,7 @@ add_dependencies(
|
||||
target_link_libraries(
|
||||
sist2
|
||||
|
||||
# m
|
||||
z
|
||||
libgit2::libgit2package
|
||||
argparse
|
||||
unofficial::mongoose::mongoose
|
||||
CURL::libcurl
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
FROM sist2app/sist2-build as build
|
||||
FROM simon987/sist2-build as build
|
||||
MAINTAINER simon987 <me@simon987.net>
|
||||
|
||||
WORKDIR /build/
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
FROM sist2app/sist2-build-arm64 as build
|
||||
FROM simon987/sist2-build-arm64 as build
|
||||
MAINTAINER simon987 <me@simon987.net>
|
||||
|
||||
WORKDIR /build/
|
||||
|
||||
|
||||
62
README.md
62
README.md
@ -1,5 +1,5 @@
|
||||
|
||||
[](https://www.codefactor.io/repository/github/sist2app/sist2)
|
||||
|
||||
[](https://www.codefactor.io/repository/github/simon987/sist2)
|
||||
[](https://files.simon987.net/.gate/sist2/simon987_sist2/)
|
||||
|
||||
**Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
|
||||
@ -38,32 +38,26 @@ sist2 (Simple incremental search tool)
|
||||
### Using Docker Compose *(Windows/Linux/Mac)*
|
||||
|
||||
```yaml
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
elasticsearch:
|
||||
image: elasticsearch:7.17.9
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
# This directory must have 1000:1000 permissions (or update PUID & PGID below)
|
||||
- /data/sist2-es-data/:/usr/share/elasticsearch/data
|
||||
environment:
|
||||
- "discovery.type=single-node"
|
||||
- "ES_JAVA_OPTS=-Xms2g -Xmx2g"
|
||||
- "PUID=1000"
|
||||
- "PGID=1000"
|
||||
sist2-admin:
|
||||
image: sist2app/sist2:x64-linux
|
||||
image: simon987/sist2:3.3.4-x64-linux
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /data/sist2-admin-data/:/sist2-admin/
|
||||
- /<path to index>/:/host
|
||||
- ./sist2-admin-data/:/sist2-admin/
|
||||
- /:/host
|
||||
ports:
|
||||
- 4090:4090
|
||||
# NOTE: Don't expose this port publicly!
|
||||
- 8080:8080
|
||||
- 4090:4090 # sist2
|
||||
- 8080:8080 # sist2-admin
|
||||
working_dir: /root/sist2-admin/
|
||||
entrypoint: python3
|
||||
command:
|
||||
- /root/sist2-admin/sist2_admin/app.py
|
||||
entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
|
||||
```
|
||||
|
||||
Navigate to http://localhost:8080/ to configure sist2-admin.
|
||||
@ -79,7 +73,7 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
|
||||
```
|
||||
* **SQLite**: No installation required
|
||||
|
||||
2. Download the [latest sist2 release](https://github.com/sist2app/sist2/releases).
|
||||
2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
|
||||
Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
|
||||
3. See [usage guide](docs/USAGE.md) for command line usage.
|
||||
|
||||
@ -88,30 +82,28 @@ Example usage:
|
||||
1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
|
||||
2. Prepare search index:
|
||||
* **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
|
||||
* **SQLite**: `sist2 sqlite-index --search-index ./search.sist2 ./documents.sist2`
|
||||
3. Start web interface:
|
||||
* **Elasticsearch**: `sist2 web ./documents.sist2`
|
||||
* **SQLite**: `sist2 web --search-index ./search.sist2 ./documents.sist2`
|
||||
* **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
|
||||
3. Start web interface: `sist2 web ./documents.sist2`
|
||||
|
||||
## Format support
|
||||
|
||||
| File type | Library | Content | Thumbnail | Metadata |
|
||||
|:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
|
||||
| cbz,cbr | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | - | yes | - |
|
||||
| cbz,cbr | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | - | yes | - |
|
||||
| `audio/*` | ffmpeg | - | yes | ID3 tags |
|
||||
| `video/*` | ffmpeg | - | yes | title, comment, artist |
|
||||
| `image/*` | ffmpeg | ocr | yes | [Common EXIF tags](https://github.com/sist2app/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
|
||||
| `image/*` | ffmpeg | ocr | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
|
||||
| raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw | no | yes | Common EXIF tags, GPS tags |
|
||||
| ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
|
||||
| `text/plain` | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes | no | - |
|
||||
| html, xml | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes | no | - |
|
||||
| `text/plain` | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | no | - |
|
||||
| html, xml | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | no | - |
|
||||
| tar, zip, rar, 7z, ar ... | Libarchive | yes\* | - | no |
|
||||
| docx, xlsx, pptx | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes | if embedded | creator, modified_by, title |
|
||||
| docx, xlsx, pptx | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | if embedded | creator, modified_by, title |
|
||||
| doc (MS Word 97-2003) | antiword | yes | no | author, title |
|
||||
| mobi, azw, azw3 | libmobi | yes | yes | author, title |
|
||||
| wpd (WordPerfect) | libwpd | yes | no | *planned* |
|
||||
| json, jsonl, ndjson | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes | - | - |
|
||||
| json, jsonl, ndjson | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes | - | - |
|
||||
|
||||
\* *See [Archive files](#archive-files)*
|
||||
|
||||
@ -135,7 +127,7 @@ You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with
|
||||
Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
|
||||
directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
|
||||
|
||||
The `sist2app/sist2` image comes with common languages
|
||||
The `simon987/sist2` image comes with common languages
|
||||
(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
|
||||
|
||||
You can use the `+` separator to specify multiple languages. The language
|
||||
@ -175,13 +167,13 @@ sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported re
|
||||
to enable it.
|
||||
|
||||
The text processing is done in your browser, no data is sent to any third-party services.
|
||||
See [sist2app/sist2-ner-models](https://github.com/sist2app/sist2-ner-models) for more details.
|
||||
See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
|
||||
|
||||
#### List of available repositories:
|
||||
|
||||
| URL | Maintainer | Purpose |
|
||||
|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
|
||||
| [sist2app/sist2-ner-models](https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json) | [sist2app](https://github.com/sist2app) | General |
|
||||
| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
|
||||
|
||||
<details>
|
||||
<summary>Screenshot</summary>
|
||||
@ -197,7 +189,7 @@ You can compile **sist2** by yourself if you don't want to use the pre-compiled
|
||||
### Using docker
|
||||
|
||||
```bash
|
||||
git clone --recursive https://github.com/sist2app/sist2/
|
||||
git clone --recursive https://github.com/simon987/sist2/
|
||||
cd sist2
|
||||
docker build . -t my-sist2-image
|
||||
# Copy sist2 executable from docker image
|
||||
@ -212,18 +204,18 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
|
||||
apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
|
||||
```
|
||||
|
||||
2. Install vcpkg using my fork: https://github.com/sist2app/vcpkg
|
||||
2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
|
||||
3. Install vcpkg dependencies
|
||||
|
||||
```bash
|
||||
vcpkg install openblas curl[core,openssl] sqlite3[core,fts5,json1] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib] libgit2[core,pcre]
|
||||
vcpkg install openblas curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
|
||||
```
|
||||
|
||||
4. Build
|
||||
```bash
|
||||
git clone --recursive https://github.com/sist2app/sist2/
|
||||
git clone --recursive https://github.com/simon987/sist2/
|
||||
(cd sist2-vue; npm install; npm run build)
|
||||
(cd sist2-admin/frontend; npm install; npm run build)
|
||||
cmake -DSIST_DEBUG=off -G "Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
|
||||
cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
|
||||
make
|
||||
```
|
||||
|
||||
@ -4,21 +4,16 @@ services:
|
||||
elasticsearch:
|
||||
image: elasticsearch:7.17.9
|
||||
container_name: sist2-es
|
||||
volumes:
|
||||
# This directory must have 1000:1000 permissions (or update PUID & PGID below)
|
||||
- /data/sist2-es-data/:/usr/share/elasticsearch/data
|
||||
environment:
|
||||
- "discovery.type=single-node"
|
||||
- "ES_JAVA_OPTS=-Xms2g -Xmx2g"
|
||||
- "PUID=1000"
|
||||
- "PGID=1000"
|
||||
sist2-admin:
|
||||
build:
|
||||
context: .
|
||||
container_name: sist2-admin
|
||||
volumes:
|
||||
- /data/sist2-admin-data/:/sist2-admin/
|
||||
- /<path to index>/:/host
|
||||
- /mnt/array/sist2-admin-data/:/sist2-admin/
|
||||
- /:/host
|
||||
ports:
|
||||
- 4090:4090
|
||||
# NOTE: Don't export this port publicly!
|
||||
|
||||
@ -108,27 +108,6 @@ sist scan ~/Documents -o ./documents.sist2 --incremental
|
||||
sist scan ~/Documents -o ./documents.sist2 --incremental
|
||||
```
|
||||
|
||||
### Excluding files
|
||||
|
||||
You can use the `--exclude` option to specify exclude patterns. For more complex setups, you can create a
|
||||
`.sist2ignore` file at the root of the scan path (For example, `~/Documents/.sist2ignore` for the example above).
|
||||
|
||||
The syntax for sist2ignore is the same as .gitignore for Git (reference [here](https://git-scm.com/docs/gitignore)).
|
||||
|
||||
Example:
|
||||
|
||||
**.sist2ignore**
|
||||
```gitignore
|
||||
# Ignore all PDF files
|
||||
*.pdf
|
||||
|
||||
# But don't ignore them for the /important_files/ directory
|
||||
!/important_files/*.pdf
|
||||
|
||||
# Ignore all files in _staging/ directories
|
||||
_staging/
|
||||
```
|
||||
|
||||
### Index documents to Elasticsearch search backend
|
||||
|
||||
```bash
|
||||
@ -193,10 +172,6 @@ Using a version >=7.14.0 is recommended to enable the following features:
|
||||
|
||||
- Bug fix for large documents (See #198)
|
||||
|
||||
Using a version >=8.0.0 is recommended to enable the following features:
|
||||
|
||||
- Approximate KNN search for Embeddings search (faster queries).
|
||||
|
||||
When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
|
||||
If you don't care about the features above, you can ignore it or disable it in the configuration page.
|
||||
|
||||
@ -210,7 +185,7 @@ Since v3.2.0, User scripts can be used to generate _embeddings_ (vector of float
|
||||
|
||||
In theory, embeddings can be created for any type of documents (image, text, audio etc.).
|
||||
|
||||
For example, the [clip](https://github.com/sist2app/sist2-script-clip) User Script, generates 512-d embeddings of images
|
||||
For example, the [clip](https://github.com/simon987/sist2-script-clip) User Script, generates 512-d embeddings of images
|
||||
(videos are also supported using the thumbnails generated by sist2). When the user enters a query in the "Embeddings Search"
|
||||
textbox, the query's embedding is generated in their browser, leveraging the ONNX web runtime.
|
||||
|
||||
@ -250,4 +225,4 @@ See [Automatic tagging](#automatic-tagging) for information about tag
|
||||
|
||||
### Automatic tagging
|
||||
|
||||
See [scripting](scripting.md) documentation.
|
||||
See [scripting](scripting.md) documentation.
|
||||
@ -2,6 +2,8 @@
|
||||
|
||||
VCPKG_ROOT="/vcpkg"
|
||||
|
||||
git submodule update --init --recursive
|
||||
|
||||
(
|
||||
cd sist2-vue/
|
||||
npm install
|
||||
|
||||
@ -1,16 +1,8 @@
|
||||
|
||||
MAGIC_PATHS = [
|
||||
"/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
|
||||
"/work/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
|
||||
"/usr/lib/file/magic.mgc"
|
||||
]
|
||||
|
||||
for path in MAGIC_PATHS:
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
break
|
||||
except:
|
||||
continue
|
||||
try:
|
||||
with open("/usr/lib/file/magic.mgc", "rb") as f:
|
||||
data = f.read()
|
||||
except:
|
||||
data = bytes([])
|
||||
|
||||
print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))
|
||||
|
||||
@ -449,5 +449,4 @@ image/x-sigma-x3f, xf3
|
||||
image/x-sony-arw, arw
|
||||
image/x-sony-sr2, sr2
|
||||
image/x-sony-srf, srf
|
||||
image/x-epson-erf, erf
|
||||
image/jp2, jp2
|
||||
image/x-epson-erf, erf
|
||||
|
671
sist2-admin/frontend/package-lock.json
generated
671
sist2-admin/frontend/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -8,7 +8,7 @@
|
||||
"watch": "vue-cli-service build --watch"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.6.0",
|
||||
"axios": "^0.27.2",
|
||||
"bootstrap-vue": "^2.21.2",
|
||||
"core-js": "^3.6.5",
|
||||
"moment": "^2.29.3",
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
<b-container class="pt-4">
|
||||
<b-alert show dismissible variant="info">
|
||||
This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
|
||||
to the <a href="https://github.com/sist2app/sist2/issues/new/choose" target="_blank">issue tracker on
|
||||
to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
|
||||
Github</a>. Thank you!
|
||||
</b-alert>
|
||||
<router-view v-if="$store.state.sist2AdminInfo"/>
|
||||
|
||||
@ -89,12 +89,9 @@ class Sist2AdminApi {
|
||||
|
||||
/**
|
||||
* @param {string} name
|
||||
* @param {bool} full
|
||||
*/
|
||||
runJob(name, full) {
|
||||
return axios.get(`${this.baseUrl}/api/job/${name}/run`, {
|
||||
params: {full}
|
||||
});
|
||||
runJob(name) {
|
||||
return axios.get(`${this.baseUrl}/api/job/${name}/run`);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -95,7 +95,6 @@ export default {
|
||||
methods: {
|
||||
onOcrLangChange() {
|
||||
this.options.ocr_lang = this.selectedOcrLangs.join("+");
|
||||
this.update();
|
||||
},
|
||||
update() {
|
||||
this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
|
||||
|
||||
@ -1,70 +1,59 @@
|
||||
<template>
|
||||
<div>
|
||||
<h4>{{ $t("webOptions.title") }}</h4>
|
||||
<b-card>
|
||||
<label>{{ $t("webOptions.lang") }}</label>
|
||||
<b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
|
||||
@change="update()"></b-form-select>
|
||||
<div>
|
||||
<h4>{{ $t("webOptions.title") }}</h4>
|
||||
<b-card>
|
||||
<label>{{ $t("webOptions.lang") }}</label>
|
||||
<b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
|
||||
@change="update()"></b-form-select>
|
||||
|
||||
<label>{{ $t("webOptions.bind") }}</label>
|
||||
<b-form-input v-model="options.bind" @change="update()"></b-form-input>
|
||||
<label>{{ $t("webOptions.bind") }}</label>
|
||||
<b-form-input v-model="options.bind" @change="update()"></b-form-input>
|
||||
|
||||
<label>{{ $t("webOptions.tagline") }}</label>
|
||||
<b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
|
||||
<label>{{ $t("webOptions.tagline") }}</label>
|
||||
<b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
|
||||
|
||||
<label>{{ $t("webOptions.auth") }}</label>
|
||||
<b-form-input v-model="options.auth" @change="update()"></b-form-input>
|
||||
<label>{{ $t("webOptions.auth") }}</label>
|
||||
<b-form-input v-model="options.auth" @change="update()"></b-form-input>
|
||||
|
||||
<label>{{ $t("webOptions.tagAuth") }}</label>
|
||||
<b-form-input v-model="options.tag_auth" @change="update()" :disabled="Boolean(options.auth)"></b-form-input>
|
||||
<label>{{ $t("webOptions.tagAuth") }}</label>
|
||||
<b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
|
||||
</b-card>
|
||||
|
||||
<b-form-checkbox v-model="options.verbose" @change="update()">
|
||||
{{$t("webOptions.verbose")}}
|
||||
</b-form-checkbox>
|
||||
</b-card>
|
||||
<br>
|
||||
<h4>Auth0 options</h4>
|
||||
<b-card>
|
||||
<label>{{ $t("webOptions.auth0Audience") }}</label>
|
||||
<b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
|
||||
|
||||
<br>
|
||||
<h4>Auth0 options</h4>
|
||||
<b-card>
|
||||
<label>{{ $t("webOptions.auth0Audience") }}</label>
|
||||
<b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
|
||||
<label>{{ $t("webOptions.auth0Domain") }}</label>
|
||||
<b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
|
||||
|
||||
<label>{{ $t("webOptions.auth0Domain") }}</label>
|
||||
<b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
|
||||
<label>{{ $t("webOptions.auth0ClientId") }}</label>
|
||||
<b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
|
||||
|
||||
<label>{{ $t("webOptions.auth0ClientId") }}</label>
|
||||
<b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
|
||||
|
||||
<label>{{ $t("webOptions.auth0PublicKey") }}</label>
|
||||
<b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
|
||||
</b-card>
|
||||
</div>
|
||||
<label>{{ $t("webOptions.auth0PublicKey") }}</label>
|
||||
<b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
|
||||
</b-card>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
|
||||
export default {
|
||||
name: "WebOptions",
|
||||
props: ["options", "frontendName"],
|
||||
data() {
|
||||
return {
|
||||
showEsTestAlert: false,
|
||||
esTestOk: false,
|
||||
esTestMessage: ""
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
update() {
|
||||
|
||||
console.log(this.options)
|
||||
if (this.options.auth && this.options.tag_auth) {
|
||||
// If both are set, remove tagAuth
|
||||
this.options.tag_auth = "";
|
||||
}
|
||||
|
||||
this.$emit("change", this.options);
|
||||
name: "WebOptions",
|
||||
props: ["options", "frontendName"],
|
||||
data() {
|
||||
return {
|
||||
showEsTestAlert: false,
|
||||
esTestOk: false,
|
||||
esTestMessage: "",
|
||||
}
|
||||
},
|
||||
}
|
||||
methods: {
|
||||
update() {
|
||||
this.$emit("change", this.options);
|
||||
},
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
@ -8,7 +8,6 @@ export default {
|
||||
view: "View",
|
||||
delete: "Delete",
|
||||
runNow: "Index now",
|
||||
runNowFull: "Full re-index",
|
||||
create: "Create",
|
||||
cancel: "Cancel",
|
||||
test: "Test",
|
||||
@ -65,9 +64,6 @@ export default {
|
||||
gitRepository: "Git repository URL",
|
||||
extraArgs: "Extra command line arguments",
|
||||
|
||||
couldNotStartFrontend: "Could not start frontend",
|
||||
couldNotStartFrontendBody: "Unable to start the frontend, check server logs for more details.",
|
||||
|
||||
selectJobs: "Available jobs",
|
||||
selectJob: "Select a job",
|
||||
webOptions: {
|
||||
@ -81,7 +77,6 @@ export default {
|
||||
auth0Domain: "Auth0 domain",
|
||||
auth0ClientId: "Auth0 client ID",
|
||||
auth0PublicKey: "Auth0 public key",
|
||||
verbose: "Verbose logs"
|
||||
},
|
||||
backendOptions: {
|
||||
title: "Search backend options",
|
||||
@ -93,9 +88,7 @@ export default {
|
||||
threads: "Number of threads",
|
||||
batchSize: "Index batch size",
|
||||
script: "User script",
|
||||
searchIndex: "Search index file location",
|
||||
esMappings: "Elasticsearch mappings file override",
|
||||
esSettings: "Elasticsearch settings file override"
|
||||
searchIndex: "Search index file location"
|
||||
},
|
||||
scanOptions: {
|
||||
title: "Scanning options",
|
||||
|
||||
@ -1,63 +1,63 @@
|
||||
<template>
|
||||
<b-card>
|
||||
<b-card-title>
|
||||
{{ name }}
|
||||
<small style="vertical-align: top">
|
||||
<b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
|
||||
<b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
|
||||
</small>
|
||||
</b-card-title>
|
||||
<b-card>
|
||||
<b-card-title>
|
||||
{{ name }}
|
||||
<small style="vertical-align: top">
|
||||
<b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
|
||||
<b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
|
||||
</small>
|
||||
</b-card-title>
|
||||
|
||||
<!-- Action buttons-->
|
||||
<div class="mb-3" v-if="!loading">
|
||||
<b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
|
||||
$t("start")
|
||||
}}
|
||||
</b-button>
|
||||
<b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
|
||||
$t("stop")
|
||||
}}
|
||||
</b-button>
|
||||
<b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
|
||||
{{ $t("go") }}
|
||||
</b-button>
|
||||
<b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
|
||||
</div>
|
||||
<!-- Action buttons-->
|
||||
<div class="mb-3" v-if="!loading">
|
||||
<b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
|
||||
$t("start")
|
||||
}}
|
||||
</b-button>
|
||||
<b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
|
||||
$t("stop")
|
||||
}}
|
||||
</b-button>
|
||||
<b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
|
||||
{{ $t("go") }}
|
||||
</b-button>
|
||||
<b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
|
||||
</div>
|
||||
|
||||
<b-progress v-if="loading" striped animated value="100"></b-progress>
|
||||
<b-card-body v-else>
|
||||
<b-progress v-if="loading" striped animated value="100"></b-progress>
|
||||
<b-card-body v-else>
|
||||
|
||||
<h4>{{ $t("backendOptions.title") }}</h4>
|
||||
<b-card>
|
||||
<b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
|
||||
<h4>{{ $t("backendOptions.title") }}</h4>
|
||||
<b-card>
|
||||
<b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
|
||||
|
||||
<SearchBackendSelect :value="frontend.web_options.search_backend"
|
||||
@change="onBackendSelect($event)"></SearchBackendSelect>
|
||||
<SearchBackendSelect :value="frontend.web_options.search_backend"
|
||||
@change="onBackendSelect($event)"></SearchBackendSelect>
|
||||
|
||||
<br>
|
||||
<JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
|
||||
</b-card>
|
||||
<br>
|
||||
<JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
|
||||
</b-card>
|
||||
|
||||
<br/>
|
||||
<br/>
|
||||
|
||||
<WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
|
||||
@change="update()"></WebOptions>
|
||||
<br/>
|
||||
<WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
|
||||
@change="update()"></WebOptions>
|
||||
<br/>
|
||||
|
||||
<h4>{{ $t("frontendOptions.title") }}</h4>
|
||||
<b-card>
|
||||
<b-form-checkbox v-model="frontend.auto_start" @change="update()">
|
||||
{{ $t("autoStart") }}
|
||||
</b-form-checkbox>
|
||||
<h4>{{ $t("frontendOptions.title") }}</h4>
|
||||
<b-card>
|
||||
<b-form-checkbox v-model="frontend.auto_start" @change="update()">
|
||||
{{ $t("autoStart") }}
|
||||
</b-form-checkbox>
|
||||
|
||||
<label>{{ $t("extraQueryArgs") }}</label>
|
||||
<b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
|
||||
<label>{{ $t("extraQueryArgs") }}</label>
|
||||
<b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
|
||||
|
||||
<label>{{ $t("customUrl") }}</label>
|
||||
<b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
|
||||
</b-card>
|
||||
</b-card-body>
|
||||
</b-card>
|
||||
<label>{{ $t("customUrl") }}</label>
|
||||
<b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
|
||||
</b-card>
|
||||
</b-card-body>
|
||||
</b-card>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
@ -68,78 +68,71 @@ import WebOptions from "@/components/WebOptions";
|
||||
import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
|
||||
|
||||
export default {
|
||||
name: 'Frontend',
|
||||
components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
|
||||
data() {
|
||||
return {
|
||||
loading: true,
|
||||
frontend: null,
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
valid() {
|
||||
return !this.loading && this.frontend.jobs.length > 0;
|
||||
name: 'Frontend',
|
||||
components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
|
||||
data() {
|
||||
return {
|
||||
loading: true,
|
||||
frontend: null,
|
||||
}
|
||||
},
|
||||
frontendUrl() {
|
||||
if (this.frontend.custom_url) {
|
||||
return this.frontend.custom_url + this.args;
|
||||
}
|
||||
computed: {
|
||||
valid() {
|
||||
return !this.loading && this.frontend.jobs.length > 0;
|
||||
},
|
||||
frontendUrl() {
|
||||
if (this.frontend.custom_url) {
|
||||
return this.frontend.custom_url + this.args;
|
||||
}
|
||||
|
||||
if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
|
||||
return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
|
||||
}
|
||||
if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
|
||||
return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
|
||||
}
|
||||
|
||||
return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
|
||||
return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
|
||||
},
|
||||
name() {
|
||||
return this.$route.params.name;
|
||||
},
|
||||
port() {
|
||||
return this.frontend.web_options.bind.split(":")[1]
|
||||
},
|
||||
args() {
|
||||
const args = this.frontend.extra_query_args;
|
||||
if (args !== "") {
|
||||
return "#" + (args.startsWith("?") ? (args) : ("?" + args));
|
||||
}
|
||||
return "";
|
||||
}
|
||||
},
|
||||
name() {
|
||||
return this.$route.params.name;
|
||||
},
|
||||
port() {
|
||||
return this.frontend.web_options.bind.split(":")[1]
|
||||
},
|
||||
args() {
|
||||
const args = this.frontend.extra_query_args;
|
||||
if (args !== "") {
|
||||
return "#" + (args.startsWith("?") ? (args) : ("?" + args));
|
||||
}
|
||||
return "";
|
||||
}
|
||||
},
|
||||
mounted() {
|
||||
Sist2AdminApi.getFrontend(this.name).then(resp => {
|
||||
this.frontend = resp.data;
|
||||
this.loading = false;
|
||||
});
|
||||
},
|
||||
methods: {
|
||||
start() {
|
||||
Sist2AdminApi.startFrontend(this.name).then(() => {
|
||||
this.frontend.running = true;
|
||||
}).catch(() => {
|
||||
this.$bvToast.toast(this.$t("couldNotStartFrontendBody"), {
|
||||
title: this.$t("couldNotStartFrontend"),
|
||||
variant: "danger",
|
||||
toaster: "b-toaster-bottom-right"
|
||||
mounted() {
|
||||
Sist2AdminApi.getFrontend(this.name).then(resp => {
|
||||
this.frontend = resp.data;
|
||||
this.loading = false;
|
||||
});
|
||||
});
|
||||
},
|
||||
stop() {
|
||||
this.frontend.running = false;
|
||||
Sist2AdminApi.stopFrontend(this.name)
|
||||
},
|
||||
deleteFrontend() {
|
||||
Sist2AdminApi.deleteFrontend(this.name).then(() => {
|
||||
this.$router.push("/");
|
||||
});
|
||||
},
|
||||
update() {
|
||||
Sist2AdminApi.updateFrontend(this.name, this.frontend);
|
||||
},
|
||||
onBackendSelect(backend) {
|
||||
this.frontend.web_options.search_backend = backend;
|
||||
this.frontend.jobs = [];
|
||||
this.update();
|
||||
methods: {
|
||||
start() {
|
||||
this.frontend.running = true;
|
||||
Sist2AdminApi.startFrontend(this.name)
|
||||
},
|
||||
stop() {
|
||||
this.frontend.running = false;
|
||||
Sist2AdminApi.stopFrontend(this.name)
|
||||
},
|
||||
deleteFrontend() {
|
||||
Sist2AdminApi.deleteFrontend(this.name).then(() => {
|
||||
this.$router.push("/");
|
||||
});
|
||||
},
|
||||
update() {
|
||||
Sist2AdminApi.updateFrontend(this.name, this.frontend);
|
||||
},
|
||||
onBackendSelect(backend) {
|
||||
this.frontend.web_options.search_backend = backend;
|
||||
this.frontend.jobs = [];
|
||||
this.update();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@ -6,19 +6,7 @@
|
||||
</b-card-title>
|
||||
|
||||
<div class="mb-3">
|
||||
|
||||
<b-dropdown
|
||||
split
|
||||
split-variant="primary"
|
||||
variant="primary"
|
||||
:text="$t('runNow')"
|
||||
class="mr-1"
|
||||
:disabled="!valid"
|
||||
@click="runJob()"
|
||||
>
|
||||
<b-dropdown-item href="#" @click="runJob(true)">{{ $t("runNowFull") }}</b-dropdown-item>
|
||||
</b-dropdown>
|
||||
|
||||
<b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
|
||||
<b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
|
||||
</div>
|
||||
|
||||
@ -81,7 +69,6 @@ export default {
|
||||
return {
|
||||
loading: true,
|
||||
job: null,
|
||||
console: console
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
@ -91,8 +78,8 @@ export default {
|
||||
update() {
|
||||
Sist2AdminApi.updateJob(this.getName(), this.job);
|
||||
},
|
||||
runJob(full = false) {
|
||||
Sist2AdminApi.runJob(this.getName(), full).then(() => {
|
||||
runJob() {
|
||||
Sist2AdminApi.runJob(this.getName()).then(() => {
|
||||
this.$bvToast.toast(this.$t("runJobConfirmation"), {
|
||||
title: this.$t("runJobConfirmationTitle"),
|
||||
variant: "success",
|
||||
|
||||
@ -44,12 +44,6 @@
|
||||
|
||||
<label>{{ $t("backendOptions.batchSize") }}</label>
|
||||
<b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
|
||||
|
||||
<label>{{ $t("backendOptions.esMappings") }}</label>
|
||||
<b-form-textarea v-model="backend.es_mappings" rows="4" @change="update()"></b-form-textarea>
|
||||
|
||||
<label>{{ $t("backendOptions.esSettings") }}</label>
|
||||
<b-form-textarea v-model="backend.es_settings" rows="4" @change="update()"></b-form-textarea>
|
||||
</template>
|
||||
<template v-else>
|
||||
<label>{{ $t("backendOptions.searchIndex") }}</label>
|
||||
|
||||
@ -170,6 +170,6 @@ span.ADMIN {
|
||||
margin: 3px;
|
||||
white-space: pre;
|
||||
color: #000;
|
||||
overflow-y: hidden;
|
||||
overflow: hidden;
|
||||
}
|
||||
</style>
|
||||
</style>
|
||||
File diff suppressed because it is too large
Load Diff
@ -4,4 +4,4 @@ uvicorn
|
||||
websockets
|
||||
pycron
|
||||
GitPython
|
||||
git+https://github.com/sist2app/sist2-python.git@2.1
|
||||
git+https://github.com/simon987/sist2-python.git
|
||||
@ -2,7 +2,6 @@ import asyncio
|
||||
import os
|
||||
import signal
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
@ -26,7 +25,6 @@ from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEM
|
||||
get_log_files_to_remove, delete_log_file, create_default_search_backends
|
||||
from web import Sist2Frontend
|
||||
from script import UserScript, SCRIPT_TEMPLATES
|
||||
from util import tail_sync, pid_is_running
|
||||
|
||||
sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
|
||||
db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
|
||||
@ -171,14 +169,11 @@ def _run_job(job: Sist2Job):
|
||||
|
||||
|
||||
@app.get("/api/job/{name:str}/run")
|
||||
async def run_job(name: str, full: bool = False):
|
||||
job: Sist2Job = db["jobs"][name]
|
||||
async def run_job(name: str):
|
||||
job = db["jobs"][name]
|
||||
if not job:
|
||||
raise HTTPException(status_code=404)
|
||||
|
||||
if full:
|
||||
job.do_full_scan = True
|
||||
|
||||
_run_job(job)
|
||||
|
||||
return "ok"
|
||||
@ -326,18 +321,7 @@ def start_frontend_(frontend: Sist2Frontend):
|
||||
logger.debug(f"Fetched search backend options for {backend_name}")
|
||||
|
||||
pid = sist2.web(frontend.web_options, search_backend, frontend.name)
|
||||
|
||||
sleep(0.2)
|
||||
if not pid_is_running(pid):
|
||||
frontend_log = frontend.get_log_path(LOG_FOLDER)
|
||||
logger.error(f"Frontend exited too quickly, check {frontend_log} for more details:")
|
||||
for line in tail_sync(frontend.get_log_path(LOG_FOLDER), 3):
|
||||
logger.error(line.strip())
|
||||
|
||||
return False
|
||||
|
||||
RUNNING_FRONTENDS[frontend.name] = pid
|
||||
return True
|
||||
|
||||
|
||||
@app.post("/api/frontend/{name:str}/start")
|
||||
@ -346,12 +330,7 @@ async def start_frontend(name: str):
|
||||
if not frontend:
|
||||
raise HTTPException(status_code=404)
|
||||
|
||||
ok = start_frontend_(frontend)
|
||||
|
||||
if not ok:
|
||||
raise HTTPException(status_code=500)
|
||||
|
||||
return "ok"
|
||||
start_frontend_(frontend)
|
||||
|
||||
|
||||
@app.post("/api/frontend/{name:str}/stop")
|
||||
|
||||
@ -204,7 +204,7 @@ class Sist2IndexTask(Sist2Task):
|
||||
self.job.previous_index_path = self.job.index_path
|
||||
db["jobs"][self.job.name] = self.job
|
||||
|
||||
self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}, {ok=}"}))
|
||||
self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
|
||||
|
||||
logger.info(f"Completed {self.display_name} ({return_code=})")
|
||||
|
||||
@ -220,7 +220,7 @@ class Sist2IndexTask(Sist2Task):
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
os.waitpid(pid, 0)
|
||||
os.wait()
|
||||
except ChildProcessError:
|
||||
pass
|
||||
|
||||
|
||||
@ -96,7 +96,7 @@ SCRIPT_TEMPLATES = {
|
||||
"CLIP - Generate embeddings to predict the most relevant image based on the text prompt": lambda name: UserScript(
|
||||
name=name,
|
||||
type=ScriptType.GIT,
|
||||
git_repository="https://github.com/sist2app/sist2-script-clip",
|
||||
git_repository="https://github.com/simon987/sist2-script-clip",
|
||||
extra_args="--num-tags=1 --tags-file=general.txt --color=#dcd7ff"
|
||||
),
|
||||
"Whisper - Speech to text with OpenAI Whisper": lambda name: UserScript(
|
||||
|
||||
@ -2,15 +2,14 @@ import datetime
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from io import TextIOWrapper
|
||||
from logging import FileHandler, StreamHandler
|
||||
from logging import FileHandler
|
||||
from subprocess import Popen, PIPE
|
||||
from tempfile import NamedTemporaryFile
|
||||
from threading import Thread
|
||||
from typing import List, Optional
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
@ -40,8 +39,6 @@ class Sist2SearchBackend(BaseModel):
|
||||
|
||||
es_url: str = "http://elasticsearch:9200"
|
||||
es_insecure_ssl: bool = False
|
||||
es_mappings: Optional[str] = None
|
||||
es_settings: Optional[str] = None
|
||||
es_index: str = "sist2"
|
||||
threads: int = 1
|
||||
batch_size: int = 70
|
||||
@ -59,8 +56,6 @@ class IndexOptions(BaseModel):
|
||||
path: str = None
|
||||
incremental_index: bool = True
|
||||
search_backend: str = None
|
||||
es_mappings_file: Optional[str] = None
|
||||
es_settings_file: Optional[str] = None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@ -79,12 +74,6 @@ class IndexOptions(BaseModel):
|
||||
|
||||
if search_backend.es_insecure_ssl:
|
||||
args.append(f"--es-insecure-ssl")
|
||||
|
||||
if self.es_mappings_file:
|
||||
args.append(f"--mappings-file={self.es_mappings_file}")
|
||||
if self.es_settings_file:
|
||||
args.append(f"--settings-file={self.es_settings_file}")
|
||||
|
||||
if self.incremental_index:
|
||||
args.append(f"--incremental-index")
|
||||
|
||||
@ -211,7 +200,6 @@ class WebOptions(BaseModel):
|
||||
auth0_client_id: str = None
|
||||
auth0_public_key: str = None
|
||||
auth0_public_key_file: str = None
|
||||
verbose: bool = False
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@ -243,8 +231,6 @@ class WebOptions(BaseModel):
|
||||
args.append(f"--tag-auth={self.tag_auth}")
|
||||
if self.dev:
|
||||
args.append(f"--dev")
|
||||
if self.verbose:
|
||||
args.append(f"--very-verbose")
|
||||
|
||||
args.extend(self.indices)
|
||||
|
||||
@ -259,20 +245,6 @@ class Sist2:
|
||||
|
||||
def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb):
|
||||
|
||||
if search_backend.es_mappings:
|
||||
with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
|
||||
f.write(search_backend.es_mappings)
|
||||
options.es_mappings_file = f.name
|
||||
else:
|
||||
options.es_mappings_file = None
|
||||
|
||||
if search_backend.es_settings:
|
||||
with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
|
||||
f.write(search_backend.es_settings)
|
||||
options.es_settings_file = f.name
|
||||
else:
|
||||
options.es_settings_file = None
|
||||
|
||||
args = [
|
||||
self.bin_path,
|
||||
*options.args(search_backend),
|
||||
@ -285,7 +257,7 @@ class Sist2:
|
||||
|
||||
set_pid_cb(proc.pid)
|
||||
|
||||
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
|
||||
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
|
||||
t_stderr.start()
|
||||
|
||||
self._consume_logs_stdout(logs_cb, proc)
|
||||
@ -312,7 +284,7 @@ class Sist2:
|
||||
|
||||
set_pid_cb(proc.pid)
|
||||
|
||||
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
|
||||
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
|
||||
t_stderr.start()
|
||||
|
||||
self._consume_logs_stdout(logs_cb, proc)
|
||||
@ -322,7 +294,7 @@ class Sist2:
|
||||
return proc.returncode
|
||||
|
||||
@staticmethod
|
||||
def _consume_logs_stderr(logs_cb, exit_cb, proc):
|
||||
def _consume_logs_stderr(logs_cb, proc):
|
||||
pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8", errors="ignore")
|
||||
try:
|
||||
for line in pipe_wrapper:
|
||||
@ -330,9 +302,7 @@ class Sist2:
|
||||
continue
|
||||
logs_cb({"stderr": line})
|
||||
finally:
|
||||
return_code = proc.wait()
|
||||
if exit_cb:
|
||||
exit_cb(return_code)
|
||||
proc.wait()
|
||||
pipe_wrapper.close()
|
||||
|
||||
@staticmethod
|
||||
@ -366,19 +336,15 @@ class Sist2:
|
||||
|
||||
web_logger = logging.Logger(name=f"sist2-frontend-{name}")
|
||||
web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
|
||||
web_logger.addHandler(StreamHandler())
|
||||
|
||||
def logs_cb(message):
|
||||
web_logger.info(json.dumps(message))
|
||||
|
||||
def exit_cb(return_code):
|
||||
logger.info(f"Web frontend exited with return code {return_code}")
|
||||
|
||||
logger.info(f"Starting frontend {' '.join(args)}")
|
||||
|
||||
proc = Popen(args, stdout=PIPE, stderr=PIPE)
|
||||
|
||||
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, exit_cb, proc))
|
||||
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
|
||||
t_stderr.start()
|
||||
|
||||
t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))
|
||||
|
||||
@ -1,41 +0,0 @@
|
||||
from glob import glob
|
||||
import os
|
||||
from config import DATA_FOLDER
|
||||
|
||||
|
||||
def get_old_index_files(name):
|
||||
files = glob(os.path.join(DATA_FOLDER, f"scan-{name.replace('/', '_')}-*.sist2"))
|
||||
files = list(sorted(files, key=lambda f: os.stat(f).st_mtime))
|
||||
files = files[-1:]
|
||||
|
||||
return files
|
||||
|
||||
|
||||
def tail_sync(filename, lines=1, _buffer=4098):
|
||||
with open(filename) as f:
|
||||
lines_found = []
|
||||
|
||||
block_counter = -1
|
||||
|
||||
while len(lines_found) < lines:
|
||||
try:
|
||||
f.seek(block_counter * _buffer, os.SEEK_END)
|
||||
except IOError:
|
||||
f.seek(0)
|
||||
lines_found = f.readlines()
|
||||
break
|
||||
|
||||
lines_found = f.readlines()
|
||||
|
||||
block_counter -= 1
|
||||
|
||||
return lines_found[-lines:]
|
||||
|
||||
|
||||
def pid_is_running(pid):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
return True
|
||||
Binary file not shown.
14732
sist2-vue/package-lock.json
generated
14732
sist2-vue/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -9,7 +9,7 @@
|
||||
"dependencies": {
|
||||
"@auth0/auth0-spa-js": "^2.0.2",
|
||||
"@egjs/vue-infinitegrid": "3.3.0",
|
||||
"axios": "^1.6.0",
|
||||
"axios": "^0.25.0",
|
||||
"bootstrap-vue": "^2.21.2",
|
||||
"core-js": "^3.6.5",
|
||||
"d3": "^5.6.1",
|
||||
@ -17,7 +17,7 @@
|
||||
"dom-to-image": "^2.6.0",
|
||||
"fslightbox-vue": "fslightbox-vue.tgz",
|
||||
"nouislider": "^15.2.0",
|
||||
"onnxruntime-web": "1.15.1",
|
||||
"onnxruntime-web": "^1.15.1",
|
||||
"underscore": "^1.13.1",
|
||||
"vue": "^2.6.12",
|
||||
"vue-color": "^2.8.1",
|
||||
|
||||
@ -69,8 +69,7 @@ class Sist2Api {
|
||||
hit._props.isImage = true;
|
||||
}
|
||||
if ("width" in hit._source && !hit._props.isSubDocument && hit._source.videoc !== "tiff"
|
||||
&& hit._source.videoc !== "raw" && hit._source.videoc !== "ppm"
|
||||
&& hit._source.mime !== "image/jp2") {
|
||||
&& hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
|
||||
hit._props.isPlayableImage = true;
|
||||
}
|
||||
if ("width" in hit._source && "height" in hit._source) {
|
||||
@ -145,6 +144,8 @@ class Sist2Api {
|
||||
}
|
||||
|
||||
_getIndexRoot(indexId) {
|
||||
console.log(indexId)
|
||||
console.log(this.sist2Info.indices.find(idx => idx.id === indexId))
|
||||
return this.sist2Info.indices.find(idx => idx.id === indexId).root;
|
||||
}
|
||||
|
||||
@ -310,7 +311,7 @@ class Sist2Api {
|
||||
}
|
||||
|
||||
getTagsSqlite() {
|
||||
return axios.get(`${this.baseUrl}fts/tags`)
|
||||
return axios.get(`${this.baseUrl}/fts/tags`)
|
||||
.then(resp => {
|
||||
return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
|
||||
});
|
||||
@ -567,7 +568,7 @@ class Sist2Api {
|
||||
}
|
||||
|
||||
getDocumentSqlite(sid) {
|
||||
return axios.get(`${this.baseUrl}fts/d/${sid}`)
|
||||
return axios.get(`${this.baseUrl}/fts/d/${sid}`)
|
||||
.then(resp => ({
|
||||
_source: resp.data
|
||||
}));
|
||||
@ -590,7 +591,7 @@ class Sist2Api {
|
||||
}
|
||||
|
||||
getTagSuggestionsSqlite(prefix) {
|
||||
return axios.post(`${this.baseUrl}fts/suggestTags`, prefix)
|
||||
return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
|
||||
.then(resp => (resp.data));
|
||||
}
|
||||
|
||||
@ -621,7 +622,7 @@ class Sist2Api {
|
||||
}
|
||||
|
||||
getEmbeddings(sid, modelId) {
|
||||
return axios.post(`${this.baseUrl}e/${sid}/${modelId.toString().padStart(3, '0')}`)
|
||||
return axios.post(`${this.baseUrl}/e/${sid}/${modelId.toString().padStart(3, '0')}`)
|
||||
.then(resp => (resp.data));
|
||||
}
|
||||
}
|
||||
|
||||
@ -117,11 +117,11 @@ class Sist2ElasticsearchQuery {
|
||||
}
|
||||
|
||||
if (dateMin && dateMax) {
|
||||
filters.push({range: {mtime: {gte: dateMin, lte: dateMax, format: "epoch_second"}}})
|
||||
filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
|
||||
} else if (dateMin) {
|
||||
filters.push({range: {mtime: {gte: dateMin, format: "epoch_second"}}})
|
||||
filters.push({range: {mtime: {gte: dateMin}}})
|
||||
} else if (dateMax) {
|
||||
filters.push({range: {mtime: {lte: dateMax, format: "epoch_second"}}})
|
||||
filters.push({range: {mtime: {lte: dateMax}}})
|
||||
}
|
||||
|
||||
const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
|
||||
@ -238,7 +238,7 @@ class Sist2ElasticsearchQuery {
|
||||
pre_tags: ["<mark>"],
|
||||
post_tags: ["</mark>"],
|
||||
fragment_size: getters.optFragmentSize,
|
||||
number_of_fragments: getters.optFragmentCount,
|
||||
number_of_fragments: 1,
|
||||
order: "score",
|
||||
fields: {
|
||||
content: {},
|
||||
|
||||
@ -3,8 +3,6 @@
|
||||
</template>
|
||||
|
||||
<script>
|
||||
const FRAGMENT_SEPARATOR = "<br /><i style='line-height: 2.4'>[…]</i><br/>";
|
||||
|
||||
export default {
|
||||
name: "ContentDiv",
|
||||
props: ["doc"],
|
||||
@ -15,10 +13,10 @@ export default {
|
||||
}
|
||||
|
||||
if (this.doc.highlight["content.nGram"]) {
|
||||
return this.doc.highlight["content.nGram"].join(FRAGMENT_SEPARATOR);
|
||||
return this.doc.highlight["content.nGram"][0];
|
||||
}
|
||||
if (this.doc.highlight.content) {
|
||||
return this.doc.highlight.content.join(FRAGMENT_SEPARATOR);
|
||||
return this.doc.highlight.content[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -59,7 +59,7 @@ export default {
|
||||
|
||||
const fields = [
|
||||
"title", "duration", "audioc", "videoc",
|
||||
"bitrate", "artist", "album", "album_artist", "genre", "font_name", "author", "media_comment",
|
||||
"bitrate", "artist", "album", "album_artist", "genre", "font_name", "author",
|
||||
"modified_by", "pages", "tag",
|
||||
"exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
|
||||
"exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",
|
||||
|
||||
@ -77,7 +77,6 @@ export default {
|
||||
return listener(e);
|
||||
}
|
||||
};
|
||||
|
||||
},
|
||||
methods: {
|
||||
keyDownListener(e) {
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
|
||||
<span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
|
||||
v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
|
||||
href="https://github.com/sist2app/sist2/blob/master/docs/USAGE.md#elasticsearch"
|
||||
href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
|
||||
target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
|
||||
</span>
|
||||
|
||||
|
||||
@ -59,7 +59,6 @@ export default {
|
||||
searchInPath: "Enable matching query against document path",
|
||||
suggestPath: "Enable auto-complete in path filter bar",
|
||||
fragmentSize: "Highlight context size",
|
||||
fragmentCount: "Number of highlight snippets",
|
||||
queryMode: "Search mode",
|
||||
displayMode: "Display",
|
||||
columns: "Column count",
|
||||
@ -139,7 +138,7 @@ export default {
|
||||
},
|
||||
debug: "Debug information",
|
||||
debugDescription: "Information useful for debugging. If you encounter bugs or have suggestions for" +
|
||||
" new features, please submit a new issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>here</a>.",
|
||||
" new features, please submit a new issue <a href='https://github.com/simon987/sist2/issues/new/choose'>here</a>.",
|
||||
tagline: "Tagline",
|
||||
toast: {
|
||||
esConnErrTitle: "Elasticsearch connection error",
|
||||
@ -243,7 +242,6 @@ export default {
|
||||
searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
|
||||
suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
|
||||
fragmentSize: "Kontextgröße",
|
||||
fragmentCount: "Anzahl der hervorgehobenen Snippets",
|
||||
queryMode: "Such-Modus",
|
||||
displayMode: "Ansicht",
|
||||
columns: "Anzahl Spalten",
|
||||
@ -320,7 +318,7 @@ export default {
|
||||
},
|
||||
debug: "Debug Informationen",
|
||||
debugDescription: "Informationen für das Debugging. Wenn du Bugs gefunden oder Anregungen für " +
|
||||
"neue Features hast, poste sie bitte <a href='https://github.com/sist2app/sist2/issues/new/choose'>hier</a>.",
|
||||
"neue Features hast, poste sie bitte <a href='https://github.com/simon987/sist2/issues/new/choose'>hier</a>.",
|
||||
tagline: "Tagline",
|
||||
toast: {
|
||||
esConnErrTitle: "Elasticsearch Verbindungsfehler",
|
||||
@ -419,7 +417,6 @@ export default {
|
||||
searchInPath: "Activer la recherche dans le chemin des documents",
|
||||
suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
|
||||
fragmentSize: "Longueur du contexte de surlignage",
|
||||
fragmentCount: "Nombre d'extraits surlignés",
|
||||
queryMode: "Mode de recherche",
|
||||
displayMode: "Affichage",
|
||||
columns: "Nombre de colonnes",
|
||||
@ -497,7 +494,7 @@ export default {
|
||||
debug: "Information de débogage",
|
||||
debugDescription: "Informations utiles pour le débogage\n" +
|
||||
"Si vous rencontrez des bogues ou si vous avez des suggestions pour de nouvelles fonctionnalités," +
|
||||
" veuillez soumettre un nouvel Issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>ici</a>.",
|
||||
" veuillez soumettre un nouvel Issue <a href='https://github.com/simon987/sist2/issues/new/choose'>ici</a>.",
|
||||
tagline: "Tagline",
|
||||
toast: {
|
||||
esConnErrTitle: "Erreur de connexion Elasticsearch",
|
||||
@ -595,7 +592,6 @@ export default {
|
||||
searchInPath: "匹配文档路径",
|
||||
suggestPath: "搜索框启用自动补全",
|
||||
fragmentSize: "高亮上下文大小",
|
||||
fragmentCount: "突出显示的项目数",
|
||||
queryMode: "搜索模式",
|
||||
displayMode: "显示",
|
||||
columns: "列数",
|
||||
@ -672,7 +668,7 @@ export default {
|
||||
},
|
||||
debug: "调试信息",
|
||||
debugDescription: "对调试除错有用的信息。 若您遇到bug或者想建议新功能，请提交新Issue到" +
|
||||
"<a href='https://github.com/sist2app/sist2/issues/new/choose'>这里</a>.",
|
||||
"<a href='https://github.com/simon987/sist2/issues/new/choose'>这里</a>.",
|
||||
tagline: "标签栏",
|
||||
toast: {
|
||||
esConnErrTitle: "Elasticsearch连接错误",
|
||||
@ -771,7 +767,6 @@ export default {
|
||||
searchInPath: "Włącz szukanie również w ścieżce dokumentu",
|
||||
suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
|
||||
fragmentSize: "Podświetl wielkość kontekstu w znakach",
|
||||
fragmentCount: "Liczba wyróżnionych fragmentów",
|
||||
queryMode: "Tryb szukania",
|
||||
displayMode: "Wyświetlanie",
|
||||
columns: "Liczba kolumn",
|
||||
@ -851,7 +846,7 @@ export default {
|
||||
},
|
||||
debug: "Informacje dla programistów",
|
||||
debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
|
||||
" propozycje zmian, zgłoś to proszę <a href='https://github.com/sist2app/sist2/issues/new/choose'>tutaj</a>.",
|
||||
" propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
|
||||
tagline: "Slogan",
|
||||
toast: {
|
||||
esConnErrTitle: "Problem z połączeniem z Elasticsearch",
|
||||
|
||||
@ -22,9 +22,7 @@ export class CLIPTransformerModel {
|
||||
|
||||
async loadModel(onProgress) {
|
||||
ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
|
||||
if (window.crossOriginIsolated) {
|
||||
ort.env.wasm.numThreads = 2;
|
||||
}
|
||||
ort.env.wasm.numThreads = 2;
|
||||
|
||||
let buf = await ModelStore.get(this._modelUrl);
|
||||
if (!buf) {
|
||||
|
||||
@ -3,7 +3,7 @@ import Vuex from "vuex"
|
||||
import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
|
||||
import {getInstance} from "@/plugins/auth0.js";
|
||||
|
||||
const CONF_VERSION = 4;
|
||||
const CONF_VERSION = 3;
|
||||
|
||||
Vue.use(Vuex);
|
||||
|
||||
@ -41,7 +41,6 @@ export default new Vuex.Store({
|
||||
optTagOrOperator: false,
|
||||
optFuzzy: true,
|
||||
optFragmentSize: 200,
|
||||
optFragmentCount: 1,
|
||||
optQueryMode: "simple",
|
||||
optSearchInPath: false,
|
||||
optColumns: "auto",
|
||||
@ -59,7 +58,7 @@ export default new Vuex.Store({
|
||||
optVidPreviewInterval: 700,
|
||||
optSimpleLightbox: true,
|
||||
optShowTagPickerFilter: true,
|
||||
optMlRepositories: "https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json",
|
||||
optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
|
||||
optAutoAnalyze: false,
|
||||
optMlDefaultModel: null,
|
||||
|
||||
@ -171,7 +170,6 @@ export default new Vuex.Store({
|
||||
setOptSearchInPath: (state, val) => state.optSearchInPath = val,
|
||||
setOptSuggestPath: (state, val) => state.optSuggestPath = val,
|
||||
setOptFragmentSize: (state, val) => state.optFragmentSize = val,
|
||||
setOptFragmentCount: (state, val) => state.optFragmentCount = val,
|
||||
setOptQueryMode: (state, val) => state.optQueryMode = val,
|
||||
setOptResultSize: (state, val) => state.optSize = val,
|
||||
setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
|
||||
@ -432,7 +430,6 @@ export default new Vuex.Store({
|
||||
optSearchInPath: state => state.optSearchInPath,
|
||||
optSuggestPath: state => state.optSuggestPath,
|
||||
optFragmentSize: state => state.optFragmentSize,
|
||||
optFragmentCount: state => state.optFragmentCount,
|
||||
optQueryMode: state => state.optQueryMode,
|
||||
optTreemapType: state => state.optTreemapType,
|
||||
optTreemapTiling: state => state.optTreemapTiling,
|
||||
|
||||
@ -81,7 +81,6 @@
|
||||
<li><code>doc.artist</code></li>
|
||||
<li><code>doc.title</code></li>
|
||||
<li><code>doc.genre</code></li>
|
||||
<li><code>doc.media_comment</code></li>
|
||||
<li><code>doc.album_artist</code></li>
|
||||
<li><code>doc.exif_make</code></li>
|
||||
<li><code>doc.exif_model</code></li>
|
||||
@ -151,10 +150,6 @@
|
||||
<b-form-input :value="optFragmentSize" step="10" type="number" min="0"
|
||||
@input="setOptFragmentSize"></b-form-input>
|
||||
|
||||
<label :class="{'text-muted': uiSqliteMode}">{{ $t("opt.fragmentCount") }}</label>
|
||||
<b-form-input :value="optFragmentCount" :disabled="uiSqliteMode" step="1" type="number" min="1"
|
||||
@input="setOptFragmentCount"></b-form-input>
|
||||
|
||||
<label>{{ $t("opt.resultSize") }}</label>
|
||||
<b-form-input :value="optResultSize" type="number" min="10"
|
||||
@input="setOptResultSize"></b-form-input>
|
||||
@ -318,7 +313,6 @@ export default {
|
||||
"optSearchInPath",
|
||||
"optSuggestPath",
|
||||
"optFragmentSize",
|
||||
"optFragmentCount",
|
||||
"optQueryMode",
|
||||
"optTreemapType",
|
||||
"optTreemapTiling",
|
||||
@ -365,7 +359,6 @@ export default {
|
||||
"setOptSearchInPath",
|
||||
"setOptSuggestPath",
|
||||
"setOptFragmentSize",
|
||||
"setOptFragmentCount",
|
||||
"setOptQueryMode",
|
||||
"setOptTreemapType",
|
||||
"setOptTreemapTiling",
|
||||
|
||||
@ -1,5 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export NODE_OPTIONS=--openssl-legacy-provider
|
||||
|
||||
./node_modules/@vue/cli-service/bin/vue-cli-service.js build --watch
|
||||
@ -25,7 +25,6 @@ const char *TESS_DATAPATHS[] = {
|
||||
"/usr/share/tessdata/",
|
||||
"/usr/share/tesseract-ocr/tessdata/",
|
||||
"/usr/share/tesseract-ocr/4.00/tessdata/",
|
||||
"/usr/share/tesseract-ocr/5/tessdata/",
|
||||
"./",
|
||||
NULL
|
||||
};
|
||||
|
||||
@ -19,7 +19,6 @@
|
||||
#include "src/database/database.h"
|
||||
#include "src/index/elastic.h"
|
||||
#include "sqlite3.h"
|
||||
#include "ignorelist.h"
|
||||
|
||||
#include <pcre.h>
|
||||
|
||||
@ -35,7 +34,6 @@ typedef struct {
|
||||
pcre *exclude;
|
||||
pcre_extra *exclude_extra;
|
||||
int fast;
|
||||
ignorelist_t *ignorelist;
|
||||
|
||||
scan_arc_ctx_t arc_ctx;
|
||||
scan_comic_ctx_t comic_ctx;
|
||||
|
||||
@ -114,7 +114,7 @@ void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv)
|
||||
char buf[PATH_MAX];
|
||||
strcpy(buf, current_job);
|
||||
|
||||
SET_CURRENT_JOB(ipc_ctx, current_job);
|
||||
strcpy(ipc_ctx->current_job[ProcData.thread_id], current_job);
|
||||
|
||||
sqlite3_result_text(ctx, "ok", -1, SQLITE_STATIC);
|
||||
}
|
||||
@ -478,7 +478,8 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
|
||||
database_iterator_t *database_create_delete_list_iterator(database_t *db) {
|
||||
|
||||
sqlite3_stmt *stmt;
|
||||
sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list", -1, &stmt, NULL);
|
||||
sqlite3_prepare_v2(db->db, "SELECT doc.id FROM delete_list "
|
||||
"INNER JOIN document doc ON doc.ROWID = delete_list.id;", -1, &stmt, NULL);
|
||||
|
||||
database_iterator_t *iter = malloc(sizeof(database_iterator_t));
|
||||
|
||||
|
||||
@ -64,8 +64,6 @@ typedef struct {
|
||||
char current_job[MAX_THREADS][PATH_MAX * 2];
|
||||
} database_ipc_ctx_t;
|
||||
|
||||
#define SET_CURRENT_JOB(ctx, job) (strcpy((ctx)->current_job[ProcData.thread_id], job))
|
||||
|
||||
typedef struct {
|
||||
double date_min;
|
||||
double date_max;
|
||||
|
||||
@ -102,9 +102,7 @@ void database_fts_index(database_t *db) {
|
||||
db->db, "DELETE FROM fts.mime_index;", NULL, NULL, NULL));
|
||||
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
|
||||
db->db, "INSERT INTO fts.mime_index (index_id, mime, count) "
|
||||
"SELECT index_id, mime, count(*) FROM fts.document_index "
|
||||
"WHERE mime IS NOT NULL "
|
||||
"GROUP BY index_id, mime",
|
||||
"SELECT index_id, mime, count(*) FROM fts.document_index GROUP BY index_id, mime",
|
||||
NULL, NULL, NULL));
|
||||
|
||||
LOG_DEBUG("database_fts.c", "Generating path index");
|
||||
|
||||
106
src/ignorelist.c
106
src/ignorelist.c
@ -1,106 +0,0 @@
|
||||
#include "ignorelist.h"
|
||||
#include "ctx.h"
|
||||
#include <git2.h>
|
||||
|
||||
typedef struct ignorelist {
|
||||
git_repository *repo;
|
||||
char repo_path[PATH_MAX];
|
||||
int has_rules;
|
||||
} ignorelist_t;
|
||||
|
||||
char *get_tempdir() {
|
||||
char *tempdir_env = getenv("TMPDIR");
|
||||
|
||||
if (tempdir_env != NULL) {
|
||||
return tempdir_env;
|
||||
}
|
||||
|
||||
return "/tmp/";
|
||||
}
|
||||
|
||||
void ignorelist_destroy(ignorelist_t* ignorelist) {
|
||||
git_libgit2_shutdown();
|
||||
|
||||
if (ignorelist->repo != NULL) {
|
||||
git_repository_free(ignorelist->repo);
|
||||
}
|
||||
|
||||
free(ignorelist);
|
||||
}
|
||||
|
||||
ignorelist_t *ignorelist_create() {
|
||||
git_libgit2_init();
|
||||
|
||||
ignorelist_t *ignorelist = malloc(sizeof(ignorelist_t));
|
||||
|
||||
ignorelist->repo = NULL;
|
||||
ignorelist->has_rules = FALSE;
|
||||
|
||||
char *tempdir = get_tempdir();
|
||||
|
||||
if (tempdir[strlen(tempdir) - 1] == '/') {
|
||||
sprintf(ignorelist->repo_path, "%ssist2-ignorelist-%d", tempdir, getpid());
|
||||
} else {
|
||||
sprintf(ignorelist->repo_path, "%s/sist2-ignorelist-%d", tempdir, getpid());
|
||||
}
|
||||
|
||||
return ignorelist;
|
||||
}
|
||||
|
||||
void ignorelist_load_ignore_file(ignorelist_t *ignorelist, const char *filepath) {
|
||||
|
||||
FILE *file;
|
||||
char line[PATH_MAX * 2];
|
||||
|
||||
file = fopen(filepath, "r");
|
||||
|
||||
if(file == NULL) {
|
||||
// No ignore list
|
||||
return;
|
||||
}
|
||||
|
||||
LOG_DEBUGF("ignorelist.c", "Opening temporary git repository %s", ignorelist->repo_path);
|
||||
int init_result = git_repository_init(&ignorelist->repo, ignorelist->repo_path, TRUE);
|
||||
|
||||
if (init_result != 0) {
|
||||
LOG_FATALF("ignorelist.c", "Got error code from git_repository_init(): %d", init_result);
|
||||
}
|
||||
|
||||
git_ignore_clear_internal_rules(ignorelist->repo);
|
||||
|
||||
while(fgets(line, PATH_MAX * 2, file)){
|
||||
|
||||
line[strlen(line) - 1] = '\0'; // Strip trailing newline
|
||||
char *rules = {line,};
|
||||
|
||||
int result = git_ignore_add_rule(ignorelist->repo, rules);
|
||||
|
||||
if (result == 0) {
|
||||
LOG_DEBUGF("ignorelist.c", "Load ignore rule: %s", line);
|
||||
ignorelist->has_rules = TRUE;
|
||||
} else {
|
||||
LOG_FATALF("ignorelist.c", "Invalid ignore rule: %s", line);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(file);
|
||||
}
|
||||
|
||||
int ignorelist_is_ignored(ignorelist_t *ignorelist, const char *filepath) {
|
||||
|
||||
if (!ignorelist->has_rules) {
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
const char *rel_path = filepath + ScanCtx.index.desc.root_len;
|
||||
|
||||
int ignored = -1;
|
||||
|
||||
int result = git_ignore_path_is_ignored(&ignored, ignorelist->repo, rel_path);
|
||||
|
||||
if (result != 0) {
|
||||
LOG_FATALF("ignorelist.c", "git_ignore_path_is_ignored returned error code: %d", result);
|
||||
}
|
||||
|
||||
return ignored;
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
#ifndef SIST2_IGNORELIST_H
|
||||
#define SIST2_IGNORELIST_H
|
||||
|
||||
#include "src/sist.h"
|
||||
|
||||
typedef struct ignorelist ignorelist_t;
|
||||
|
||||
ignorelist_t *ignorelist_create();
|
||||
|
||||
void ignorelist_destroy(ignorelist_t* ignorelist);
|
||||
|
||||
void ignorelist_load_ignore_file(ignorelist_t* ignorelist, const char* filepath);
|
||||
|
||||
int ignorelist_is_ignored(ignorelist_t* ignorelist, const char* filepath);
|
||||
|
||||
#endif //SIST2_IGNORELIST_H
|
||||
@ -206,7 +206,7 @@ response_t *web_put(const char *url, const char *data, int insecure) {
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
|
||||
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
|
||||
curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
|
||||
curl_easy_setopt(curl, CURLOPT_SHARE, 0);
|
||||
curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
|
||||
curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4);
|
||||
if (insecure) {
|
||||
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
|
||||
|
||||
@ -30,8 +30,6 @@ char *get_meta_key_text(enum metakey meta_key) {
|
||||
return "genre";
|
||||
case MetaTitle:
|
||||
return "title";
|
||||
case MetaMediaComment:
|
||||
return "media_comment";
|
||||
case MetaFontName:
|
||||
return "font_name";
|
||||
case MetaExifMake:
|
||||
@ -161,7 +159,6 @@ void write_document(document_t *doc) {
|
||||
case MetaExifGpsLatitudeDec:
|
||||
case MetaExifGpsLatitudeRef:
|
||||
case MetaChecksum:
|
||||
case MetaMediaComment:
|
||||
case MetaTitle: {
|
||||
cJSON_AddStringToObject(json, get_meta_key_text(meta->key), meta->str_val);
|
||||
buffer_size_guess += (int) strlen(meta->str_val);
|
||||
|
||||
@ -23,17 +23,8 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st
|
||||
if (ScanCtx.exclude != NULL && EXCLUDED(filepath)) {
|
||||
LOG_DEBUGF("walk.c", "Excluded: %s", filepath);
|
||||
|
||||
if (typeflag == FTW_D) {
|
||||
return FTW_SKIP_SUBTREE;
|
||||
}
|
||||
|
||||
return FTW_CONTINUE;
|
||||
}
|
||||
|
||||
if (ignorelist_is_ignored(ScanCtx.ignorelist, filepath)) {
|
||||
LOG_DEBUGF("walk.c", "Ignored: %s", filepath);
|
||||
|
||||
if (typeflag == FTW_D) {
|
||||
if (typeflag == FTW_F && S_ISREG(info->st_mode)) {
|
||||
} else if (typeflag == FTW_D) {
|
||||
return FTW_SKIP_SUBTREE;
|
||||
}
|
||||
|
||||
|
||||
14
src/main.c
14
src/main.c
@ -11,7 +11,7 @@
|
||||
#include "web/serve.h"
|
||||
#include "parsing/mime.h"
|
||||
#include "parsing/parse.h"
|
||||
#include "ignorelist.h"
|
||||
#include "auth0/auth0_c_api.h"
|
||||
|
||||
#include <signal.h>
|
||||
#include <pthread.h>
|
||||
@ -240,13 +240,6 @@ void sist2_scan(scan_args_t *args) {
|
||||
|
||||
LOG_INFOF("main.c", "sist2 v%s", Version);
|
||||
|
||||
ScanCtx.ignorelist = ignorelist_create();
|
||||
|
||||
char ignore_filepath[PATH_MAX];
|
||||
sprintf(ignore_filepath, "%s.sist2ignore", args->path);
|
||||
|
||||
ignorelist_load_ignore_file(ScanCtx.ignorelist, ignore_filepath);
|
||||
|
||||
ScanCtx.pool = tpool_create(ScanCtx.threads, TRUE);
|
||||
tpool_start(ScanCtx.pool);
|
||||
|
||||
@ -276,7 +269,6 @@ void sist2_scan(scan_args_t *args) {
|
||||
|
||||
database_generate_stats(db, args->treemap_threshold);
|
||||
database_close(db, args->optimize_database);
|
||||
ignorelist_destroy(ScanCtx.ignorelist);
|
||||
}
|
||||
|
||||
void sist2_index(index_args_t *args) {
|
||||
@ -433,8 +425,6 @@ int set_to_negative_if_value_is_zero(UNUSED(struct argparse *self), const struct
|
||||
fprintf(stderr, "error: option `--%s` Value must be >= 0\n", option->long_name);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
@ -554,7 +544,7 @@ int main(int argc, const char *argv[]) {
|
||||
OPT_END(),
|
||||
};
|
||||
|
||||
struct argparse argparse = {};
|
||||
struct argparse argparse;
|
||||
argparse_init(&argparse, options, usage, 0);
|
||||
argparse_describe(
|
||||
&argparse,
|
||||
|
||||
@ -295,167 +295,166 @@ image_g3fax=524575,
|
||||
image_gif=524576,
|
||||
image_heic=524577,
|
||||
image_ief=524578,
|
||||
image_jp2=524579,
|
||||
image_jpeg=524580,
|
||||
image_jutvision=524581,
|
||||
image_naplps=524582,
|
||||
image_pict=524583,
|
||||
image_png=524584,
|
||||
image_svg=524585 | 0x80000000,
|
||||
image_svg_xml=524586 | 0x80000000,
|
||||
image_tiff=524587,
|
||||
image_vnd_adobe_photoshop=524588 | 0x80000000,
|
||||
image_vnd_djvu=524589 | 0x80000000,
|
||||
image_vnd_fpx=524590,
|
||||
image_vnd_microsoft_icon=524591,
|
||||
image_vnd_rn_realflash=524592,
|
||||
image_vnd_rn_realpix=524593,
|
||||
image_vnd_wap_wbmp=524594,
|
||||
image_vnd_xiff=524595,
|
||||
image_webp=524596,
|
||||
image_wmf=524597,
|
||||
image_x_3ds=524598,
|
||||
image_x_adobe_dng=524599 | 0x00800000,
|
||||
image_x_award_bioslogo=524600,
|
||||
image_x_canon_cr2=524601 | 0x00800000,
|
||||
image_x_canon_crw=524602 | 0x00800000,
|
||||
image_x_cmu_raster=524603,
|
||||
image_x_cur=524604,
|
||||
image_x_dcraw=524605 | 0x00800000,
|
||||
image_x_dwg=524606,
|
||||
image_x_eps=524607,
|
||||
image_x_epson_erf=524608 | 0x00800000,
|
||||
image_x_exr=524609,
|
||||
image_x_fuji_raf=524610 | 0x00800000,
|
||||
image_x_gem=524611,
|
||||
image_x_icns=524612,
|
||||
image_x_icon=524613 | 0x80000000,
|
||||
image_x_jg=524614,
|
||||
image_x_jps=524615,
|
||||
image_x_kodak_dcr=524616 | 0x00800000,
|
||||
image_x_kodak_k25=524617 | 0x00800000,
|
||||
image_x_kodak_kdc=524618 | 0x00800000,
|
||||
image_x_minolta_mrw=524619 | 0x00800000,
|
||||
image_x_ms_bmp=524620,
|
||||
image_x_niff=524621,
|
||||
image_x_nikon_nef=524622 | 0x00800000,
|
||||
image_x_olympus_orf=524623 | 0x00800000,
|
||||
image_x_panasonic_raw=524624 | 0x00800000,
|
||||
image_x_pcx=524625,
|
||||
image_x_pentax_pef=524626 | 0x00800000,
|
||||
image_x_pict=524627,
|
||||
image_x_portable_bitmap=524628,
|
||||
image_x_portable_graymap=524629,
|
||||
image_x_portable_pixmap=524630,
|
||||
image_x_quicktime=524631,
|
||||
image_x_rgb=524632,
|
||||
image_x_sigma_x3f=524633 | 0x00800000,
|
||||
image_x_sony_arw=524634 | 0x00800000,
|
||||
image_x_sony_sr2=524635 | 0x00800000,
|
||||
image_x_sony_srf=524636 | 0x00800000,
|
||||
image_x_tga=524637,
|
||||
image_x_tiff=524638,
|
||||
image_x_win_bitmap=524639,
|
||||
image_x_xcf=524640 | 0x80000000,
|
||||
image_x_xpixmap=524641 | 0x80000000,
|
||||
image_x_xwindowdump=524642,
|
||||
message_news=196963,
|
||||
message_rfc822=196964,
|
||||
model_vnd_dwf=65893,
|
||||
model_vnd_gdl=65894,
|
||||
model_vnd_gs_gdl=65895,
|
||||
model_vrml=65896,
|
||||
model_x_pov=65897,
|
||||
text_PGP=590186,
|
||||
text_asp=590187,
|
||||
text_css=590188,
|
||||
text_csv=590189,
|
||||
text_html=590190 | 0x01000000,
|
||||
text_javascript=590191,
|
||||
text_mcf=590192,
|
||||
text_pascal=590193,
|
||||
text_plain=590194,
|
||||
text_richtext=590195,
|
||||
text_rtf=590196,
|
||||
text_scriplet=590197,
|
||||
text_tab_separated_values=590198,
|
||||
text_troff=590199,
|
||||
text_uri_list=590200,
|
||||
text_vnd_abc=590201,
|
||||
text_vnd_fmi_flexstor=590202,
|
||||
text_vnd_wap_wml=590203,
|
||||
text_vnd_wap_wmlscript=590204,
|
||||
text_webviewhtml=590205,
|
||||
text_x_Algol68=590206,
|
||||
text_x_asm=590207,
|
||||
text_x_audiosoft_intra=590208,
|
||||
text_x_awk=590209,
|
||||
text_x_bcpl=590210,
|
||||
text_x_c=590211,
|
||||
text_x_c__=590212,
|
||||
text_x_component=590213,
|
||||
text_x_diff=590214,
|
||||
text_x_fortran=590215,
|
||||
text_x_java=590216,
|
||||
text_x_la_asf=590217,
|
||||
text_x_lisp=590218,
|
||||
text_x_m=590219,
|
||||
text_x_m4=590220,
|
||||
text_x_makefile=590221,
|
||||
text_x_ms_regedit=590222,
|
||||
text_x_msdos_batch=590223,
|
||||
text_x_objective_c=590224,
|
||||
text_x_pascal=590225,
|
||||
text_x_perl=590226,
|
||||
text_x_php=590227,
|
||||
text_x_po=590228,
|
||||
text_x_python=590229,
|
||||
text_x_ruby=590230,
|
||||
text_x_sass=590231,
|
||||
text_x_script_python=590232,
|
||||
text_x_scss=590233,
|
||||
text_x_server_parsed_html=590234,
|
||||
text_x_setext=590235,
|
||||
text_x_sgml=590236 | 0x01000000,
|
||||
text_x_shellscript=590237,
|
||||
text_x_speech=590238,
|
||||
text_x_tcl=590239,
|
||||
text_x_tex=590240,
|
||||
text_x_uil=590241,
|
||||
text_x_uuencode=590242,
|
||||
text_x_vcalendar=590243,
|
||||
text_x_vcard=590244,
|
||||
text_xml=590245 | 0x01000000,
|
||||
video_MP2T=393638,
|
||||
video_animaflex=393639,
|
||||
video_avi=393640,
|
||||
video_avs_video=393641,
|
||||
video_mp4=393642,
|
||||
video_mpeg=393643,
|
||||
video_quicktime=393644,
|
||||
video_vdo=393645,
|
||||
video_vivo=393646,
|
||||
video_vnd_rn_realvideo=393647,
|
||||
video_vosaic=393648,
|
||||
video_webm=393649,
|
||||
video_x_amt_demorun=393650,
|
||||
video_x_amt_showrun=393651,
|
||||
video_x_atomic3d_feature=393652,
|
||||
video_x_dl=393653,
|
||||
video_x_dv=393654,
|
||||
video_x_fli=393655,
|
||||
video_x_flv=393656,
|
||||
video_x_isvideo=393657,
|
||||
video_x_jng=393658 | 0x80000000,
|
||||
video_x_m4v=393659,
|
||||
video_x_matroska=393660,
|
||||
video_x_mng=393661,
|
||||
video_x_motion_jpeg=393662,
|
||||
video_x_ms_asf=393663,
|
||||
video_x_msvideo=393664,
|
||||
video_x_qtc=393665,
|
||||
video_x_sgi_movie=393666,
|
||||
x_epoc_x_sisx_app=721347,
|
||||
image_jpeg=524579,
|
||||
image_jutvision=524580,
|
||||
image_naplps=524581,
|
||||
image_pict=524582,
|
||||
image_png=524583,
|
||||
image_svg=524584 | 0x80000000,
|
||||
image_svg_xml=524585 | 0x80000000,
|
||||
image_tiff=524586,
|
||||
image_vnd_adobe_photoshop=524587 | 0x80000000,
|
||||
image_vnd_djvu=524588 | 0x80000000,
|
||||
image_vnd_fpx=524589,
|
||||
image_vnd_microsoft_icon=524590,
|
||||
image_vnd_rn_realflash=524591,
|
||||
image_vnd_rn_realpix=524592,
|
||||
image_vnd_wap_wbmp=524593,
|
||||
image_vnd_xiff=524594,
|
||||
image_webp=524595,
|
||||
image_wmf=524596,
|
||||
image_x_3ds=524597,
|
||||
image_x_adobe_dng=524598 | 0x00800000,
|
||||
image_x_award_bioslogo=524599,
|
||||
image_x_canon_cr2=524600 | 0x00800000,
|
||||
image_x_canon_crw=524601 | 0x00800000,
|
||||
image_x_cmu_raster=524602,
|
||||
image_x_cur=524603,
|
||||
image_x_dcraw=524604 | 0x00800000,
|
||||
image_x_dwg=524605,
|
||||
image_x_eps=524606,
|
||||
image_x_epson_erf=524607 | 0x00800000,
|
||||
image_x_exr=524608,
|
||||
image_x_fuji_raf=524609 | 0x00800000,
|
||||
image_x_gem=524610,
|
||||
image_x_icns=524611,
|
||||
image_x_icon=524612 | 0x80000000,
|
||||
image_x_jg=524613,
|
||||
image_x_jps=524614,
|
||||
image_x_kodak_dcr=524615 | 0x00800000,
|
||||
image_x_kodak_k25=524616 | 0x00800000,
|
||||
image_x_kodak_kdc=524617 | 0x00800000,
|
||||
image_x_minolta_mrw=524618 | 0x00800000,
|
||||
image_x_ms_bmp=524619,
|
||||
image_x_niff=524620,
|
||||
image_x_nikon_nef=524621 | 0x00800000,
|
||||
image_x_olympus_orf=524622 | 0x00800000,
|
||||
image_x_panasonic_raw=524623 | 0x00800000,
|
||||
image_x_pcx=524624,
|
||||
image_x_pentax_pef=524625 | 0x00800000,
|
||||
image_x_pict=524626,
|
||||
image_x_portable_bitmap=524627,
|
||||
image_x_portable_graymap=524628,
|
||||
image_x_portable_pixmap=524629,
|
||||
image_x_quicktime=524630,
|
||||
image_x_rgb=524631,
|
||||
image_x_sigma_x3f=524632 | 0x00800000,
|
||||
image_x_sony_arw=524633 | 0x00800000,
|
||||
image_x_sony_sr2=524634 | 0x00800000,
|
||||
image_x_sony_srf=524635 | 0x00800000,
|
||||
image_x_tga=524636,
|
||||
image_x_tiff=524637,
|
||||
image_x_win_bitmap=524638,
|
||||
image_x_xcf=524639 | 0x80000000,
|
||||
image_x_xpixmap=524640 | 0x80000000,
|
||||
image_x_xwindowdump=524641,
|
||||
message_news=196962,
|
||||
message_rfc822=196963,
|
||||
model_vnd_dwf=65892,
|
||||
model_vnd_gdl=65893,
|
||||
model_vnd_gs_gdl=65894,
|
||||
model_vrml=65895,
|
||||
model_x_pov=65896,
|
||||
text_PGP=590185,
|
||||
text_asp=590186,
|
||||
text_css=590187,
|
||||
text_csv=590188,
|
||||
text_html=590189 | 0x01000000,
|
||||
text_javascript=590190,
|
||||
text_mcf=590191,
|
||||
text_pascal=590192,
|
||||
text_plain=590193,
|
||||
text_richtext=590194,
|
||||
text_rtf=590195,
|
||||
text_scriplet=590196,
|
||||
text_tab_separated_values=590197,
|
||||
text_troff=590198,
|
||||
text_uri_list=590199,
|
||||
text_vnd_abc=590200,
|
||||
text_vnd_fmi_flexstor=590201,
|
||||
text_vnd_wap_wml=590202,
|
||||
text_vnd_wap_wmlscript=590203,
|
||||
text_webviewhtml=590204,
|
||||
text_x_Algol68=590205,
|
||||
text_x_asm=590206,
|
||||
text_x_audiosoft_intra=590207,
|
||||
text_x_awk=590208,
|
||||
text_x_bcpl=590209,
|
||||
text_x_c=590210,
|
||||
text_x_c__=590211,
|
||||
text_x_component=590212,
|
||||
text_x_diff=590213,
|
||||
text_x_fortran=590214,
|
||||
text_x_java=590215,
|
||||
text_x_la_asf=590216,
|
||||
text_x_lisp=590217,
|
||||
text_x_m=590218,
|
||||
text_x_m4=590219,
|
||||
text_x_makefile=590220,
|
||||
text_x_ms_regedit=590221,
|
||||
text_x_msdos_batch=590222,
|
||||
text_x_objective_c=590223,
|
||||
text_x_pascal=590224,
|
||||
text_x_perl=590225,
|
||||
text_x_php=590226,
|
||||
text_x_po=590227,
|
||||
text_x_python=590228,
|
||||
text_x_ruby=590229,
|
||||
text_x_sass=590230,
|
||||
text_x_script_python=590231,
|
||||
text_x_scss=590232,
|
||||
text_x_server_parsed_html=590233,
|
||||
text_x_setext=590234,
|
||||
text_x_sgml=590235 | 0x01000000,
|
||||
text_x_shellscript=590236,
|
||||
text_x_speech=590237,
|
||||
text_x_tcl=590238,
|
||||
text_x_tex=590239,
|
||||
text_x_uil=590240,
|
||||
text_x_uuencode=590241,
|
||||
text_x_vcalendar=590242,
|
||||
text_x_vcard=590243,
|
||||
text_xml=590244 | 0x01000000,
|
||||
video_MP2T=393637,
|
||||
video_animaflex=393638,
|
||||
video_avi=393639,
|
||||
video_avs_video=393640,
|
||||
video_mp4=393641,
|
||||
video_mpeg=393642,
|
||||
video_quicktime=393643,
|
||||
video_vdo=393644,
|
||||
video_vivo=393645,
|
||||
video_vnd_rn_realvideo=393646,
|
||||
video_vosaic=393647,
|
||||
video_webm=393648,
|
||||
video_x_amt_demorun=393649,
|
||||
video_x_amt_showrun=393650,
|
||||
video_x_atomic3d_feature=393651,
|
||||
video_x_dl=393652,
|
||||
video_x_dv=393653,
|
||||
video_x_fli=393654,
|
||||
video_x_flv=393655,
|
||||
video_x_isvideo=393656,
|
||||
video_x_jng=393657 | 0x80000000,
|
||||
video_x_m4v=393658,
|
||||
video_x_matroska=393659,
|
||||
video_x_mng=393660,
|
||||
video_x_motion_jpeg=393661,
|
||||
video_x_ms_asf=393662,
|
||||
video_x_msvideo=393663,
|
||||
video_x_qtc=393664,
|
||||
video_x_sgi_movie=393665,
|
||||
x_epoc_x_sisx_app=721346,
|
||||
};
|
||||
char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
|
||||
case application_x_matlab_data: return "application/x-matlab-data";
|
||||
@ -909,7 +908,6 @@ case image_x_sony_arw: return "image/x-sony-arw";
|
||||
case image_x_sony_sr2: return "image/x-sony-sr2";
|
||||
case image_x_sony_srf: return "image/x-sony-srf";
|
||||
case image_x_epson_erf: return "image/x-epson-erf";
|
||||
case image_jp2: return "image/jp2";
|
||||
default: return NULL;}}
|
||||
unsigned int mime_extension_lookup(unsigned long extension_crc32) {switch (extension_crc32) {
|
||||
case 2495639202:return application_x_matlab_data;
|
||||
@ -1293,7 +1291,6 @@ case 1698465774:return image_x_sony_arw;
|
||||
case 2083014127:return image_x_sony_sr2;
|
||||
case 271503362:return image_x_sony_srf;
|
||||
case 142938048:return image_x_epson_erf;
|
||||
case 1575600018:return image_jp2;
|
||||
default: return 0;}}
|
||||
unsigned int mime_name_lookup(unsigned long mime_crc32) {switch (mime_crc32) {
|
||||
case 3272851765: return application_x_matlab_data;
|
||||
@ -1747,8 +1744,7 @@ case 3060720351: return image_x_sony_arw;
|
||||
case 2944016606: return image_x_sony_sr2;
|
||||
case 3279729971: return image_x_sony_srf;
|
||||
case 1665206815: return image_x_epson_erf;
|
||||
case 1849479005: return image_jp2;
|
||||
default: return 0;}}
|
||||
unsigned int mime_ids[] = {655530,655363,655364,655365,655366,655362,655361,655367,655368,655369,655370,655371,655372 | 0x40000000,655373,655374,655375,655376 | 0x08000000,655377,655378,655379,655380,655382,655381,655383,655384,655390,655385,655386,655387,655388,655389,655391,655392,655393,655394,655395 | 0x40000000,655396,655397,655398,655399,655400,655401,655402,655403,655404,655405,655406,655407,655408,655411,655412,655413,655414,655415,655416,655417,655418,655419 | 0x20000000,655421,655422,655423,655424,655425,655426,655427,655428,655429,655430,655431,655432 | 0x04000000,655433 | 0x04000000,655434 | 0x04000000,655435,655436,655437,655438,655439,655440,655441,655442,655443,655444,655445,655446 | 0x10000000,655447,655448,655449 | 0x10000000,655450,655451,655452,655453,655454,655455,655456,655457,655458,655459,655461 | 0x08000000,655460,655462,655463,655464,655465,655466,655467,655468,655469,655470,655471,655472,655473,655474,655475,655476,655477,655478,655479,655480,1,655481,655482,655483,655484,655485,655486,655487,655488,655489 | 0x20000000,655490,655491,655492,655493,655494,655495,655496,655497,655498,655499,655500,655501,655502,655503,655504,655505,655506,655507,655508,655509,655510,655511,655512,655513,655514,655515,655516,655517,655519,655518 | 0x08000000,655521,655520,655522 | 0x08000000,655523 | 0x08000000,655524 | 0x08000000,655525,655526,655527,655528,655529,655531,655532,655533,655534,655535,655599,655536 | 0x02000000,655409 | 0x02000000,655540,655537,655538,655539,655541,655542,655543,655544,655545,655546,655547,655548,655549,655550,655552,655551,655553,655554,655555,655556,655557,655558,655559,655560,655561,655562 | 0x10000000,655563,655564,655565,655566,655567,655569,655568,655570,655571,655572,655573,655574,655575,655576,655577,655578 | 0x10000000,655579,655580,655581,655583,655582,655584,655585,655586,655587,655588,655589,655590,655591,655592,655593,655594,655595 | 0x08000000,655596,655597 | 0x08000000,655600 | 0x10000000,655601,458994 | 0x80000000,458995,458996,458998,458997,458999,459000,459001,459002,459003,459004,459005,459006,459007,459008,459009,459010,459011,459012,459013,459014,459015,459016,459017,459018,459030,459019,459020,459021,459022,459023,459025,459024,459026,459027,459029 | 0x80000000,459028 | 0x80000000,327959 | 0x20000000,327960 | 0x20000000,327962 | 0x20000000,327961 | 0x20000000,524571,524572,524573,524574,524575,524576,524577,524578,524580,524581,524582,524583,524584,524585 | 0x80000000,524586 | 0x80000000,524587,524588 | 0x80000000,524589 | 0x80000000,524590,524591,524592,524593,524594,524595,524596,524597,524598,524600,524603,524604,524606,524607,524609,524611,524612,524613 | 0x80000000,524614,524615,524620,524621,524625,524627,524628,524629,524630,524631,524632,524637,524638,524639,524640 | 0x80000000,524641 | 0x80000000,524642,196963,196964,65893,65894,65895,65896,65897,590187,590188,590190 | 0x01000000,590191,590192,590193,590186,590194,590232,590189,655410,590195,590196,590197,590198,590199,590200,590201,590202,590204,590203,590205,590206,590207,590208,590209,590210,590211,590212,590213,590214,590215,590216,590217,590218,590220,590221,590245 | 0x01000000,590219,590223,590222,590224,590225,590226,590227,590228,590229,590230,590231,590233,590234,590235,590236 | 0x01000000,590237,590238,590239,590240,590241,590242,590243,590244,393639,393640,393641,393638,393642,393643,393644,393645,393646,393647,393648,393649,393650,393651,393652,393653,393654,393655,393656,393657,393658 | 0x80000000,393659,393660,393661,393662,393663,393664,393665,393666,721347,655598,655420,524623 | 0x00800000,524622 | 0x00800000,524610 | 0x00800000,524624 | 0x00800000,524599 | 0x00800000,524601 | 0x00800000,524602 | 0x00800000,524605 | 0x00800000,524616 | 0x00800000,524617 | 0x00800000,524618 | 0x00800000,524619 | 0x00800000,524626 | 0x00800000,524633 | 0x00800000,524634 | 0x00800000,524635 | 0x00800000,524636 | 0x00800000,524608 | 0x00800000,524579,0};
|
||||
unsigned int mime_ids[] = {655530,655363,655364,655365,655366,655362,655361,655367,655368,655369,655370,655371,655372 | 0x40000000,655373,655374,655375,655376 | 0x08000000,655377,655378,655379,655380,655382,655381,655383,655384,655390,655385,655386,655387,655388,655389,655391,655392,655393,655394,655395 | 0x40000000,655396,655397,655398,655399,655400,655401,655402,655403,655404,655405,655406,655407,655408,655411,655412,655413,655414,655415,655416,655417,655418,655419 | 0x20000000,655421,655422,655423,655424,655425,655426,655427,655428,655429,655430,655431,655432 | 0x04000000,655433 | 0x04000000,655434 | 0x04000000,655435,655436,655437,655438,655439,655440,655441,655442,655443,655444,655445,655446 | 0x10000000,655447,655448,655449 | 0x10000000,655450,655451,655452,655453,655454,655455,655456,655457,655458,655459,655461 | 0x08000000,655460,655462,655463,655464,655465,655466,655467,655468,655469,655470,655471,655472,655473,655474,655475,655476,655477,655478,655479,655480,1,655481,655482,655483,655484,655485,655486,655487,655488,655489 | 0x20000000,655490,655491,655492,655493,655494,655495,655496,655497,655498,655499,655500,655501,655502,655503,655504,655505,655506,655507,655508,655509,655510,655511,655512,655513,655514,655515,655516,655517,655519,655518 | 0x08000000,655521,655520,655522 | 0x08000000,655523 | 0x08000000,655524 | 0x08000000,655525,655526,655527,655528,655529,655531,655532,655533,655534,655535,655599,655536 | 0x02000000,655409 | 0x02000000,655540,655537,655538,655539,655541,655542,655543,655544,655545,655546,655547,655548,655549,655550,655552,655551,655553,655554,655555,655556,655557,655558,655559,655560,655561,655562 | 0x10000000,655563,655564,655565,655566,655567,655569,655568,655570,655571,655572,655573,655574,655575,655576,655577,655578 | 0x10000000,655579,655580,655581,655583,655582,655584,655585,655586,655587,655588,655589,655590,655591,655592,655593,655594,655595 | 0x08000000,655596,655597 | 0x08000000,655600 | 0x10000000,655601,458994 | 0x80000000,458995,458996,458998,458997,458999,459000,459001,459002,459003,459004,459005,459006,459007,459008,459009,459010,459011,459012,459013,459014,459015,459016,459017,459018,459030,459019,459020,459021,459022,459023,459025,459024,459026,459027,459029 | 0x80000000,459028 | 0x80000000,327959 | 0x20000000,327960 | 0x20000000,327962 | 0x20000000,327961 | 0x20000000,524571,524572,524573,524574,524575,524576,524577,524578,524579,524580,524581,524582,524583,524584 | 0x80000000,524585 | 0x80000000,524586,524587 | 0x80000000,524588 | 0x80000000,524589,524590,524591,524592,524593,524594,524595,524596,524597,524599,524602,524603,524605,524606,524608,524610,524611,524612 | 0x80000000,524613,524614,524619,524620,524624,524626,524627,524628,524629,524630,524631,524636,524637,524638,524639 | 0x80000000,524640 | 0x80000000,524641,196962,196963,65892,65893,65894,65895,65896,590186,590187,590189 | 0x01000000,590190,590191,590192,590185,590193,590231,590188,655410,590194,590195,590196,590197,590198,590199,590200,590201,590203,590202,590204,590205,590206,590207,590208,590209,590210,590211,590212,590213,590214,590215,590216,590217,590219,590220,590244 | 0x01000000,590218,590222,590221,590223,590224,590225,590226,590227,590228,590229,590230,590232,590233,590234,590235 | 0x01000000,590236,590237,590238,590239,590240,590241,590242,590243,393638,393639,393640,393637,393641,393642,393643,393644,393645,393646,393647,393648,393649,393650,393651,393652,393653,393654,393655,393656,393657 | 0x80000000,393658,393659,393660,393661,393662,393663,393664,393665,721346,655598,655420,524622 | 0x00800000,524621 | 0x00800000,524609 | 0x00800000,524623 | 0x00800000,524598 | 0x00800000,524600 | 0x00800000,524601 | 0x00800000,524604 | 0x00800000,524615 | 0x00800000,524616 | 0x00800000,524617 | 0x00800000,524618 | 0x00800000,524625 | 0x00800000,524632 | 0x00800000,524633 | 0x00800000,524634 | 0x00800000,524635 | 0x00800000,524607 | 0x00800000,0};
|
||||
unsigned int* get_mime_ids() { return mime_ids; }
|
||||
#endif
|
||||
|
||||
@ -142,10 +142,6 @@ void parse(parse_job_t *job) {
|
||||
job->vfile.calculate_checksum = ScanCtx.calculate_checksums;
|
||||
}
|
||||
|
||||
if (IS_SUB_JOB(job)) {
|
||||
SET_CURRENT_JOB(ProcData.ipc_db->ipc_ctx, job->filepath);
|
||||
}
|
||||
|
||||
document_t *doc = malloc(sizeof(document_t));
|
||||
|
||||
strcpy(doc->filepath, job->filepath);
|
||||
@ -165,8 +161,7 @@ void parse(parse_job_t *job) {
|
||||
return;
|
||||
}
|
||||
|
||||
int document_exists = database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime);
|
||||
if (document_exists) {
|
||||
if (database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime)) {
|
||||
CLOSE_FILE(job->vfile)
|
||||
free(doc);
|
||||
return;
|
||||
|
||||
@ -51,17 +51,17 @@
|
||||
#include <ctype.h>
|
||||
#include "git_hash.h"
|
||||
|
||||
#define VERSION "3.5.0"
|
||||
#define VERSION "3.4.0"
|
||||
static const char *const Version = VERSION;
|
||||
static const int VersionMajor = 3;
|
||||
static const int VersionMinor = 5;
|
||||
static const int VersionMinor = 4;
|
||||
static const int VersionPatch = 0;
|
||||
|
||||
#ifndef SIST_PLATFORM
|
||||
#define SIST_PLATFORM unknown
|
||||
#endif
|
||||
|
||||
#define EXPECTED_MONGOOSE_VERSION "7.16"
|
||||
#define EXPECTED_MONGOOSE_VERSION "7.7"
|
||||
|
||||
#define Q(x) #x
|
||||
#define QUOTE(x) Q(x)
|
||||
|
||||
@ -50,13 +50,13 @@ void get_embedding(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
|
||||
sist_id_t sid;
|
||||
|
||||
if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.buf + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid embedding path: %.*s", (int) hm->uri.len, hm->uri.buf);
|
||||
if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid embedding path: %.*s", (int) hm->uri.len, hm->uri.ptr);
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
return;
|
||||
}
|
||||
|
||||
int model_id = (int) strtol(hm->uri.buf + SIST_SID_LEN + 3, NULL, 10);
|
||||
int model_id = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
|
||||
|
||||
database_t *db = web_get_database(sid.index_id);
|
||||
if (db == NULL) {
|
||||
@ -86,11 +86,11 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
char index_id_str[9];
|
||||
char arg_stat_type[5];
|
||||
|
||||
memcpy(index_id_str, hm->uri.buf + 3, 8);
|
||||
memcpy(index_id_str, hm->uri.ptr + 3, 8);
|
||||
*(index_id_str + 8) = '\0';
|
||||
int index_id = (int) strtol(index_id_str, NULL, 16);
|
||||
|
||||
memcpy(arg_stat_type, hm->uri.buf + 3 + 9, 4);
|
||||
memcpy(arg_stat_type, hm->uri.ptr + 3 + 9, 4);
|
||||
*(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';
|
||||
|
||||
database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type);
|
||||
@ -108,6 +108,7 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
|
||||
cJSON *json = database_get_stats(db, stat_type);
|
||||
mg_send_json(nc, json);
|
||||
|
||||
cJSON_Delete(json);
|
||||
}
|
||||
|
||||
@ -135,19 +136,19 @@ void serve_chunk_vendors_js(struct mg_connection *nc, struct mg_http_message *hm
|
||||
}
|
||||
}
|
||||
|
||||
void serve_favicon_ico(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
|
||||
void serve_favicon_ico(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
web_serve_asset_favicon_ico(nc);
|
||||
}
|
||||
|
||||
void serve_style_css(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
|
||||
void serve_style_css(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
web_serve_asset_style_css(nc);
|
||||
}
|
||||
|
||||
void serve_chunk_vendors_css(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
|
||||
void serve_chunk_vendors_css(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
web_serve_asset_chunk_vendors_css(nc);
|
||||
}
|
||||
|
||||
void serve_thumbnail(struct mg_connection *nc, UNUSED(struct mg_http_message *hm), int index_id,
|
||||
void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, int index_id,
|
||||
int doc_id, int arg_num) {
|
||||
|
||||
database_t *db = web_get_database(index_id);
|
||||
@ -168,7 +169,6 @@ void serve_thumbnail(struct mg_connection *nc, UNUSED(struct mg_http_message *hm
|
||||
"Cache-Control: max-age=31536000"
|
||||
);
|
||||
mg_send(nc, data, data_len);
|
||||
nc->is_resp = 0;
|
||||
free(data);
|
||||
} else {
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
@ -179,13 +179,13 @@ void serve_thumbnail(struct mg_connection *nc, UNUSED(struct mg_http_message *hm
|
||||
void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
sist_id_t sid;
|
||||
|
||||
if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.buf + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.buf);
|
||||
if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
return;
|
||||
}
|
||||
|
||||
int num = (int) strtol(hm->uri.buf + SIST_SID_LEN + 3, NULL, 10);
|
||||
int num = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
|
||||
|
||||
serve_thumbnail(nc, hm, sid.index_id, sid.doc_id, num);
|
||||
}
|
||||
@ -193,8 +193,8 @@ void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
sist_id_t sid;
|
||||
|
||||
if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.buf + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.buf);
|
||||
if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
return;
|
||||
}
|
||||
@ -210,14 +210,13 @@ void search(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
}
|
||||
|
||||
char *body = malloc(hm->body.len + 1);
|
||||
memcpy(body, hm->body.buf, hm->body.len);
|
||||
memcpy(body, hm->body.ptr, hm->body.len);
|
||||
*(body + hm->body.len) = '\0';
|
||||
|
||||
char url[4096];
|
||||
snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index);
|
||||
|
||||
nc->fn_data = web_post_async(url, body, WebCtx.es_insecure_ssl);
|
||||
nc->is_resp = 1;
|
||||
}
|
||||
|
||||
void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
|
||||
@ -383,7 +382,11 @@ void index_info(struct mg_connection *nc) {
|
||||
cJSON_AddStringToObject(json, "searchBackend", "elasticsearch");
|
||||
}
|
||||
|
||||
mg_send_json(nc, json);
|
||||
char *json_str = cJSON_PrintUnformatted(json);
|
||||
|
||||
web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
|
||||
mg_send(nc, json_str, strlen(json_str));
|
||||
free(json_str);
|
||||
cJSON_Delete(json);
|
||||
}
|
||||
|
||||
@ -416,8 +419,8 @@ cJSON *get_root_document_by_id(int index_id, int doc_id) {
|
||||
void file(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
sist_id_t sid;
|
||||
|
||||
if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.buf + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.buf);
|
||||
if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr);
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
return;
|
||||
}
|
||||
@ -430,11 +433,6 @@ void file(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
|
||||
cJSON *source = get_root_document_by_id(sid.index_id, sid.doc_id);
|
||||
|
||||
if (source == NULL) {
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
return;
|
||||
}
|
||||
|
||||
if (strlen(idx->desc.rewrite_url) == 0) {
|
||||
serve_file_from_disk(source, idx, nc, hm);
|
||||
} else {
|
||||
@ -452,7 +450,6 @@ void status(struct mg_connection *nc) {
|
||||
}
|
||||
|
||||
free(status);
|
||||
nc->is_resp = 0;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
@ -528,14 +525,14 @@ subreq_ctx_t *elastic_write_tag(const char *sid, const tag_req_t *req) {
|
||||
|
||||
void tag(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
sist_id_t sid;
|
||||
if (hm->uri.len != 22 || !parse_sid(&sid, hm->uri.buf + 5)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.buf);
|
||||
if (hm->uri.len != 22 || !parse_sid(&sid, hm->uri.ptr + 5)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr);
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
return;
|
||||
}
|
||||
|
||||
char *body = malloc(hm->body.len + 1);
|
||||
memcpy(body, hm->body.buf, hm->body.len);
|
||||
memcpy(body, hm->body.ptr, hm->body.len);
|
||||
*(body + hm->body.len) = '\0';
|
||||
cJSON *json = cJSON_Parse(body);
|
||||
free(body);
|
||||
@ -612,7 +609,7 @@ int check_auth0(struct mg_http_message *hm) {
|
||||
}
|
||||
|
||||
token_str = malloc(token.len + 1);
|
||||
strncpy(token_str, token.buf, token.len);
|
||||
strncpy(token_str, token.ptr, token.len);
|
||||
*(token_str + token.len) = '\0';
|
||||
|
||||
int res = auth0_verify_jwt(
|
||||
@ -630,7 +627,7 @@ int check_auth0(struct mg_http_message *hm) {
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
|
||||
static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(void *fn_data)) {
|
||||
|
||||
if (ev == MG_EV_HTTP_MSG) {
|
||||
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
|
||||
@ -642,15 +639,13 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
|
||||
}
|
||||
|
||||
char uri[256];
|
||||
memcpy(uri, hm->uri.buf, hm->uri.len);
|
||||
memcpy(uri, hm->uri.ptr, hm->uri.len);
|
||||
*(uri + hm->uri.len) = '\0';
|
||||
LOG_DEBUGF("serve.c", "<%s> GET %s",
|
||||
web_address_to_string(&(nc->rem)),
|
||||
uri
|
||||
);
|
||||
|
||||
#define mg_http_match_uri(hm, pattern) mg_match((hm)->uri, mg_str(pattern), NULL)
|
||||
|
||||
if (mg_http_match_uri(hm, "/")) {
|
||||
serve_index_html(nc, hm);
|
||||
return;
|
||||
@ -743,7 +738,6 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
|
||||
if (r->status_code == 200) {
|
||||
web_send_headers(nc, 200, r->size, "Content-Type: application/json");
|
||||
mg_send(nc, r->body, r->size);
|
||||
nc->is_resp = 0;
|
||||
} else if (r->status_code == 0) {
|
||||
sist_log("serve.c", LOG_SIST_ERROR, "Could not connect to elasticsearch!");
|
||||
|
||||
|
||||
@ -420,8 +420,8 @@ void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm) {
|
||||
|
||||
sist_id_t sid;
|
||||
|
||||
if (hm->uri.len != 24 || !parse_sid(&sid, hm->uri.buf + 7)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid /fts/d/ path: %.*s", (int) hm->uri.len, hm->uri.buf);
|
||||
if (hm->uri.len != 24 || !parse_sid(&sid, hm->uri.ptr + 7)) {
|
||||
LOG_DEBUGF("serve.c", "Invalid /fts/d/ path: %.*s", (int) hm->uri.len, hm->uri.ptr);
|
||||
HTTP_REPLY_NOT_FOUND
|
||||
return;
|
||||
}
|
||||
|
||||
@ -5,37 +5,31 @@
|
||||
void web_serve_asset_index_html(struct mg_connection *nc) {
|
||||
web_send_headers(nc, 200, sizeof(index_html), HTTP_CROSS_ORIGIN_HEADERS "Content-Type: text/html");
|
||||
mg_send(nc, index_html, sizeof(index_html));
|
||||
nc->is_resp = 0;
|
||||
}
|
||||
|
||||
void web_serve_asset_index_js(struct mg_connection *nc) {
|
||||
web_send_headers(nc, 200, sizeof(index_js), "Content-Type: application/javascript");
|
||||
mg_send(nc, index_js, sizeof(index_js));
|
||||
nc->is_resp = 0;
|
||||
}
|
||||
|
||||
void web_serve_asset_chunk_vendors_js(struct mg_connection *nc) {
|
||||
web_send_headers(nc, 200, sizeof(chunk_vendors_js), "Content-Type: application/javascript");
|
||||
mg_send(nc, chunk_vendors_js, sizeof(chunk_vendors_js));
|
||||
nc->is_resp = 0;
|
||||
}
|
||||
|
||||
void web_serve_asset_favicon_ico(struct mg_connection *nc) {
|
||||
web_send_headers(nc, 200, sizeof(favicon_ico), "Content-Type: image/x-icon");
|
||||
mg_send(nc, favicon_ico, sizeof(favicon_ico));
|
||||
nc->is_resp = 0;
|
||||
}
|
||||
|
||||
void web_serve_asset_style_css(struct mg_connection *nc) {
|
||||
web_send_headers(nc, 200, sizeof(index_css), "Content-Type: text/css");
|
||||
mg_send(nc, index_css, sizeof(index_css));
|
||||
nc->is_resp = 0;
|
||||
}
|
||||
|
||||
void web_serve_asset_chunk_vendors_css(struct mg_connection *nc) {
|
||||
web_send_headers(nc, 200, sizeof(chunk_vendors_css), "Content-Type: text/css");
|
||||
mg_send(nc, chunk_vendors_css, sizeof(chunk_vendors_css));
|
||||
nc->is_resp = 0;
|
||||
}
|
||||
|
||||
index_t *web_get_index_by_id(int index_id) {
|
||||
@ -73,7 +67,7 @@ cJSON *web_get_json_body(struct mg_http_message *hm) {
|
||||
}
|
||||
|
||||
char *body = malloc(hm->body.len + 1);
|
||||
memcpy(body, hm->body.buf, hm->body.len);
|
||||
memcpy(body, hm->body.ptr, hm->body.len);
|
||||
*(body + hm->body.len) = '\0';
|
||||
cJSON *json = cJSON_Parse(body);
|
||||
free(body);
|
||||
@ -87,7 +81,7 @@ char *web_get_string_body(struct mg_http_message *hm) {
|
||||
}
|
||||
|
||||
char *body = malloc(hm->body.len + 1);
|
||||
memcpy(body, hm->body.buf, hm->body.len);
|
||||
memcpy(body, hm->body.ptr, hm->body.len);
|
||||
*(body + hm->body.len) = '\0';
|
||||
|
||||
return body;
|
||||
@ -98,7 +92,6 @@ void mg_send_json(struct mg_connection *nc, const cJSON *json) {
|
||||
|
||||
web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
|
||||
mg_send(nc, json_str, strlen(json_str));
|
||||
nc->is_resp = 0;
|
||||
|
||||
free(json_str);
|
||||
}
|
||||
|
||||
@ -16,26 +16,9 @@ database_t *web_get_database(int index_id);
|
||||
|
||||
__always_inline
|
||||
static char *web_address_to_string(struct mg_addr *addr) {
|
||||
static char address_to_string_buf[64];
|
||||
static char address_to_string_buf[INET6_ADDRSTRLEN];
|
||||
|
||||
if (addr->is_ip6) {
|
||||
snprintf(address_to_string_buf, sizeof(address_to_string_buf),
|
||||
"%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x",
|
||||
addr->ip[0], addr->ip[1],
|
||||
addr->ip[2], addr->ip[3],
|
||||
addr->ip[4], addr->ip[5],
|
||||
addr->ip[6], addr->ip[7],
|
||||
addr->ip[8], addr->ip[9],
|
||||
addr->ip[10], addr->ip[11],
|
||||
addr->ip[12], addr->ip[13],
|
||||
addr->ip[14], addr->ip[15]);
|
||||
} else {
|
||||
snprintf(address_to_string_buf, sizeof(address_to_string_buf),
|
||||
"%d.%d.%d.%d",
|
||||
addr->ip[0], addr->ip[1], addr->ip[2], addr->ip[3]);
|
||||
}
|
||||
|
||||
return address_to_string_buf;
|
||||
return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
|
||||
}
|
||||
|
||||
void web_send_headers(struct mg_connection *nc, int status_code, size_t length, char *extra_headers);
|
||||
|
||||
26
third-party/libscan/CMakeLists.txt
vendored
26
third-party/libscan/CMakeLists.txt
vendored
@ -106,33 +106,12 @@ find_library(MUPDF_LIB NAMES liblibmupdf.a)
|
||||
find_library(CMS_LIB NAMES lcms2)
|
||||
find_library(JAS_LIB NAMES jasper)
|
||||
find_library(GUMBO_LIB NAMES gumbo)
|
||||
find_library(GOMP_LIB NAMES libgomp.a gomp
|
||||
PATHS
|
||||
/usr/lib/gcc/x86_64-linux-gnu/5/
|
||||
/usr/lib/gcc/x86_64-linux-gnu/6/
|
||||
/usr/lib/gcc/x86_64-linux-gnu/7/
|
||||
/usr/lib/gcc/x86_64-linux-gnu/8/
|
||||
/usr/lib/gcc/x86_64-linux-gnu/9/
|
||||
/usr/lib/gcc/x86_64-linux-gnu/10/
|
||||
/usr/lib/gcc/x86_64-linux-gnu/11/
|
||||
/usr/lib/gcc/x86_64-linux-gnu/12/
|
||||
|
||||
/usr/lib/gcc/aarch64-linux-gnu/5/
|
||||
/usr/lib/gcc/aarch64-linux-gnu/6/
|
||||
/usr/lib/gcc/aarch64-linux-gnu/7/
|
||||
/usr/lib/gcc/aarch64-linux-gnu/8/
|
||||
/usr/lib/gcc/aarch64-linux-gnu/9/
|
||||
/usr/lib/gcc/aarch64-linux-gnu/10/
|
||||
/usr/lib/gcc/aarch64-linux-gnu/11/
|
||||
/usr/lib/gcc/aarch64-linux-gnu/12/
|
||||
)
|
||||
find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/8/ /usr/lib/gcc/aarch64-linux-gnu/8/)
|
||||
find_package(Leptonica CONFIG REQUIRED)
|
||||
find_package(FFMPEG REQUIRED)
|
||||
find_package(libraw CONFIG REQUIRED)
|
||||
find_package(Freetype REQUIRED)
|
||||
|
||||
find_package(FFMPEG REQUIRED)
|
||||
list(REMOVE_ITEM FFMPEG_LIBRARIES /usr/lib/x86_64-linux-gnu/libm.a)
|
||||
list(REMOVE_ITEM FFMPEG_LIBRARIES /usr/lib/aarch64-linux-gnu/libm.a)
|
||||
|
||||
target_compile_options(
|
||||
scan
|
||||
@ -187,6 +166,7 @@ target_link_libraries(
|
||||
${WPD_LIB_DIR}/libwpd-0.9.a
|
||||
${WPD_LIB_DIR}/libwpd-stream-0.9.a
|
||||
|
||||
${FREETYPE_LIB}
|
||||
${HARFBUZZ_LIB}
|
||||
${JBIG2DEC_LIB}
|
||||
|
||||
|
||||
14
third-party/libscan/libscan/ebook/ebook.c
vendored
14
third-party/libscan/libscan/ebook/ebook.c
vendored
@ -175,19 +175,9 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
#define IS_IGNORED_MESSAGE(message) \
|
||||
( \
|
||||
strstr(message, "invalid glyph index") \
|
||||
|| strstr(message, "... repeated") \
|
||||
) \
|
||||
|
||||
void fz_err_callback(void *user, const char *message) {
|
||||
document_t *doc = (document_t *) user;
|
||||
|
||||
if (IS_IGNORED_MESSAGE(message)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const scan_ebook_ctx_t *ctx = &thread_ctx;
|
||||
CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message);
|
||||
}
|
||||
@ -195,10 +185,6 @@ void fz_err_callback(void *user, const char *message) {
|
||||
void fz_warn_callback(void *user, const char *message) {
|
||||
document_t *doc = (document_t *) user;
|
||||
|
||||
if (IS_IGNORED_MESSAGE(message)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const scan_ebook_ctx_t *ctx = &thread_ctx;
|
||||
CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message);
|
||||
}
|
||||
|
||||
18
third-party/libscan/libscan/media/media.c
vendored
18
third-party/libscan/libscan/media/media.c
vendored
@ -223,10 +223,14 @@ read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *d
|
||||
|
||||
void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDictionaryEntry *tag, enum metakey key) {
|
||||
|
||||
if (meta_contains_key(doc->meta_head, key)) {
|
||||
CTX_LOG_DEBUGF(doc->filepath, "Ignoring duplicate tag: '%02x=%s'",
|
||||
key, tag->value);
|
||||
return;
|
||||
meta_line_t *meta = doc->meta_head;
|
||||
while (meta != NULL) {
|
||||
if (meta->key == key) {
|
||||
CTX_LOG_DEBUGF(doc->filepath, "Ignoring duplicate tag: '%02x=%s' and '%02x=%s'",
|
||||
key, meta->str_val, key, tag->value);
|
||||
return;
|
||||
}
|
||||
meta = meta->next;
|
||||
}
|
||||
|
||||
text_buffer_t tex = text_buffer_create(-1);
|
||||
@ -268,7 +272,6 @@ static void append_audio_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx
|
||||
APPEND_TAG_META(MetaAlbum);
|
||||
} else if (strcmp(key, "comment") == 0) {
|
||||
append_tag_meta_if_not_exists(ctx, doc, tag, MetaContent);
|
||||
APPEND_TAG_META(MetaMediaComment);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -441,7 +444,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
|
||||
return SAVE_THUMBNAIL_FAILED;
|
||||
}
|
||||
|
||||
if (ctx->tesseract_lang != NULL && thumbnail_index == 0 && !meta_contains_key(doc->meta_head, MetaContent)) {
|
||||
if (ctx->tesseract_lang != NULL && thumbnail_index == 0) {
|
||||
ocr_image(ctx, doc, decoder, frame_and_packet->frame);
|
||||
}
|
||||
|
||||
@ -565,9 +568,6 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
|
||||
AVStream *stream = pFormatCtx->streams[video_stream];
|
||||
|
||||
if (stream->codecpar->width <= MIN_SIZE || stream->codecpar->height <= MIN_SIZE) {
|
||||
CTX_LOG_DEBUGF(doc->filepath,
|
||||
"Will not generate thumbnail because image is too small: %dx%d",
|
||||
stream->codecpar->width, stream->codecpar->width);
|
||||
avformat_close_input(&pFormatCtx);
|
||||
avformat_free_context(pFormatCtx);
|
||||
return;
|
||||
|
||||
12
third-party/libscan/libscan/ocr/ocr.h
vendored
12
third-party/libscan/libscan/ocr/ocr.h
vendored
@ -5,8 +5,8 @@
|
||||
#include <tesseract/capi.h>
|
||||
|
||||
#define MIN_OCR_WIDTH 350
|
||||
#define MIN_OCR_HEIGHT 33
|
||||
#define MIN_OCR_LEN 3
|
||||
#define MIN_OCR_HEIGHT 100
|
||||
#define MIN_OCR_LEN 10
|
||||
|
||||
#define OCR_IS_VALID_BPP(d) \
|
||||
((d) == 1 || (d) == 2 || (d) == 4 || (d) == 8 || (d) == 16 || (d) == 24 || \
|
||||
@ -28,13 +28,7 @@ ocr_extract_text(const char *tesseract_path, const char *tesseract_lang,
|
||||
TessBaseAPI *api = TessBaseAPICreate();
|
||||
TessBaseAPIInit3(api, tesseract_path, tesseract_lang);
|
||||
|
||||
// https://github.com/simon987/sist2/issues/443
|
||||
if (strstr(tesseract_lang, "chi") != NULL) {
|
||||
TessBaseAPISetVariable(api, "preserve_interword_spaces", "1");
|
||||
}
|
||||
|
||||
// TODO: add this as param?
|
||||
// TessBaseAPISetPageSegMode(api, PSM_AUTO_OSD);
|
||||
TessBaseAPISetPageSegMode(api, PSM_AUTO_OSD);
|
||||
|
||||
TessBaseAPISetImage(api, img_buf, img_w, img_h, img_bpp, img_stride);
|
||||
TessBaseAPISetSourceResolution(api, img_xres);
|
||||
|
||||
3
third-party/libscan/libscan/scan.h
vendored
3
third-party/libscan/libscan/scan.h
vendored
@ -63,7 +63,6 @@ enum metakey {
|
||||
MetaAlbumArtist,
|
||||
MetaGenre,
|
||||
MetaTitle,
|
||||
MetaMediaComment,
|
||||
MetaFontName,
|
||||
MetaExifMake,
|
||||
MetaExifDescription,
|
||||
@ -172,8 +171,6 @@ typedef struct {
|
||||
char filepath[PATH_MAX * 2 + 1];
|
||||
} parse_job_t;
|
||||
|
||||
#define IS_SUB_JOB(job) ((job)->parent[0] != '\0')
|
||||
|
||||
|
||||
#include "util.h"
|
||||
|
||||
|
||||
14
third-party/libscan/libscan/util.h
vendored
14
third-party/libscan/libscan/util.h
vendored
@ -392,18 +392,4 @@ static parse_job_t *create_parse_job(const char *filepath, int mtime, size_t st_
|
||||
return job;
|
||||
}
|
||||
|
||||
|
||||
static int meta_contains_key (meta_line_t *meta_head, enum metakey key) {
|
||||
|
||||
meta_line_t *meta = meta_head;
|
||||
while (meta != NULL) {
|
||||
if (meta->key == key) {
|
||||
return TRUE;
|
||||
}
|
||||
meta = meta->next;
|
||||
}
|
||||
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user