simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-03 07:22:59 +00:00
Code Issues Projects Releases Wiki Activity

refactor index schema, remove sidecar parsing, remove TS

Browse Source
This commit is contained in:
simon987 2023-09-05 18:59:18 -04:00
parent b81ccebdb1
commit 8fdb832c85
84 changed files with 1420 additions and 2445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.devcontainer/Dockerfile
View File

@ -1,9 +0,0 @@
FROM simon987/sist2-build
RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
ENV DEBIAN_FRONTEND=noninteractive
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8

16
.devcontainer/devcontainer.json
View File

@ -1,16 +0,0 @@
{
"name": "sist2-dev",
"dockerComposeFile": [
"docker-compose.yml"
],
"service": "sist2-dev",
"customizations": {
"vscode": {
"extensions": [
"ms-vscode.cpptools-extension-pack"
]
}
},
"remoteUser": "root",
"workspaceFolder": "/app/"
}

8
.devcontainer/docker-compose.yml
View File

@ -1,8 +0,0 @@
version: "3"
services:
sist2-dev:
build: .
command: sleep infinity
volumes:
- ../:/app

4
.dockerignore
View File

@ -38,4 +38,6 @@ build/
__pycache__/
sist2-vue/dist
sist2-admin/frontend/dist
*.fts
*.fts
.git
third-party/libscan/third-party/ext_*/*

1
CMakeLists.txt
View File

@ -53,7 +53,6 @@ add_executable(
src/types.h
src/log.c src/log.h
src/cli.c src/cli.h
src/parsing/sidecar.c src/parsing/sidecar.h
src/database/database.c src/database/database.h
src/parsing/fs_util.h

12
docs/USAGE.md
View File

@ -16,9 +16,9 @@ Lightning-fast file system indexer and search tool.
Scan options
-t, --threads=<int> Number of threads. DEFAULT: 1
-q, --thumbnail-quality=<int> Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
--thumbnail-size=<int> Thumbnail size, in pixels. DEFAULT: 552
--thumbnail-count=<int> Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
-q, --thumbnail_count-quality=<int> Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
--thumbnail_count-size=<int> Thumbnail size, in pixels. DEFAULT: 552
--thumbnail_count-count=<int> Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
--content-size=<int> Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
-o, --output=<str> Output index file path. DEFAULT: index.sist2
--incremental If the output file path exists, only scan new or modified files.
@ -78,9 +78,9 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
#### Thumbnail database size estimation
See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
See chart below for rough estimate of thumbnail_count size vs. thumbnail_count size & quality arguments:
For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database
For example, `--thumbnail_count-size=500`, `--thumbnail_count-quality=50` for a directory with 8 million images will create a thumbnail_count database
that is about `8000000 * 11.8kB = 94.4GB`.
![thumbnail_size](thumbnail_size.png)
@ -92,7 +92,7 @@ Simple scan
sist2 scan ~/Documents
sist2 scan \
--threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
--threads 4 --content-size 16000000 --thumbnail_count-quality 2 --archive shallow \
--name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
~/Documents -o ./documents.sist2
```

131
scripts/generate_big_index.py
View File

@ -1,131 +0,0 @@
import sqlite3
import orjson as json
import os
import string
from hashlib import md5
import random
from tqdm import tqdm
schema = """
CREATE TABLE thumbnail (
id TEXT NOT NULL CHECK (
length(id) = 32
),
num INTEGER NOT NULL,
data BLOB NOT NULL,
PRIMARY KEY(id, num)
) WITHOUT ROWID;
CREATE TABLE version (
id INTEGER PRIMARY KEY AUTOINCREMENT,
date TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP)
);
CREATE TABLE document (
id TEXT PRIMARY KEY NOT NULL CHECK (
length(id) = 32
),
marked INTEGER NOT NULL DEFAULT (1),
version INTEGER NOT NULL REFERENCES version(id),
mtime INTEGER NOT NULL,
size INTEGER NOT NULL,
json_data TEXT NOT NULL CHECK (
json_valid(json_data)
)
);
CREATE TABLE delete_list (
id TEXT PRIMARY KEY CHECK (
length(id) = 32
)
) WITHOUT ROWID;
CREATE TABLE tag (
id TEXT NOT NULL,
tag TEXT NOT NULL,
PRIMARY KEY (id, tag)
);
CREATE TABLE document_sidecar (
id TEXT PRIMARY KEY NOT NULL, json_data TEXT NOT NULL
) WITHOUT ROWID;
CREATE TABLE descriptor (
id TEXT NOT NULL, version_major INTEGER NOT NULL,
version_minor INTEGER NOT NULL, version_patch INTEGER NOT NULL,
root TEXT NOT NULL, name TEXT NOT NULL,
rewrite_url TEXT, timestamp INTEGER NOT NULL
);
CREATE TABLE stats_treemap (
path TEXT NOT NULL, size INTEGER NOT NULL
);
CREATE TABLE stats_size_agg (
bucket INTEGER NOT NULL, count INTEGER NOT NULL
);
CREATE TABLE stats_date_agg (
bucket INTEGER NOT NULL, count INTEGER NOT NULL
);
CREATE TABLE stats_mime_agg (
mime TEXT NOT NULL, size INTEGER NOT NULL,
count INTEGER NOT NULL
);
CREATE TABLE embedding (
id TEXT REFERENCES document(id),
model_id INTEGER NOT NULL references model(id),
start INTEGER NOT NULL,
end INTEGER,
embedding BLOB NOT NULL,
PRIMARY KEY (id, model_id, start)
);
CREATE TABLE model (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE CHECK (
length(name) < 16
),
url TEXT,
path TEXT NOT NULL UNIQUE,
size INTEGER NOT NULL,
type TEXT NOT NULL CHECK (
type IN ('flat', 'nested')
)
);
"""
content = "".join(random.choices(string.ascii_letters, k=500))
def gen_document():
return [
md5(random.randbytes(8)).hexdigest(),
json.dumps({
"content": content,
"mime": "image/jpeg",
"extension": "jpeg",
"name": "test",
"path": "",
})
]
if __name__ == "__main__":
DB_NAME = "big_index.sist2"
SIZE = 30_000_000
os.remove(DB_NAME)
db = sqlite3.connect(DB_NAME)
db.executescript(schema)
db.executescript("""
PRAGMA journal_mode = OFF;
PRAGMA synchronous = 0;
""")
for _ in tqdm(range(SIZE), total=SIZE):
db.execute(
"INSERT INTO document (id, version, mtime, size, json_data) VALUES (?, 1, 1000000, 10000, ?)",
gen_document()
)
# 1. Enable rowid from document
# 2. CREATE TABLE marked (
# id INTEGER PRIMARY KEY,
# marked int
# );
# 3. Set FK for document_sidecar, embedding, tag, thumbnail
# 4. Toggle FK if debug
db.commit()

3
scripts/mime.csv
View File

@ -449,5 +449,4 @@ image/x-sigma-x3f, xf3
image/x-sony-arw, arw
image/x-sony-sr2, sr2
image/x-sony-srf, srf
image/x-epson-erf, erf
sist2/sidecar, s2meta
image/x-epson-erf, erf
1 application/x-matlab-data mat
449 image/x-sony-arw arw
450 image/x-sony-sr2 sr2
451 image/x-sony-srf srf
452 image/x-epson-erf erf
sist2/sidecar s2meta

16
scripts/mime.py
View File

@ -3,6 +3,7 @@ import zlib
mimes = {}
noparse = set()
ext_in_hash = set()
mime_ids = {}
major_mime = {
"sist2": 0,
@ -102,6 +103,9 @@ cnt = 1
def mime_id(mime):
if mime in mime_ids:
return mime_ids[mime]
global cnt
major = mime.split("/")[0]
mime_id = str((major_mime[major] << 16) + cnt)
@ -127,9 +131,7 @@ def mime_id(mime):
elif mime == "application/x-empty":
cnt -= 1
return "1"
elif mime == "sist2/sidecar":
cnt -= 1
return "2"
mime_ids[mime] = mime_id
return mime_id
@ -197,4 +199,12 @@ with open("scripts/mime.csv") as f:
print(f"case {crc(mime)}: return {clean(mime)};")
print("default: return 0;}}")
# mime list
mime_list = ",".join(mime_id(x) for x in mimes.keys()) + ",0"
print(f"unsigned int mime_ids[] = {{{mime_list}}};")
print("unsigned int* get_mime_ids() { return mime_ids; }")
print("#endif")

5
sist2-admin/sist2_admin/app.py
View File

@ -222,7 +222,10 @@ async def delete_job(name: str):
@app.delete("/api/frontend/{name:str}")
async def delete_frontend(name: str):
if name in RUNNING_FRONTENDS:
os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
try:
os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
except ProcessLookupError:
pass
del RUNNING_FRONTENDS[name]
frontend = db["frontends"][name]

769
sist2-vue/package-lock.json generated
View File

@ -33,7 +33,6 @@
"@types/underscore": "^1.11.6",
"@vue/cli-plugin-babel": "~5.0.8",
"@vue/cli-plugin-router": "~5.0.8",
"@vue/cli-plugin-typescript": "^5.0.8",
"@vue/cli-plugin-vuex": "~5.0.8",
"@vue/cli-service": "^5.0.8",
"@vue/test-utils": "^1.0.3",
@ -45,7 +44,6 @@
"portal-vue": "^2.1.7",
"sass": "^1.26.11",
"sass-loader": "^10.0.2",
"typescript": "^4.9.5",
"vue-cli-plugin-bootstrap-vue": "~0.8.2",
"vue-template-compiler": "^2.6.11"
}
@ -2332,12 +2330,6 @@
"integrity": "sha512-G2oC64I/sR817KDL2b2Mc7+diXyxcibyUeLMyexU4K/sG8hyt/YMlbBK0TVhx/YQ1ehfzgXhLuq2YQHIL4bXUQ==",
"dev": true
},
"node_modules/@types/webpack-env": {
"version": "1.16.0",
"resolved": "https://registry.npmjs.org/@types/webpack-env/-/webpack-env-1.16.0.tgz",
"integrity": "sha512-Fx+NpfOO0CpeYX2g9bkvX8O5qh9wrU1sOF4g8sft4Mu7z+qfe387YlyY8w8daDyDsKY5vUxM0yxkAYnbkRbZEw==",
"dev": true
},
"node_modules/@types/ws": {
"version": "8.5.4",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
@ -2638,218 +2630,6 @@
"@vue/cli-service": "^3.0.0 || ^4.0.0 || ^5.0.0-0"
}
},
"node_modules/@vue/cli-plugin-typescript": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/@vue/cli-plugin-typescript/-/cli-plugin-typescript-5.0.8.tgz",
"integrity": "sha512-JKJOwzJshBqsmp4yLBexwVMebOZ4VGJgbnYvmHVxasJOStF2RxwyW28ZF+zIvASGdat4sAUuo/3mAQyVhm7JHg==",
"dev": true,
"dependencies": {
"@babel/core": "^7.12.16",
"@types/webpack-env": "^1.15.2",
"@vue/cli-shared-utils": "^5.0.8",
"babel-loader": "^8.2.2",
"fork-ts-checker-webpack-plugin": "^6.4.0",
"globby": "^11.0.2",
"thread-loader": "^3.0.0",
"ts-loader": "^9.2.5",
"webpack": "^5.54.0"
},
"peerDependencies": {
"@vue/cli-service": "^3.0.0 || ^4.0.0 || ^5.0.0-0",
"cache-loader": "^4.1.0",
"typescript": ">=2",
"vue": "^2 || ^3.2.13",
"vue-template-compiler": "^2.0.0"
},
"peerDependenciesMeta": {
"cache-loader": {
"optional": true
},
"vue-template-compiler": {
"optional": true
}
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dev": true,
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/braces": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
"dev": true,
"dependencies": {
"fill-range": "^7.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dev": true,
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
},
"node_modules/@vue/cli-plugin-typescript/node_modules/fill-range": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
"dev": true,
"dependencies": {
"to-regex-range": "^5.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true,
"engines": {
"node": ">=8"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/is-number": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true,
"engines": {
"node": ">=0.12.0"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"dependencies": {
"yallist": "^4.0.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/micromatch": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
"integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
"dev": true,
"dependencies": {
"braces": "^3.0.2",
"picomatch": "^2.3.1"
},
"engines": {
"node": ">=8.6"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/semver": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"dependencies": {
"lru-cache": "^6.0.0"
},
"bin": {
"semver": "bin/semver.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"dependencies": {
"has-flag": "^4.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
"dev": true,
"dependencies": {
"is-number": "^7.0.0"
},
"engines": {
"node": ">=8.0"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/ts-loader": {
"version": "9.4.2",
"resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.4.2.tgz",
"integrity": "sha512-OmlC4WVmFv5I0PpaxYb+qGeGOdm5giHU7HwDDUjw59emP2UYMHy9fFSDcYgSNoH8sXcj4hGCSEhlDZ9ULeDraA==",
"dev": true,
"dependencies": {
"chalk": "^4.1.0",
"enhanced-resolve": "^5.0.0",
"micromatch": "^4.0.0",
"semver": "^7.3.4"
},
"engines": {
"node": ">=12.0.0"
},
"peerDependencies": {
"typescript": "*",
"webpack": "^5.0.0"
}
},
"node_modules/@vue/cli-plugin-typescript/node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true
},
"node_modules/@vue/cli-plugin-vuex": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/@vue/cli-plugin-vuex/-/cli-plugin-vuex-5.0.8.tgz",
@ -4279,14 +4059,6 @@
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
},
"node_modules/buffer-json": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/buffer-json/-/buffer-json-2.0.0.tgz",
"integrity": "sha512-+jjPFVqyfF1esi9fvfUs3NqM0pH1ziZ36VP4hmA/y/Ssfo/5w5xHKfTw9BwQjoJ1w/oVtpLomqwUHKdefGyuHw==",
"dev": true,
"optional": true,
"peer": true
},
"node_modules/bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
@ -4296,25 +4068,6 @@
"node": ">= 0.8"
}
},
"node_modules/cache-loader": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/cache-loader/-/cache-loader-4.1.0.tgz",
"integrity": "sha512-ftOayxve0PwKzBF/GLsZNC9fJBXl8lkZE3TOsjkboHfVHVkL39iUEs1FO07A33mizmci5Dudt38UZrrYXDtbhw==",
"dev": true,
"optional": true,
"peer": true,
"dependencies": {
"buffer-json": "^2.0.0",
"find-cache-dir": "^3.0.0",
"loader-utils": "^1.2.3",
"mkdirp": "^0.5.1",
"neo-async": "^2.6.1",
"schema-utils": "^2.0.0"
},
"engines": {
"node": ">= 8.9.0"
}
},
"node_modules/call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
@ -4938,22 +4691,6 @@
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
},
"node_modules/cosmiconfig": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz",
"integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==",
"dev": true,
"dependencies": {
"@types/parse-json": "^4.0.0",
"import-fresh": "^3.1.0",
"parse-json": "^5.0.0",
"path-type": "^4.0.0",
"yaml": "^1.7.2"
},
"engines": {
"node": ">=8"
}
},
"node_modules/cross-spawn": {
"version": "6.0.5",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
@ -5442,15 +5179,6 @@
"node": ">=6.0"
}
},
"node_modules/deepmerge": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
"integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/default-gateway": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz",
@ -6392,166 +6120,6 @@
}
}
},
"node_modules/fork-ts-checker-webpack-plugin": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-6.5.2.tgz",
"integrity": "sha512-m5cUmF30xkZ7h4tWUgTAcEaKmUW7tfyUyTqNNOz7OxWJ0v1VWKTcOvH8FWHUwSjlW/356Ijc9vi3XfcPstpQKA==",
"dev": true,
"dependencies": {
"@babel/code-frame": "^7.8.3",
"@types/json-schema": "^7.0.5",
"chalk": "^4.1.0",
"chokidar": "^3.4.2",
"cosmiconfig": "^6.0.0",
"deepmerge": "^4.2.2",
"fs-extra": "^9.0.0",
"glob": "^7.1.6",
"memfs": "^3.1.2",
"minimatch": "^3.0.4",
"schema-utils": "2.7.0",
"semver": "^7.3.2",
"tapable": "^1.0.0"
},
"engines": {
"node": ">=10",
"yarn": ">=1.0.0"
},
"peerDependencies": {
"eslint": ">= 6",
"typescript": ">= 2.7",
"vue-template-compiler": "*",
"webpack": ">= 4"
},
"peerDependenciesMeta": {
"eslint": {
"optional": true
},
"vue-template-compiler": {
"optional": true
}
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dev": true,
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dev": true,
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true,
"engines": {
"node": ">=8"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"dependencies": {
"yallist": "^4.0.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/schema-utils": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz",
"integrity": "sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A==",
"dev": true,
"dependencies": {
"@types/json-schema": "^7.0.4",
"ajv": "^6.12.2",
"ajv-keywords": "^3.4.1"
},
"engines": {
"node": ">= 8.9.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/webpack"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/semver": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"dependencies": {
"lru-cache": "^6.0.0"
},
"bin": {
"semver": "bin/semver.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"dependencies": {
"has-flag": "^4.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/fork-ts-checker-webpack-plugin/node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -10503,15 +10071,6 @@
"node": ">= 10"
}
},
"node_modules/tapable": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz",
"integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==",
"dev": true,
"engines": {
"node": ">=6"
}
},
"node_modules/terser": {
"version": "5.16.1",
"resolved": "https://registry.npmjs.org/terser/-/terser-5.16.1.tgz",
@ -10732,19 +10291,6 @@
"node": ">= 0.6"
}
},
"node_modules/typescript": {
"version": "4.9.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=4.2.0"
}
},
"node_modules/underscore": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz",
@ -13492,12 +13038,6 @@
"integrity": "sha512-G2oC64I/sR817KDL2b2Mc7+diXyxcibyUeLMyexU4K/sG8hyt/YMlbBK0TVhx/YQ1ehfzgXhLuq2YQHIL4bXUQ==",
"dev": true
},
"@types/webpack-env": {
"version": "1.16.0",
"resolved": "https://registry.npmjs.org/@types/webpack-env/-/webpack-env-1.16.0.tgz",
"integrity": "sha512-Fx+NpfOO0CpeYX2g9bkvX8O5qh9wrU1sOF4g8sft4Mu7z+qfe387YlyY8w8daDyDsKY5vUxM0yxkAYnbkRbZEw==",
"dev": true
},
"@types/ws": {
"version": "8.5.4",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
@ -13736,153 +13276,6 @@
"@vue/cli-shared-utils": "^5.0.8"
}
},
"@vue/cli-plugin-typescript": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/@vue/cli-plugin-typescript/-/cli-plugin-typescript-5.0.8.tgz",
"integrity": "sha512-JKJOwzJshBqsmp4yLBexwVMebOZ4VGJgbnYvmHVxasJOStF2RxwyW28ZF+zIvASGdat4sAUuo/3mAQyVhm7JHg==",
"dev": true,
"requires": {
"@babel/core": "^7.12.16",
"@types/webpack-env": "^1.15.2",
"@vue/cli-shared-utils": "^5.0.8",
"babel-loader": "^8.2.2",
"fork-ts-checker-webpack-plugin": "^6.4.0",
"globby": "^11.0.2",
"thread-loader": "^3.0.0",
"ts-loader": "^9.2.5",
"webpack": "^5.54.0"
},
"dependencies": {
"ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dev": true,
"requires": {
"color-convert": "^2.0.1"
}
},
"braces": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
"dev": true,
"requires": {
"fill-range": "^7.0.1"
}
},
"chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dev": true,
"requires": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
},
"fill-range": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
"dev": true,
"requires": {
"to-regex-range": "^5.0.1"
}
},
"has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true
},
"is-number": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true
},
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"requires": {
"yallist": "^4.0.0"
}
},
"micromatch": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
"integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
"dev": true,
"requires": {
"braces": "^3.0.2",
"picomatch": "^2.3.1"
}
},
"semver": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
}
},
"supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"requires": {
"has-flag": "^4.0.0"
}
},
"to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
"dev": true,
"requires": {
"is-number": "^7.0.0"
}
},
"ts-loader": {
"version": "9.4.2",
"resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.4.2.tgz",
"integrity": "sha512-OmlC4WVmFv5I0PpaxYb+qGeGOdm5giHU7HwDDUjw59emP2UYMHy9fFSDcYgSNoH8sXcj4hGCSEhlDZ9ULeDraA==",
"dev": true,
"requires": {
"chalk": "^4.1.0",
"enhanced-resolve": "^5.0.0",
"micromatch": "^4.0.0",
"semver": "^7.3.4"
}
},
"yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true
}
}
},
"@vue/cli-plugin-vuex": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/@vue/cli-plugin-vuex/-/cli-plugin-vuex-5.0.8.tgz",
@ -14956,36 +14349,12 @@
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
},
"buffer-json": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/buffer-json/-/buffer-json-2.0.0.tgz",
"integrity": "sha512-+jjPFVqyfF1esi9fvfUs3NqM0pH1ziZ36VP4hmA/y/Ssfo/5w5xHKfTw9BwQjoJ1w/oVtpLomqwUHKdefGyuHw==",
"dev": true,
"optional": true,
"peer": true
},
"bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==",
"dev": true
},
"cache-loader": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/cache-loader/-/cache-loader-4.1.0.tgz",
"integrity": "sha512-ftOayxve0PwKzBF/GLsZNC9fJBXl8lkZE3TOsjkboHfVHVkL39iUEs1FO07A33mizmci5Dudt38UZrrYXDtbhw==",
"dev": true,
"optional": true,
"peer": true,
"requires": {
"buffer-json": "^2.0.0",
"find-cache-dir": "^3.0.0",
"loader-utils": "^1.2.3",
"mkdirp": "^0.5.1",
"neo-async": "^2.6.1",
"schema-utils": "^2.0.0"
}
},
"call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
@ -15467,19 +14836,6 @@
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
},
"cosmiconfig": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz",
"integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==",
"dev": true,
"requires": {
"@types/parse-json": "^4.0.0",
"import-fresh": "^3.1.0",
"parse-json": "^5.0.0",
"path-type": "^4.0.0",
"yaml": "^1.7.2"
}
},
"cross-spawn": {
"version": "6.0.5",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
@ -15899,12 +15255,6 @@
"ms": "2.1.2"
}
},
"deepmerge": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
"integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
"dev": true
},
"default-gateway": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz",
@ -16636,113 +15986,6 @@
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
},
"fork-ts-checker-webpack-plugin": {
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-6.5.2.tgz",
"integrity": "sha512-m5cUmF30xkZ7h4tWUgTAcEaKmUW7tfyUyTqNNOz7OxWJ0v1VWKTcOvH8FWHUwSjlW/356Ijc9vi3XfcPstpQKA==",
"dev": true,
"requires": {
"@babel/code-frame": "^7.8.3",
"@types/json-schema": "^7.0.5",
"chalk": "^4.1.0",
"chokidar": "^3.4.2",
"cosmiconfig": "^6.0.0",
"deepmerge": "^4.2.2",
"fs-extra": "^9.0.0",
"glob": "^7.1.6",
"memfs": "^3.1.2",
"minimatch": "^3.0.4",
"schema-utils": "2.7.0",
"semver": "^7.3.2",
"tapable": "^1.0.0"
},
"dependencies": {
"ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dev": true,
"requires": {
"color-convert": "^2.0.1"
}
},
"chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dev": true,
"requires": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
},
"has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true
},
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"requires": {
"yallist": "^4.0.0"
}
},
"schema-utils": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz",
"integrity": "sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A==",
"dev": true,
"requires": {
"@types/json-schema": "^7.0.4",
"ajv": "^6.12.2",
"ajv-keywords": "^3.4.1"
}
},
"semver": {
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
}
},
"supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"requires": {
"has-flag": "^4.0.0"
}
},
"yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true
}
}
},
"forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -19757,12 +19000,6 @@
}
}
},
"tapable": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz",
"integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==",
"dev": true
},
"terser": {
"version": "5.16.1",
"resolved": "https://registry.npmjs.org/terser/-/terser-5.16.1.tgz",
@ -19912,12 +19149,6 @@
"mime-types": "~2.1.24"
}
},
"typescript": {
"version": "4.9.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"dev": true
},
"underscore": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz",

2
sist2-vue/package.json
View File

@ -32,7 +32,6 @@
"@types/underscore": "^1.11.6",
"@vue/cli-plugin-babel": "~5.0.8",
"@vue/cli-plugin-router": "~5.0.8",
"@vue/cli-plugin-typescript": "^5.0.8",
"@vue/cli-plugin-vuex": "~5.0.8",
"@vue/cli-service": "^5.0.8",
"@vue/test-utils": "^1.0.3",
@ -44,7 +43,6 @@
"portal-vue": "^2.1.7",
"sass": "^1.26.11",
"sass-loader": "^10.0.2",
"typescript": "^4.9.5",
"vue-cli-plugin-bootstrap-vue": "~0.8.2",
"vue-template-compiler": "^2.6.11"
},

243
sist2-vue/src/Sist2Api.ts → sist2-vue/src/Sist2Api.js
View File

@ -1,116 +1,20 @@
import axios from "axios";
import {ext, strUnescape, lum} from "./util";
import {strUnescape, lum, sid} from "./util";
import Sist2Query from "@/Sist2ElasticsearchQuery";
import store from "@/store";
export interface EsTag {
id: string
count: number
color: string | undefined
isLeaf: boolean
}
export interface Tag {
style: string
text: string
rawText: string
fg: string
bg: string
userTag: boolean
}
export interface Index {
name: string
version: string
id: string
idPrefix: string
timestamp: number
models: []
}
export interface EsHit {
_index: string
_id: string
_score: number
_type: string
_tags: Tag[]
_seq: number
_source: {
path: string
size: number
mime: string
name: string
extension: string
index: string
_depth: number
mtime: number
videoc: string
audioc: string
parent: string
width: number
height: number
duration: number
tag: string[]
checksum: string
thumbnail: string
}
_props: {
isSubDocument: boolean
isImage: boolean
isGif: boolean
isVideo: boolean
isPlayableVideo: boolean
isPlayableImage: boolean
isAudio: boolean
hasThumbnail: boolean
hasVidPreview: boolean
imageAspectRatio: number
/** Number of thumbnails available */
tnNum: number
}
highlight: {
name: string[] | undefined,
content: string[] | undefined,
}
}
function getIdPrefix(indices: Index[], id: string): string {
for (let i = 4; i < 32; i++) {
const prefix = id.slice(0, i);
if (indices.filter(idx => idx.id.slice(0, i) == prefix).length == 1) {
return prefix;
}
}
return id;
}
export interface EsResult {
took: number
hits: {
// TODO: ES 6.X ?
total: {
value: number
}
hits: EsHit[]
}
aggregations: any
}
class Sist2Api {
private readonly baseUrl: string
private sist2Info: any
private queryfunc: () => EsResult;
baseUrl;
sist2Info;
queryfunc;
constructor(baseUrl: string) {
constructor(baseUrl) {
this.baseUrl = baseUrl;
}
init(queryFunc: () => EsResult) {
init(queryFunc) {
this.queryfunc = queryFunc;
}
@ -127,9 +31,9 @@ class Sist2Api {
.filter((v, i, a) => a.findIndex(v2 => (v2.id === v.id)) === i)
}
getSist2Info(): Promise<any> {
getSist2Info() {
return axios.get(`${this.baseUrl}i`).then(resp => {
const indices = resp.data.indices as Index[];
const indices = resp.data.indices;
resp.data.indices = indices.map(idx => {
return {
@ -138,8 +42,7 @@ class Sist2Api {
timestamp: idx.timestamp,
version: idx.version,
models: idx.models,
idPrefix: getIdPrefix(indices, idx.id),
} as Index;
};
});
this.sist2Info = resp.data;
@ -148,8 +51,8 @@ class Sist2Api {
})
}
setHitProps(hit: EsHit): void {
hit["_props"] = {} as any;
setHitProps(hit) {
hit["_props"] = {};
const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
@ -157,7 +60,7 @@ class Sist2Api {
hit._props.isSubDocument = true;
}
if ("thumbnail" in hit._source) {
if ("thumbnail" in hit._source && hit._source.thumbnail > 0) {
hit._props.hasThumbnail = true;
if (Number.isNaN(Number(hit._source.thumbnail))) {
@ -213,8 +116,8 @@ class Sist2Api {
}
}
setHitTags(hit: EsHit): void {
const tags = [] as Tag[];
setHitTags(hit) {
const tags = [];
// User tags
if ("tag" in hit._source) {
@ -226,10 +129,10 @@ class Sist2Api {
hit._tags = tags;
}
createUserTag(tag: string): Tag {
createUserTag(tag) {
const tokens = tag.split(".");
const colorToken = tokens.pop() as string;
const colorToken = tokens.pop();
const bg = colorToken;
const fg = lum(colorToken) > 50 ? "#000" : "#fff";
@ -241,23 +144,23 @@ class Sist2Api {
text: tokens.join("."),
rawText: tag,
userTag: true,
} as Tag;
};
}
search(): Promise<EsResult> {
if (this.backend() == "sqlite") {
search() {
if (this.backend() === "sqlite") {
return this.ftsQuery(this.queryfunc())
} else {
return this.esQuery(this.queryfunc());
}
}
esQuery(query: any): Promise<EsResult> {
esQuery(query) {
return axios.post(`${this.baseUrl}es`, query).then(resp => {
const res = resp.data as EsResult;
const res = resp.data;
if (res.hits?.hits) {
res.hits.hits.forEach((hit: EsHit) => {
res.hits.hits.forEach((hit) => {
hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
@ -270,9 +173,9 @@ class Sist2Api {
});
}
ftsQuery(query: any): Promise<EsResult> {
ftsQuery(query) {
return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
const res = resp.data as any;
const res = resp.data;
if (res.hits.hits) {
res.hits.hits.forEach(hit => {
@ -293,7 +196,7 @@ class Sist2Api {
});
}
private getMimeTypesEs(query) {
getMimeTypesEs(query) {
const AGGS = {
mimeTypes: {
terms: {
@ -322,7 +225,7 @@ class Sist2Api {
});
}
private getMimeTypesSqlite(): Promise<[{ mime: string, count: number }]> {
getMimeTypesSqlite() {
return axios.get(`${this.baseUrl}fts/mimetypes`)
.then(resp => {
return resp.data;
@ -332,15 +235,15 @@ class Sist2Api {
async getMimeTypes(query = undefined) {
let buckets;
if (this.backend() == "sqlite") {
if (this.backend() === "sqlite") {
buckets = await this.getMimeTypesSqlite();
} else {
buckets = await this.getMimeTypesEs(query);
}
const mimeMap: any[] = [];
const mimeMap = [];
buckets.sort((a: any, b: any) => a.mime > b.mime).forEach((bucket: any) => {
buckets.sort((a, b) => a.mime > b.mime).forEach((bucket) => {
const tmp = bucket.mime.split("/");
const category = tmp[0];
const mime = tmp[1];
@ -374,7 +277,7 @@ class Sist2Api {
return {buckets, mimeMap};
}
_createEsTag(tag: string, count: number): EsTag {
_createEsTag(tag, count) {
const tokens = tag.split(".");
if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
@ -394,7 +297,7 @@ class Sist2Api {
};
}
private getTagsEs() {
getTagsEs() {
return this.esQuery({
aggs: {
tags: {
@ -407,21 +310,21 @@ class Sist2Api {
size: 0,
}).then(resp => {
return resp["aggregations"]["tags"]["buckets"]
.sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
.map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
.sort((a, b) => a["key"].localeCompare(b["key"]))
.map((bucket) => this._createEsTag(bucket["key"], bucket["doc_count"]));
});
}
private getTagsSqlite() {
getTagsSqlite() {
return axios.get(`${this.baseUrl}/fts/tags`)
.then(resp => {
return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
});
}
async getTags(): Promise<EsTag[]> {
async getTags() {
let tags;
if (this.backend() == "sqlite") {
if (this.backend() === "sqlite") {
tags = await this.getTagsSqlite();
} else {
tags = await this.getTagsEs();
@ -430,7 +333,7 @@ class Sist2Api {
// Remove duplicates (same tag with different color)
const seen = new Set();
return tags.filter((t: EsTag) => {
return tags.filter((t) => {
if (seen.has(t.id)) {
return false;
}
@ -439,31 +342,29 @@ class Sist2Api {
});
}
saveTag(tag: string, hit: EsHit) {
return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
saveTag(tag, hit) {
return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
delete: false,
name: tag,
doc_id: hit["_id"]
});
}
deleteTag(tag: string, hit: EsHit) {
return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
deleteTag(tag, hit) {
return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
delete: true,
name: tag,
doc_id: hit["_id"]
});
}
searchPaths(indexId, minDepth, maxDepth, prefix = null) {
if (this.backend() == "sqlite") {
if (this.backend() === "sqlite") {
return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
} else {
return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
}
}
private searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
return axios.post(`${this.baseUrl}fts/paths`, {
indexId, minDepth, maxDepth, prefix
}).then(resp => {
@ -471,7 +372,7 @@ class Sist2Api {
});
}
private searchPathsEs(indexId, minDepth, maxDepth, prefix): Promise<[{ path: string, count: number }]> {
searchPathsEs(indexId, minDepth, maxDepth, prefix) {
const query = {
query: {
@ -516,7 +417,7 @@ class Sist2Api {
});
}
private getDateRangeSqlite() {
getDateRangeSqlite() {
return axios.get(`${this.baseUrl}fts/dateRange`)
.then(resp => ({
min: resp.data.dateMin,
@ -524,15 +425,15 @@ class Sist2Api {
}));
}
getDateRange(): Promise<{ min: number, max: number }> {
if (this.backend() == "sqlite") {
getDateRange() {
if (this.backend() === "sqlite") {
return this.getDateRangeSqlite();
} else {
return this.getDateRangeEs();
}
}
private getDateRangeEs() {
getDateRangeEs() {
return this.esQuery({
// TODO: filter current selected indices
aggs: {
@ -549,7 +450,7 @@ class Sist2Api {
if (range.min == null) {
range.min = 0;
range.max = 1;
} else if (range.min == range.max) {
} else if (range.min === range.max) {
range.max += 1;
}
@ -557,7 +458,7 @@ class Sist2Api {
});
}
private getPathSuggestionsSqlite(text: string) {
getPathSuggestionsSqlite(text) {
return axios.post(`${this.baseUrl}fts/paths`, {
prefix: text,
minDepth: 1,
@ -567,7 +468,7 @@ class Sist2Api {
})
}
private getPathSuggestionsEs(text) {
getPathSuggestionsEs(text) {
return this.esQuery({
suggest: {
path: {
@ -585,31 +486,31 @@ class Sist2Api {
});
}
getPathSuggestions(text: string): Promise<string[]> {
if (this.backend() == "sqlite") {
getPathSuggestions(text) {
if (this.backend() === "sqlite") {
return this.getPathSuggestionsSqlite(text);
} else {
return this.getPathSuggestionsEs(text)
}
}
getTreemapStat(indexId: string) {
getTreemapStat(indexId) {
return `${this.baseUrl}s/${indexId}/TMAP`;
}
getMimeStat(indexId: string) {
getMimeStat(indexId) {
return `${this.baseUrl}s/${indexId}/MAGG`;
}
getSizeStat(indexId: string) {
getSizeStat(indexId) {
return `${this.baseUrl}s/${indexId}/SAGG`;
}
getDateStat(indexId: string) {
getDateStat(indexId) {
return `${this.baseUrl}s/${indexId}/DAGG`;
}
private getDocumentEs(docId: string, highlight: boolean, fuzzy: boolean) {
getDocumentEs(sid, highlight, fuzzy) {
const query = Sist2Query.searchQuery();
if (highlight) {
@ -648,7 +549,7 @@ class Sist2Api {
query.query.bool.must = [query.query.bool.must];
}
query.query.bool.must.push({match: {_id: docId}});
query.query.bool.must.push({match: {_id: sid}});
delete query["sort"];
delete query["aggs"];
@ -669,35 +570,35 @@ class Sist2Api {
});
}
private getDocumentSqlite(docId: string): Promise<EsHit> {
return axios.get(`${this.baseUrl}/fts/d/${docId}`)
getDocumentSqlite(sid) {
return axios.get(`${this.baseUrl}/fts/d/${sid}`)
.then(resp => ({
_source: resp.data
} as EsHit));
}));
}
getDocument(docId: string, highlight: boolean, fuzzy: boolean): Promise<EsHit | null> {
if (this.backend() == "sqlite") {
return this.getDocumentSqlite(docId);
getDocument(sid, highlight, fuzzy) {
if (this.backend() === "sqlite") {
return this.getDocumentSqlite(sid);
} else {
return this.getDocumentEs(docId, highlight, fuzzy);
return this.getDocumentEs(sid, highlight, fuzzy);
}
}
getTagSuggestions(prefix: string): Promise<string[]> {
if (this.backend() == "sqlite") {
getTagSuggestions(prefix) {
if (this.backend() === "sqlite") {
return this.getTagSuggestionsSqlite(prefix);
} else {
return this.getTagSuggestionsEs(prefix);
}
}
private getTagSuggestionsSqlite(prefix): Promise<string[]> {
getTagSuggestionsSqlite(prefix) {
return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
.then(resp => (resp.data));
}
private getTagSuggestionsEs(prefix): Promise<string[]> {
getTagSuggestionsEs(prefix) {
return this.esQuery({
suggest: {
tag: {
@ -723,8 +624,8 @@ class Sist2Api {
});
}
getEmbeddings(indexId, docId, modelId) {
return axios.post(`${this.baseUrl}/e/${indexId}/${docId}/${modelId.toString().padStart(3, '0')}`)
getEmbeddings(sid, modelId) {
return axios.post(`${this.baseUrl}/e/${sid}/${modelId.toString().padStart(3, '0')}`)
.then(resp => (resp.data));
}
}

34
sist2-vue/src/Sist2ElasticsearchQuery.ts → sist2-vue/src/Sist2ElasticsearchQuery.js
View File

@ -1,5 +1,5 @@
import store from "./store";
import sist2Api, {EsHit, Index} from "@/Sist2Api";
import store from "@/store";
import Sist2Api from "@/Sist2Api";
const SORT_MODES = {
score: {
@ -7,62 +7,62 @@ const SORT_MODES = {
{_score: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._score
key: (hit) => hit._score
},
random: {
mode: [
{_score: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._score
key: (hit) => hit._score
},
dateAsc: {
mode: [
{mtime: {order: "asc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.mtime
key: (hit) => hit._source.mtime
},
dateDesc: {
mode: [
{mtime: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.mtime
key: (hit) => hit._source.mtime
},
sizeAsc: {
mode: [
{size: {order: "asc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.size
key: (hit) => hit._source.size
},
sizeDesc: {
mode: [
{size: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.size
key: (hit) => hit._source.size
},
nameAsc: {
mode: [
{name: {order: "asc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.name
key: (hit) => hit._source.name
},
nameDesc: {
mode: [
{name: {order: "desc"}},
{_tie: {order: "asc"}}
],
key: (hit: EsHit) => hit._source.name
key: (hit) => hit._source.name
}
} as any;
};
class Sist2ElasticsearchQuery {
searchQuery(blankSearch: boolean = false): any {
searchQuery(blankSearch = false) {
const getters = store.getters;
@ -76,7 +76,7 @@ class Sist2ElasticsearchQuery {
const fuzzy = getters.fuzzy;
const size = getters.size;
const after = getters.lastDoc;
const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
const selectedMimeTypes = getters.selectedMimeTypes;
const selectedTags = getters.selectedTags;
const sortMode = getters.embedding ? "score" : getters.sortMode;
@ -86,7 +86,7 @@ class Sist2ElasticsearchQuery {
const filters = [
{terms: {index: selectedIndexIds}}
] as any[];
];
const fields = [
"name^8",
@ -138,7 +138,7 @@ class Sist2ElasticsearchQuery {
if (getters.optTagOrOperator) {
filters.push({terms: {"tag": selectedTags}});
} else {
selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
selectedTags.forEach((tag) => filters.push({term: {"tag": tag}}));
}
}
}
@ -173,7 +173,7 @@ class Sist2ElasticsearchQuery {
},
sort: SORT_MODES[sortMode].mode,
size: size,
} as any;
};
if (!after) {
q.aggs = {
@ -193,7 +193,7 @@ class Sist2ElasticsearchQuery {
if (getters.embedding) {
delete q.query;
const field = "emb." + sist2Api.models().find(m => m.id == getters.embeddingsModel).path;
const field = "emb." + Sist2Api.models().find(m => m.id === getters.embeddingsModel).path;
if (hasKnn) {
// Use knn (8.8+)

19
sist2-vue/src/Sist2SqliteQuery.ts → sist2-vue/src/Sist2SqliteQuery.js
View File

@ -1,5 +1,4 @@
import store from "./store";
import {EsHit, Index} from "@/Sist2Api";
import store from "@/store";
const SORT_MODES = {
score: {
@ -29,18 +28,12 @@ const SORT_MODES = {
"sort": "name",
"sortAsc": false
}
} as any;
interface SortMode {
text: string
mode: any[]
key: (hit: EsHit) => any
}
};
class Sist2ElasticsearchQuery {
searchQuery(): any {
searchQuery() {
const getters = store.getters;
@ -52,7 +45,7 @@ class Sist2ElasticsearchQuery {
const dateMax = getters.dateMax;
const size = getters.size;
const after = getters.lastDoc;
const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
const selectedMimeTypes = getters.selectedMimeTypes;
const selectedTags = getters.selectedTags;
@ -95,7 +88,7 @@ class Sist2ElasticsearchQuery {
if (selectedTags.length > 0) {
q["tags"] = selectedTags
}
if (getters.sortMode == "random") {
if (getters.sortMode === "random") {
q["seed"] = getters.seed;
}
if (getters.optHighlight) {
@ -108,7 +101,7 @@ class Sist2ElasticsearchQuery {
q["embedding"] = getters.embedding;
q["sort"] = "embedding";
q["sortAsc"] = false;
} else if (getters.sortMode == "embedding") {
} else if (getters.sortMode === "embedding") {
q["sort"] = "sort"
q["sortAsc"] = true;
}

3
sist2-vue/src/components/D3Treemap.vue
View File

@ -9,8 +9,7 @@
<script>
import * as d3 from "d3";
import {burrow} from "@/util-js"
import {humanFileSize} from "@/util";
import {humanFileSize, burrow} from "@/util";
import Sist2Api from "@/Sist2Api";
import domtoimage from "dom-to-image";

3
sist2-vue/src/components/DateSlider.vue
View File

@ -31,8 +31,7 @@
<script>
import noUiSlider from 'nouislider';
import 'nouislider/dist/nouislider.css';
import {humanDate} from "@/util";
import {mergeTooltips} from "@/util-js";
import {humanDate, mergeTooltips} from "@/util";
export default {
name: "DateSlider",

7
sist2-vue/src/components/DocCard.vue
View File

@ -16,7 +16,7 @@
<!-- Audio player-->
<audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
:type="doc._source.mime"
:src="`f/${doc._source.index}/${doc._id}`"
:src="`f/${sid(doc)}`"
@play="onAudioPlay()"></audio>
<b-card-body class="padding-03">
@ -43,7 +43,7 @@
</template>
<script>
import {ext, humanFileSize, humanTime} from "@/util";
import {ext, humanFileSize, humanTime, sid} from "@/util";
import TagContainer from "@/components/TagContainer.vue";
import DocFileTitle from "@/components/DocFileTitle.vue";
import DocInfoModal from "@/components/DocInfoModal.vue";
@ -69,13 +69,14 @@ export default {
}
},
methods: {
sid: sid,
humanFileSize: humanFileSize,
humanTime: humanTime,
onInfoClick() {
this.showInfo = true;
},
onEmbeddingClick() {
Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
this.$store.commit("setEmbeddingText", "");
this.$store.commit("setEmbedding", embeddings);
this.$store.commit("setEmbeddingDoc", this.doc);

102
sist2-vue/src/components/DocCardWall.vue
View File

@ -1,15 +1,16 @@
<template>
<GridLayout
ref="grid-layout"
:options="gridOptions"
@append="append"
@layout-complete="$emit('layout-complete')"
>
<DocCard v-for="doc in docs" :key="doc._id" :doc="doc" :width="width"></DocCard>
</GridLayout>
<GridLayout
ref="grid-layout"
:options="gridOptions"
@append="append"
@layout-complete="$emit('layout-complete')"
>
<DocCard v-for="doc in docs" :key="sid(doc)" :doc="doc" :width="width"></DocCard>
</GridLayout>
</template>
<script>
import {sid} from "@/util";
import Vue from "vue";
import DocCard from "@/components/DocCard";
@ -18,50 +19,53 @@ import VueInfiniteGrid from "@egjs/vue-infinitegrid";
Vue.use(VueInfiniteGrid);
export default Vue.extend({
components: {
DocCard,
},
props: ["docs", "append"],
data() {
return {
width: 0,
gridOptions: {
align: "center",
margin: 0,
transitionDuration: 0,
isOverflowScroll: false,
isConstantSize: false,
useFit: false,
// Indicates whether keep the number of DOMs is maintained. If the useRecycle value is 'true', keep the number
// of DOMs is maintained. If the useRecycle value is 'false', the number of DOMs will increase as card elements
// are added.
useRecycle: false
}
}
},
computed: {
colCount() {
const columns = this.$store.getters["optColumns"];
if (columns === "auto") {
return Math.round(this.$refs["grid-layout"].$el.scrollWidth / 300)
}
return columns;
components: {
DocCard,
},
},
mounted() {
this.width = this.$refs["grid-layout"].$el.scrollWidth / this.colCount;
props: ["docs", "append"],
data() {
return {
width: 0,
gridOptions: {
align: "center",
margin: 0,
transitionDuration: 0,
isOverflowScroll: false,
isConstantSize: false,
useFit: false,
// Indicates whether keep the number of DOMs is maintained. If the useRecycle value is 'true', keep the number
// of DOMs is maintained. If the useRecycle value is 'false', the number of DOMs will increase as card elements
// are added.
useRecycle: false
}
}
},
methods: {
sid,
},
computed: {
colCount() {
const columns = this.$store.getters["optColumns"];
if (this.colCount === 1) {
this.$refs["grid-layout"].$el.classList.add("grid-single-column");
}
if (columns === "auto") {
return Math.round(this.$refs["grid-layout"].$el.scrollWidth / 300)
}
return columns;
},
},
mounted() {
this.width = this.$refs["grid-layout"].$el.scrollWidth / this.colCount;
this.$store.subscribe((mutation) => {
if (mutation.type === "busUpdateWallItems" && this.$refs["grid-layout"]) {
this.$refs["grid-layout"].updateItems();
}
});
},
if (this.colCount === 1) {
this.$refs["grid-layout"].$el.classList.add("grid-single-column");
}
this.$store.subscribe((mutation) => {
if (mutation.type === "busUpdateWallItems" && this.$refs["grid-layout"]) {
this.$refs["grid-layout"].updateItems();
}
});
},
});
</script>

5
sist2-vue/src/components/DocFileTitle.vue
View File

@ -1,5 +1,5 @@
<template>
<a :href="`f/${doc._source.index}/${doc._id}`"
<a :href="`f/${sid(doc)}`"
:class="doc._source.embedding ? 'file-title-anchor-with-embedding' : 'file-title-anchor'" target="_blank">
<div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
v-html="fileName() + ext(doc)"></div>
@ -7,12 +7,13 @@
</template>
<script>
import {ext} from "@/util";
import {ext, sid} from "@/util";
export default {
name: "DocFileTitle",
props: ["doc"],
methods: {
sid: sid,
ext: ext,
fileName() {
if (!this.doc.highlight) {

39
sist2-vue/src/components/DocInfoModal.vue
View File

@ -1,33 +1,34 @@
<template>
<b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
>
<template #modal-title>
<h5 class="modal-title" :title="doc._source.name + ext(doc)">
{{ doc._source.name + ext(doc) }}
<router-link :to="`/file?byId=${doc._id}`">#</router-link>
</h5>
</template>
<b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
>
<template #modal-title>
<h5 class="modal-title" :title="doc._source.name + ext(doc)">
{{ doc._source.name + ext(doc) }}
<router-link :to="`/file?byId=${doc._id}`">#</router-link>
</h5>
</template>
<img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
<img v-if="doc._props.hasThumbnail" :src="`t/${sid(doc)}`" alt="" class="fit card-img-top">
<InfoTable :doc="doc"></InfoTable>
<InfoTable :doc="doc"></InfoTable>
<LazyContentDiv :doc-id="doc._id"></LazyContentDiv>
</b-modal>
<LazyContentDiv :sid="sid(doc)"></LazyContentDiv>
</b-modal>
</template>
<script>
import {ext} from "@/util";
import {ext, sid} from "@/util";
import InfoTable from "@/components/InfoTable";
import LazyContentDiv from "@/components/LazyContentDiv";
export default {
name: "DocInfoModal",
components: {LazyContentDiv, InfoTable},
props: ["doc", "show"],
methods: {
ext: ext,
}
name: "DocInfoModal",
components: {LazyContentDiv, InfoTable},
props: ["doc", "show"],
methods: {
ext: ext,
sid: sid
}
}
</script>

52
sist2-vue/src/components/DocList.vue
View File

@ -1,45 +1,49 @@
<template>
<b-list-group class="mt-3">
<DocListItem v-for="doc in docs" :key="doc._id" :doc="doc"></DocListItem>
</b-list-group>
<b-list-group class="mt-3">
<DocListItem v-for="doc in docs" :key="sid(doc)" :doc="doc"></DocListItem>
</b-list-group>
</template>
<script lang="ts">
<script>
import {sid} from "@/util";
import DocListItem from "@/components/DocListItem.vue";
import Vue from "vue";
export default Vue.extend({
name: "DocList",
components: {DocListItem},
props: ["docs", "append"],
mounted() {
window.addEventListener("scroll", () => {
const threshold = 400;
const app = document.getElementById("app");
name: "DocList",
components: {DocListItem},
props: ["docs", "append"],
mounted() {
window.addEventListener("scroll", () => {
const threshold = 400;
const app = document.getElementById("app");
if ((window.innerHeight + window.scrollY) >= app.offsetHeight - threshold) {
this.append();
}
});
}
if ((window.innerHeight + window.scrollY) >= app.offsetHeight - threshold) {
this.append();
}
});
},
methods: {
sid: sid
}
});
</script>
<style>
.theme-black .list-group-item {
background: #212121;
color: #e0e0e0;
background: #212121;
color: #e0e0e0;
border-bottom: none;
border-left: none;
border-right: none;
border-radius: 0;
padding: .25rem 0.5rem;
border-bottom: none;
border-left: none;
border-right: none;
border-radius: 0;
padding: .25rem 0.5rem;
}
.theme-black .list-group-item:first-child {
border-top: none;
border-top: none;
}
</style>

8
sist2-vue/src/components/DocListItem.vue
View File

@ -17,10 +17,10 @@
</div>
<img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
:src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
:src="(doc._props.isGif && hover) ? `f/${sid(doc)}` : `t/${sid(doc)}`"
alt=""
class="pointer fit-sm" @click="onThumbnailClick()">
<img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
<img v-else :src="`t/${sid(doc)}`" alt=""
class="fit-sm">
</div>
</div>
@ -70,6 +70,7 @@ import FileIcon from "@/components/icons/FileIcon";
import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
import MLIcon from "@/components/icons/MlIcon.vue";
import Sist2Api from "@/Sist2Api";
import {sid} from "@/util";
export default {
name: "DocListItem",
@ -82,12 +83,13 @@ export default {
}
},
methods: {
sid: sid,
async onThumbnailClick() {
this.$store.commit("setUiLightboxSlide", this.doc._seq);
await this.$store.dispatch("showLightbox");
},
onEmbeddingClick() {
Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
this.$store.commit("setEmbeddingText", "");
this.$store.commit("setEmbedding", embeddings);
this.$store.commit("setEmbeddingDoc", this.doc);

301
sist2-vue/src/components/FullThumbnail.vue
View File

@ -1,188 +1,189 @@
<template>
<div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()"
@touchstart="onTouchStart()">
<div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
<div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()"
@touchstart="onTouchStart()">
<div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
</div>
<div
v-if="doc._props.isImage && doc._props.imageAspectRatio < 5"
class="card-img-overlay"
:class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
</div>
<div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0"
class="card-img-overlay"
:class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
</div>
<div v-if="doc._props.isPlayableVideo" class="play">
<svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
<path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
</svg>
</div>
<img ref="tn"
v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
:src="tnSrc"
alt=""
:style="{height: (doc._props.isGif && hover) ? `${tnHeight()}px` : undefined}"
class="pointer fit card-img-top" @click="onThumbnailClick()">
<img v-else :src="tnSrc" alt=""
class="fit card-img-top">
<ThumbnailProgressBar v-if="hover && doc._props.hasVidPreview"
:progress="(currentThumbnailNum + 1) / (doc._props.tnNum)"
></ThumbnailProgressBar>
</div>
<div
v-if="doc._props.isImage && doc._props.imageAspectRatio < 5"
class="card-img-overlay"
:class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
</div>
<div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0"
class="card-img-overlay"
:class="{'small-badge': smallBadge}">
<span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
</div>
<div v-if="doc._props.isPlayableVideo" class="play">
<svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
<path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
</svg>
</div>
<img ref="tn"
v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
:src="tnSrc"
alt=""
:style="{height: (doc._props.isGif && hover) ? `${tnHeight()}px` : undefined}"
class="pointer fit card-img-top" @click="onThumbnailClick()">
<img v-else :src="tnSrc" alt=""
class="fit card-img-top">
<ThumbnailProgressBar v-if="hover && doc._props.hasVidPreview"
:progress="(currentThumbnailNum + 1) / (doc._props.tnNum)"
></ThumbnailProgressBar>
</div>
</template>
<script>
import {humanTime} from "@/util";
import {humanTime, sid} from "@/util";
import ThumbnailProgressBar from "@/components/ThumbnailProgressBar";
export default {
name: "FullThumbnail",
props: ["doc", "smallBadge"],
components: {ThumbnailProgressBar},
data() {
return {
hover: false,
currentThumbnailNum: 0,
timeoutId: null
name: "FullThumbnail",
props: ["doc", "smallBadge"],
components: {ThumbnailProgressBar},
data() {
return {
hover: false,
currentThumbnailNum: 0,
timeoutId: null
}
},
created() {
this.$store.subscribe((mutation) => {
if (mutation.type === "busTnTouchStart" && mutation.payload !== this.doc._id) {
this.onTnLeave();
}
});
},
computed: {
tnSrc() {
return this.getThumbnailSrc(this.currentThumbnailNum);
},
},
methods: {
sid: sid,
getThumbnailSrc(thumbnailNum) {
const doc = this.doc;
const props = doc._props;
if (props.isGif && this.hover) {
return `f/${sid(doc)}`;
}
return (this.currentThumbnailNum === 0)
? `t/${sid(doc)}`
: `t/${sid(doc)}/${String(thumbnailNum).padStart(4, "0")}`;
},
humanTime: humanTime,
onThumbnailClick() {
this.$emit("onThumbnailClick");
},
tnHeight() {
return this.$refs.tn.height;
},
tnWidth() {
return this.$refs.tn.width;
},
onTnEnter() {
this.hover = true;
const start = Date.now()
if (this.doc._props.hasVidPreview) {
let img = new Image();
img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
img.onload = () => {
this.currentThumbnailNum += 1;
this.scheduleNextTnNum(Date.now() - start);
}
}
},
onTnLeave() {
this.currentThumbnailNum = 0;
this.hover = false;
if (this.timeoutId !== null) {
window.clearTimeout(this.timeoutId);
this.timeoutId = null;
}
},
scheduleNextTnNum(offset = 0) {
const INTERVAL = (this.$store.state.optVidPreviewInterval ?? 700) - offset;
this.timeoutId = window.setTimeout(() => {
const start = Date.now();
if (!this.hover) {
return;
}
if (this.currentThumbnailNum === this.doc._props.tnNum - 1) {
this.currentThumbnailNum = 0;
this.scheduleNextTnNum();
} else {
let img = new Image();
img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
img.onload = () => {
this.currentThumbnailNum += 1;
this.scheduleNextTnNum(Date.now() - start);
}
}
}, INTERVAL);
},
onTouchStart() {
this.$store.commit("busTnTouchStart", this.doc._id);
if (!this.hover) {
this.onTnEnter()
}
},
}
},
created() {
this.$store.subscribe((mutation) => {
if (mutation.type === "busTnTouchStart" && mutation.payload !== this.doc._id) {
this.onTnLeave();
}
});
},
computed: {
tnSrc() {
return this.getThumbnailSrc(this.currentThumbnailNum);
},
},
methods: {
getThumbnailSrc(thumbnailNum) {
const doc = this.doc;
const props = doc._props;
if (props.isGif && this.hover) {
return `f/${doc._source.index}/${doc._id}`;
}
return (this.currentThumbnailNum === 0)
? `t/${doc._source.index}/${doc._id}`
: `t/${doc._source.index}/${doc._id}/${String(thumbnailNum).padStart(4, "0")}`;
},
humanTime: humanTime,
onThumbnailClick() {
this.$emit("onThumbnailClick");
},
tnHeight() {
return this.$refs.tn.height;
},
tnWidth() {
return this.$refs.tn.width;
},
onTnEnter() {
this.hover = true;
const start = Date.now()
if (this.doc._props.hasVidPreview) {
let img = new Image();
img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
img.onload = () => {
this.currentThumbnailNum += 1;
this.scheduleNextTnNum(Date.now() - start);
}
}
},
onTnLeave() {
this.currentThumbnailNum = 0;
this.hover = false;
if (this.timeoutId !== null) {
window.clearTimeout(this.timeoutId);
this.timeoutId = null;
}
},
scheduleNextTnNum(offset = 0) {
const INTERVAL = (this.$store.state.optVidPreviewInterval ?? 700) - offset;
this.timeoutId = window.setTimeout(() => {
const start = Date.now();
if (!this.hover) {
return;
}
if (this.currentThumbnailNum === this.doc._props.tnNum - 1) {
this.currentThumbnailNum = 0;
this.scheduleNextTnNum();
} else {
let img = new Image();
img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
img.onload = () => {
this.currentThumbnailNum += 1;
this.scheduleNextTnNum(Date.now() - start);
}
}
}, INTERVAL);
},
onTouchStart() {
this.$store.commit("busTnTouchStart", this.doc._id);
if (!this.hover) {
this.onTnEnter()
}
},
}
}
</script>
<style scoped>
.img-wrapper {
position: relative;
position: relative;
}
.img-wrapper:hover svg {
fill: rgba(0, 0, 0, 1);
fill: rgba(0, 0, 0, 1);
}
.card-img-top {
border-top-left-radius: 0;
border-top-right-radius: 0;
border-top-left-radius: 0;
border-top-right-radius: 0;
}
.play {
position: absolute;
width: 25px;
height: 25px;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
pointer-events: none;
position: absolute;
width: 25px;
height: 25px;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
pointer-events: none;
}
.play svg {
fill: rgba(0, 0, 0, 0.7);
fill: rgba(0, 0, 0, 0.7);
}
.badge-resolution {
color: #c6c6c6;
background-color: #272727CC;
padding: 2px 3px;
color: #c6c6c6;
background-color: #272727CC;
padding: 2px 3px;
}
.card-img-overlay {
pointer-events: none;
padding: 2px 6px;
bottom: 4px;
top: unset;
left: unset;
right: 0;
pointer-events: none;
padding: 2px 6px;
bottom: 4px;
top: unset;
left: unset;
right: 0;
}
.small-badge {
padding: 1px 3px;
font-size: 70%;
padding: 1px 3px;
font-size: 70%;
}
</style>

16
sist2-vue/src/components/IndexPicker.vue
View File

@ -27,11 +27,12 @@
@click.shift="shiftClick(idx, $event)"
class="d-flex justify-content-between align-items-center list-group-item-action pointer"
:class="{active: lastClickIndex === idx}"
:key="idx.id"
>
<div class="d-flex">
<b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
{{ idx.name }}
<div style="vertical-align: center; margin-left: 5px">
<div v-if="hasEmbeddings(idx)" style="vertical-align: center; margin-left: 5px">
<MLIcon small style="top: -1px; position: relative"></MLIcon>
</div>
<span class="text-muted timestamp-text ml-2"
@ -43,7 +44,7 @@
</div>
</template>
<script lang="ts">
<script>
import SmallBadge from "./SmallBadge.vue"
import {mapActions, mapGetters} from "vuex";
import Vue from "vue";
@ -111,7 +112,7 @@ export default Vue.extend({
onSelect(value) {
this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
},
formatIdxDate(timestamp: number): string {
formatIdxDate(timestamp) {
return format(new Date(timestamp * 1000), "yyyy-MM-dd");
},
toggleIndex(index, e) {
@ -121,14 +122,17 @@ export default Vue.extend({
this.lastClickIndex = index;
if (this.isSelected(index)) {
this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id !== index.id));
} else {
this.setSelectedIndices([index, ...this.selectedIndices]);
}
},
isSelected(index) {
return this.selectedIndices.find(idx => idx.id == index.id) != null;
}
return this.selectedIndices.find(idx => idx.id === index.id) != null;
},
hasEmbeddings(index) {
return index.models.length > 0;
},
},
})
</script>

6
sist2-vue/src/components/LazyContentDiv.vue
View File

@ -10,7 +10,7 @@
>{{ $t("ml.analyzeText") }}
</b-button>
<b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="nerModel">
<b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
<b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()" :key="opt.value">{{ opt.text }}
</b-select-option>
</b-select>
</b-form>
@ -46,7 +46,7 @@ import {mapGetters, mapMutations} from "vuex";
export default {
name: "LazyContentDiv",
components: {AnalyzedContentSpansContainer, Preloader},
props: ["docId"],
props: ["sid"],
data() {
return {
ModelsRepo,
@ -70,7 +70,7 @@ export default {
}
Sist2Api
.getDocument(this.docId, this.$store.state.optHighlight, this.$store.state.fuzzy)
.getDocument(this.sid, this.$store.state.optHighlight, this.$store.state.fuzzy)
.then(doc => {
this.loading = false;

10
sist2-vue/src/components/ResultsCard.vue
View File

@ -43,8 +43,7 @@
</b-card>
</template>
<script lang="ts">
import Sist2Api, {EsResult} from "@/Sist2Api";
<script>
import Vue from "vue";
import {humanFileSize} from "@/util";
import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
@ -52,6 +51,7 @@ import SortSelect from "@/components/SortSelect.vue";
import Preloader from "@/components/Preloader.vue";
import Sist2Query from "@/Sist2ElasticsearchQuery";
import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
import Sist2Api from "@/Sist2Api";
export default Vue.extend({
name: "ResultsCard",
@ -64,7 +64,7 @@ export default Vue.extend({
return this.$store.state.lastQueryResults != null;
},
hitCount() {
return (this.$store.state.firstQueryResults as EsResult).aggregations.total_count.value;
return (this.$store.state.firstQueryResults).aggregations.total_count.value;
},
tableItems() {
const items = [];
@ -79,10 +79,10 @@ export default Vue.extend({
},
methods: {
took() {
return (this.$store.state.lastQueryResults as EsResult).took + "ms";
return (this.$store.state.lastQueryResults).took + "ms";
},
totalSize() {
return humanFileSize((this.$store.state.firstQueryResults as EsResult).aggregations.total_size.value);
return humanFileSize((this.$store.state.firstQueryResults).aggregations.total_size.value);
},
onToggle() {
const show = !document.getElementById("collapse-1").classList.contains("show");

3
sist2-vue/src/components/SizeSlider.vue
View File

@ -5,8 +5,7 @@
<script>
import noUiSlider from 'nouislider';
import 'nouislider/dist/nouislider.css';
import {humanFileSize} from "@/util";
import {mergeTooltips} from "@/util-js";
import {humanFileSize, mergeTooltips} from "@/util";
export default {
name: "SizeSlider",

2
sist2-vue/src/components/SmallBadge.vue
View File

@ -2,7 +2,7 @@
<b-badge variant="secondary" :pill="pill">{{ text }}</b-badge>
</template>
<script lang="ts">
<script>
import Vue from "vue";
export default Vue.extend({

8
sist2-vue/src/components/ThumbnailProgressBar.vue
View File

@ -1,5 +1,5 @@
<template>
<div class="thumbnail-progress-bar" :style="{width: `${percentProgress}%`}"></div>
<div class="thumbnail_count-progress-bar" :style="{width: `${percentProgress}%`}"></div>
</template>
<script>
@ -16,7 +16,7 @@ export default {
<style scoped>
.thumbnail-progress-bar {
.thumbnail_count-progress-bar {
position: absolute;
left: 0;
bottom: 0;
@ -27,11 +27,11 @@ export default {
z-index: 9;
}
.theme-black .thumbnail-progress-bar {
.theme-black .thumbnail_count-progress-bar {
background: rgba(0, 188, 212, 0.95);
}
.sub-document .thumbnail-progress-bar {
.sub-document .thumbnail_count-progress-bar {
max-width: calc(100% - 8px);
left: 4px;
}

0
sist2-vue/src/i18n/messages.ts → sist2-vue/src/i18n/messages.js
View File

0
sist2-vue/src/router/auth0.ts → sist2-vue/src/router/auth0.js
View File

2
sist2-vue/src/router/index.ts → sist2-vue/src/router/index.js
View File

@ -21,7 +21,7 @@ const authGuard = (to, from, next) => {
next();
}
const routes: Array<RouteConfig> = [
const routes = [
{
path: "/",
name: "SearchPage",

74
sist2-vue/src/store/index.ts → sist2-vue/src/store/index.js
View File

@ -1,20 +1,18 @@
import Vue from "vue"
import Vuex from "vuex"
import VueRouter, {Route} from "vue-router";
import {EsHit, EsResult, EsTag, Index} from "@/Sist2Api";
import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
import {getInstance} from "@/plugins/auth0.js";
const CONF_VERSION = 3;
Vue.use(Vuex)
Vue.use(Vuex);
export default new Vuex.Store({
state: {
seed: 0,
indices: [] as Index[],
tags: [] as EsTag[],
sist2Info: null as any,
indices: [],
tags: [],
sist2Info: null,
sizeMin: undefined,
sizeMax: undefined,
@ -64,12 +62,12 @@ export default new Vuex.Store({
optAutoAnalyze: false,
optMlDefaultModel: null,
_onLoadSelectedIndices: [] as string[],
_onLoadSelectedMimeTypes: [] as string[],
_onLoadSelectedTags: [] as string[],
selectedIndices: [] as Index[],
selectedMimeTypes: [] as string[],
selectedTags: [] as string[],
_onLoadSelectedIndices: [],
_onLoadSelectedMimeTypes: [],
_onLoadSelectedTags: [],
selectedIndices: [],
selectedMimeTypes: [],
selectedTags: [],
lastQueryResults: null,
firstQueryResults: null,
@ -80,10 +78,10 @@ export default new Vuex.Store({
uiSqliteMode: false,
uiLightboxIsOpen: false,
uiShowLightbox: false,
uiLightboxSources: [] as string[],
uiLightboxThumbs: [] as string[],
uiLightboxCaptions: [] as any[],
uiLightboxTypes: [] as string[],
uiLightboxSources: [],
uiLightboxThumbs: [],
uiLightboxCaptions: [],
uiLightboxTypes: [],
uiLightboxKey: 0,
uiLightboxSlide: 0,
uiReachedScrollEnd: false,
@ -91,7 +89,7 @@ export default new Vuex.Store({
uiDetailsMimeAgg: null,
uiShowDetails: false,
uiMimeMap: [] as any[],
uiMimeMap: [],
auth0Token: null,
nerModel: {
@ -122,7 +120,7 @@ export default new Vuex.Store({
if (state._onLoadSelectedIndices.length > 0) {
state.selectedIndices = val.filter(
(idx: Index) => state._onLoadSelectedIndices.some(prefix => idx.id.startsWith(prefix))
(idx) => state._onLoadSelectedIndices.some(id => id === idx.id.toString(16))
);
} else {
state.selectedIndices = val;
@ -145,18 +143,18 @@ export default new Vuex.Store({
setSelectedIndices: (state, val) => state.selectedIndices = val,
setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
setSelectedTags: (state, val) => state.selectedTags = val,
setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
_setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
_setKeySequence: (state, val: number) => state.keySequence = val,
_setQuerySequence: (state, val: number) => state.querySequence = val,
setUiLightboxIsOpen: (state, val) => state.uiLightboxIsOpen = val,
_setUiShowLightbox: (state, val) => state.uiShowLightbox = val,
setUiLightboxKey: (state, val) => state.uiLightboxKey = val,
_setKeySequence: (state, val) => state.keySequence = val,
_setQuerySequence: (state, val) => state.querySequence = val,
addLightboxSource: (state, {source, thumbnail, caption, type}) => {
state.uiLightboxSources.push(source);
state.uiLightboxThumbs.push(thumbnail);
state.uiLightboxCaptions.push(caption);
state.uiLightboxTypes.push(type);
},
setUiLightboxSlide: (state, val: number) => state.uiLightboxSlide = val,
setUiLightboxSlide: (state, val) => state.uiLightboxSlide = val,
setUiLightboxSources: (state, val) => state.uiLightboxSources = val,
setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
@ -230,7 +228,7 @@ export default new Vuex.Store({
store.commit("setOptLang", val.lang);
}
},
loadFromArgs({commit}, route: Route) {
loadFromArgs({commit}, route) {
if (route.query.q) {
commit("setSearchText", route.query.q);
@ -265,11 +263,11 @@ export default new Vuex.Store({
}
if (route.query.m) {
commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m as string));
commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m));
}
if (route.query.t) {
commit("_setOnLoadSelectedTags", (route.query.t as string).split(","));
commit("_setOnLoadSelectedTags", (route.query.t).split(","));
}
if (route.query.sort) {
@ -280,7 +278,7 @@ export default new Vuex.Store({
commit("setSeed", Number(route.query.seed));
}
},
async updateArgs({state}, router: VueRouter) {
async updateArgs({state}, router) {
if (router.currentRoute.path !== "/") {
return;
@ -290,14 +288,14 @@ export default new Vuex.Store({
query: {
q: state.searchText.trim() ? state.searchText.trim().replace(/\s+/g, " ") : undefined,
fuzzy: state.fuzzy ? null : undefined,
i: state.selectedIndices ? state.selectedIndices.map((idx: Index) => idx.idPrefix) : undefined,
i: state.selectedIndices ? state.selectedIndices.map((idx) => idx.id.toString(16)) : undefined,
dMin: state.dateMin,
dMax: state.dateMax,
sMin: state.sizeMin,
sMax: state.sizeMax,
path: state.pathText ? state.pathText : undefined,
m: serializeMimes(state.selectedMimeTypes),
t: state.selectedTags.length == 0 ? undefined : state.selectedTags.join(","),
t: state.selectedTags.length === 0 ? undefined : state.selectedTags.join(","),
sort: state.sortMode === "score" ? undefined : state.sortMode,
seed: state.sortMode === "random" ? state.seed.toString() : undefined
}
@ -306,11 +304,11 @@ export default new Vuex.Store({
});
},
updateConfiguration({state}) {
const conf = {} as any;
const conf = {};
Object.keys(state).forEach((key) => {
if (key.startsWith("opt")) {
conf[key] = (state as any)[key];
conf[key] = (state)[key];
}
});
@ -323,14 +321,14 @@ export default new Vuex.Store({
if (confString) {
const conf = JSON.parse(confString);
if (!("version" in conf) || conf["version"] != CONF_VERSION) {
if (!("version" in conf) || conf["version"] !== CONF_VERSION) {
localStorage.removeItem("sist2_configuration");
window.location.reload();
}
Object.keys(state).forEach((key) => {
if (key.startsWith("opt")) {
(state as any)[key] = conf[key];
(state)[key] = conf[key];
}
});
}
@ -386,7 +384,7 @@ export default new Vuex.Store({
indices: state => state.indices,
sist2Info: state => state.sist2Info,
indexMap: state => {
const map = {} as any;
const map = {};
state.indices.forEach(idx => map[idx.id] = idx);
return map;
},
@ -405,12 +403,12 @@ export default new Vuex.Store({
size: state => state.optSize,
sortMode: state => state.sortMode,
lastQueryResult: state => state.lastQueryResults,
lastDoc: function (state): EsHit | null {
lastDoc: function (state) {
if (state.lastQueryResults == null) {
return null;
}
return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
return (state.lastQueryResults).hits.hits.slice(-1)[0];
},
uiShowLightbox: state => state.uiShowLightbox,
uiLightboxSources: state => state.uiLightboxSources,
@ -451,7 +449,7 @@ export default new Vuex.Store({
optMlRepositories: state => state.optMlRepositories,
mlRepositoryList: state => {
const repos = state.optMlRepositories.split("\n")
return repos[0] == "" ? [] : repos;
return repos[0] === "" ? [] : repos;
},
optMlDefaultModel: state => state.optMlDefaultModel,
optAutoAnalyze: state => state.optAutoAnalyze,

139
sist2-vue/src/util-js.js
View File

@ -1,139 +0,0 @@
export function mergeTooltips(slider, threshold, separator, fixTooltips) {
const isMobile = window.innerWidth <= 650;
if (isMobile) {
threshold = 25;
}
const textIsRtl = getComputedStyle(slider).direction === 'rtl';
const isRtl = slider.noUiSlider.options.direction === 'rtl';
const isVertical = slider.noUiSlider.options.orientation === 'vertical';
const tooltips = slider.noUiSlider.getTooltips();
const origins = slider.noUiSlider.getOrigins();
// Move tooltips into the origin element. The default stylesheet handles this.
tooltips.forEach(function (tooltip, index) {
if (tooltip) {
origins[index].appendChild(tooltip);
}
});
slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
const pools = [[]];
const poolPositions = [[]];
const poolValues = [[]];
let atPool = 0;
// Assign the first tooltip to the first pool, if the tooltip is configured
if (tooltips[0]) {
pools[0][0] = 0;
poolPositions[0][0] = positions[0];
poolValues[0][0] = values[0];
}
for (let i = 1; i < positions.length; i++) {
if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
atPool++;
pools[atPool] = [];
poolValues[atPool] = [];
poolPositions[atPool] = [];
}
if (tooltips[i]) {
pools[atPool].push(i);
poolValues[atPool].push(values[i]);
poolPositions[atPool].push(positions[i]);
}
}
pools.forEach(function (pool, poolIndex) {
const handlesInPool = pool.length;
for (let j = 0; j < handlesInPool; j++) {
const handleNumber = pool[j];
if (j === handlesInPool - 1) {
let offset = 0;
poolPositions[poolIndex].forEach(function (value) {
offset += 1000 - 10 * value;
});
const direction = isVertical ? 'bottom' : 'right';
const last = isRtl ? 0 : handlesInPool - 1;
const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
// Center this tooltip over the affected handles
tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
tooltips[handleNumber].style.display = 'block';
tooltips[handleNumber].style[direction] = offset + '%';
} else {
// Hide this tooltip
tooltips[handleNumber].style.display = 'none';
}
}
});
if (fixTooltips) {
const isMobile = window.innerWidth <= 650;
const len = isMobile ? 20 : 5;
if (positions[0] < len) {
tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
} else {
tooltips[0].style.right = "0"
}
if (positions[1] > (100 - len)) {
tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
} else {
tooltips[1].style.right = "0"
}
}
});
}
export function burrow(table, addSelfDir, rootName) {
const root = {};
table.forEach(row => {
let layer = root;
row.taxonomy.forEach(key => {
layer[key] = key in layer ? layer[key] : {};
layer = layer[key];
});
if (Object.keys(layer).length === 0) {
layer["$size$"] = row.size;
} else if (addSelfDir) {
layer["."] = {
"$size$": row.size,
};
}
});
const descend = function (obj, depth) {
return Object.keys(obj).filter(k => k !== "$size$").map(k => {
const child = {
name: k,
depth: depth,
value: 0,
children: descend(obj[k], depth + 1)
};
if ("$size$" in obj[k]) {
child.value = obj[k]["$size$"];
}
return child;
});
};
return {
name: rootName,
children: descend(root, 1),
value: 0,
depth: 0,
}
}

329
sist2-vue/src/util.js Normal file
View File

@ -0,0 +1,329 @@
export function mergeTooltips(slider, threshold, separator, fixTooltips) {
const isMobile = window.innerWidth <= 650;
if (isMobile) {
threshold = 25;
}
const textIsRtl = getComputedStyle(slider).direction === 'rtl';
const isRtl = slider.noUiSlider.options.direction === 'rtl';
const isVertical = slider.noUiSlider.options.orientation === 'vertical';
const tooltips = slider.noUiSlider.getTooltips();
const origins = slider.noUiSlider.getOrigins();
// Move tooltips into the origin element. The default stylesheet handles this.
tooltips.forEach(function (tooltip, index) {
if (tooltip) {
origins[index].appendChild(tooltip);
}
});
slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
const pools = [[]];
const poolPositions = [[]];
const poolValues = [[]];
let atPool = 0;
// Assign the first tooltip to the first pool, if the tooltip is configured
if (tooltips[0]) {
pools[0][0] = 0;
poolPositions[0][0] = positions[0];
poolValues[0][0] = values[0];
}
for (let i = 1; i < positions.length; i++) {
if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
atPool++;
pools[atPool] = [];
poolValues[atPool] = [];
poolPositions[atPool] = [];
}
if (tooltips[i]) {
pools[atPool].push(i);
poolValues[atPool].push(values[i]);
poolPositions[atPool].push(positions[i]);
}
}
pools.forEach(function (pool, poolIndex) {
const handlesInPool = pool.length;
for (let j = 0; j < handlesInPool; j++) {
const handleNumber = pool[j];
if (j === handlesInPool - 1) {
let offset = 0;
poolPositions[poolIndex].forEach(function (value) {
offset += 1000 - 10 * value;
});
const direction = isVertical ? 'bottom' : 'right';
const last = isRtl ? 0 : handlesInPool - 1;
const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
// Center this tooltip over the affected handles
tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
tooltips[handleNumber].style.display = 'block';
tooltips[handleNumber].style[direction] = offset + '%';
} else {
// Hide this tooltip
tooltips[handleNumber].style.display = 'none';
}
}
});
if (fixTooltips) {
const isMobile = window.innerWidth <= 650;
const len = isMobile ? 20 : 5;
if (positions[0] < len) {
tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
} else {
tooltips[0].style.right = "0"
}
if (positions[1] > (100 - len)) {
tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
} else {
tooltips[1].style.right = "0"
}
}
});
}
export function burrow(table, addSelfDir, rootName) {
const root = {};
table.forEach(row => {
let layer = root;
row.taxonomy.forEach(key => {
layer[key] = key in layer ? layer[key] : {};
layer = layer[key];
});
if (Object.keys(layer).length === 0) {
layer["$size$"] = row.size;
} else if (addSelfDir) {
layer["."] = {
"$size$": row.size,
};
}
});
const descend = function (obj, depth) {
return Object.keys(obj).filter(k => k !== "$size$").map(k => {
const child = {
name: k,
depth: depth,
value: 0,
children: descend(obj[k], depth + 1)
};
if ("$size$" in obj[k]) {
child.value = obj[k]["$size$"];
}
return child;
});
};
return {
name: rootName,
children: descend(root, 1),
value: 0,
depth: 0,
}
}
export function ext(hit) {
return srcExt(hit._source)
}
export function srcExt(src) {
return Object.prototype.hasOwnProperty.call(src, "extension")
&& src["extension"] !== "" ? "." + src["extension"] : "";
}
export function strUnescape(str) {
let result = "";
for (let i = 0; i < str.length; i++) {
const c = str[i];
const next = str[i + 1];
if (c === "]") {
if (next === "]") {
result += c;
i += 1;
} else {
result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
i += 2;
}
} else {
result += c;
}
}
return result;
}
const thresh = 1000;
const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
export function humanFileSize(bytes) {
if (bytes === 0) {
return "0 B"
}
if (Math.abs(bytes) < thresh) {
return bytes + ' B';
}
let u = -1;
do {
bytes /= thresh;
++u;
} while (Math.abs(bytes) >= thresh && u < units.length - 1);
return bytes.toFixed(1) + units[u];
}
export function humanTime(sec_num) {
sec_num = Math.floor(sec_num);
const hours = Math.floor(sec_num / 3600);
const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
const seconds = sec_num - (hours * 3600) - (minutes * 60);
if (sec_num < 60) {
return `${sec_num}s`
}
if (sec_num < 3600) {
return `${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
}
return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
}
export function humanDate(numMilis) {
const date = (new Date(numMilis * 1000));
return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
}
export function lum(c) {
c = c.substring(1);
const rgb = parseInt(c, 16);
const r = (rgb >> 16) & 0xff;
const g = (rgb >> 8) & 0xff;
const b = (rgb >> 0) & 0xff;
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
}
export function getSelectedTreeNodes(tree) {
const selectedNodes = new Set();
const selected = tree.selected();
for (let i = 0; i < selected.length; i++) {
if (selected[i].id === "any") {
return ["any"]
}
//Only get children
if (selected[i].text.indexOf("(") !== -1) {
if (selected[i].values) {
selectedNodes.add(selected[i].values.slice(-1)[0]);
} else {
selectedNodes.add(selected[i].id);
}
}
}
return Array.from(selectedNodes);
}
export function getTreeNodeAttributes(tree) {
const nodes = tree.selectable();
const attributes = {};
for (let i = 0; i < nodes.length; i++) {
let id = null;
if (nodes[i].text.indexOf("(") !== -1 && nodes[i].values) {
id = nodes[i].values.slice(-1)[0];
} else {
id = nodes[i].id
}
attributes[id] = {
checked: nodes[i].itree.state.checked,
collapsed: nodes[i].itree.state.collapsed,
}
}
return attributes;
}
export function serializeMimes(mimes) {
if (mimes.length === 0) {
return undefined;
}
return mimes.map(mime => compressMime(mime)).join("");
}
export function deserializeMimes(mimeString) {
return mimeString
.replaceAll(/([IVATUF])/g, "$$$&")
.split("$")
.map(mime => decompressMime(mime))
.slice(1) // Ignore the first (empty) token
}
export function compressMime(mime) {
return mime
.replace("image/", "I")
.replace("video/", "V")
.replace("application/", "A")
.replace("text/", "T")
.replace("audio/", "U")
.replace("font/", "F")
.replace("+", ",")
.replace("x-", "X")
}
export function decompressMime(mime) {
return mime
.replace("I", "image/")
.replace("V", "video/")
.replace("A", "application/")
.replace("T", "text/")
.replace("U", "audio/")
.replace("F", "font/")
.replace(",", "+")
.replace("X", "x-")
}
export function randomSeed() {
return Math.round(Math.random() * 100000);
}
export function sid(doc) {
if (doc._id.includes(".")) {
return doc._id
}
const num = BigInt(doc._id);
const indexId = (num >> BigInt(32));
const docId = num & BigInt(0xFFFFFFFF);
return indexId.toString(16).padStart(8, "0") + "." + docId.toString(16).padStart(8, "0");
}

177
sist2-vue/src/util.ts
View File

@ -1,177 +0,0 @@
import {EsHit} from "@/Sist2Api";
export function ext(hit: EsHit) {
return srcExt(hit._source)
}
export function srcExt(src) {
return Object.prototype.hasOwnProperty.call(src, "extension")
&& src["extension"] !== "" ? "." + src["extension"] : "";
}
export function strUnescape(str: string): string {
let result = "";
for (let i = 0; i < str.length; i++) {
const c = str[i];
const next = str[i + 1];
if (c === "]") {
if (next === "]") {
result += c;
i += 1;
} else {
result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
i += 2;
}
} else {
result += c;
}
}
return result;
}
const thresh = 1000;
const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
export function humanFileSize(bytes: number): string {
if (bytes === 0) {
return "0 B"
}
if (Math.abs(bytes) < thresh) {
return bytes + ' B';
}
let u = -1;
do {
bytes /= thresh;
++u;
} while (Math.abs(bytes) >= thresh && u < units.length - 1);
return bytes.toFixed(1) + units[u];
}
export function humanTime(sec_num: number): string {
sec_num = Math.floor(sec_num);
const hours = Math.floor(sec_num / 3600);
const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
const seconds = sec_num - (hours * 3600) - (minutes * 60);
if (sec_num < 60) {
return `${sec_num}s`
}
if (sec_num < 3600) {
return `${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
}
return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
}
export function humanDate(numMilis: number): string {
const date = (new Date(numMilis * 1000));
return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
}
export function lum(c: string) {
c = c.substring(1);
const rgb = parseInt(c, 16);
const r = (rgb >> 16) & 0xff;
const g = (rgb >> 8) & 0xff;
const b = (rgb >> 0) & 0xff;
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
}
export function getSelectedTreeNodes(tree: any) {
const selectedNodes = new Set();
const selected = tree.selected();
for (let i = 0; i < selected.length; i++) {
if (selected[i].id === "any") {
return ["any"]
}
//Only get children
if (selected[i].text.indexOf("(") !== -1) {
if (selected[i].values) {
selectedNodes.add(selected[i].values.slice(-1)[0]);
} else {
selectedNodes.add(selected[i].id);
}
}
}
return Array.from(selectedNodes);
}
export function getTreeNodeAttributes(tree: any) {
const nodes = tree.selectable();
const attributes = {};
for (let i = 0; i < nodes.length; i++) {
let id = null;
if (nodes[i].text.indexOf("(") !== -1 && nodes[i].values) {
id = nodes[i].values.slice(-1)[0];
} else {
id = nodes[i].id
}
attributes[id] = {
checked: nodes[i].itree.state.checked,
collapsed: nodes[i].itree.state.collapsed,
}
}
return attributes;
}
export function serializeMimes(mimes: string[]): string | undefined {
if (mimes.length == 0) {
return undefined;
}
return mimes.map(mime => compressMime(mime)).join("");
}
export function deserializeMimes(mimeString: string): string[] {
return mimeString
.replaceAll(/([IVATUF])/g, "$$$&")
.split("$")
.map(mime => decompressMime(mime))
.slice(1) // Ignore the first (empty) token
}
export function compressMime(mime: string): string {
return mime
.replace("image/", "I")
.replace("video/", "V")
.replace("application/", "A")
.replace("text/", "T")
.replace("audio/", "U")
.replace("font/", "F")
.replace("+", ",")
.replace("x-", "X")
}
export function decompressMime(mime: string): string {
return mime
.replace("I", "image/")
.replace("V", "video/")
.replace("A", "application/")
.replace("T", "text/")
.replace("U", "audio/")
.replace("F", "font/")
.replace(",", "+")
.replace("X", "x-")
}
export function randomSeed(): number {
return Math.round(Math.random() * 100000);
}

224
sist2-vue/src/views/FilePage.vue
View File

@ -1,142 +1,142 @@
<template>
<div style="margin-left: auto; margin-right: auto;" class="container">
<Preloader v-if="loading"></Preloader>
<b-card v-else-if="!loading && found">
<b-card-title :title="doc._source.name + ext(doc)">
{{ doc._source.name + ext(doc) }}
</b-card-title>
<div style="margin-left: auto; margin-right: auto;" class="container">
<Preloader v-if="loading"></Preloader>
<b-card v-else-if="!loading && found">
<b-card-title :title="doc._source.name + ext(doc)">
{{ doc._source.name + ext(doc) }}
</b-card-title>
<!-- Thumbnail-->
<div style="position: relative; margin-left: auto; margin-right: auto; text-align: center">
<FullThumbnail :doc="doc" :small-badge="false" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
</div>
<!-- Thumbnail-->
<div style="position: relative; margin-left: auto; margin-right: auto; text-align: center">
<FullThumbnail :doc="doc" :small-badge="false" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
</div>
<!-- Audio player-->
<audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
:type="doc._source.mime"
:src="`f/${doc._source.index}/${doc._id}`"></audio>
<!-- Audio player-->
<audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
:type="doc._source.mime"
:src="`f/${sid(doc)}`"></audio>
<InfoTable :doc="doc" v-if="doc"></InfoTable>
<InfoTable :doc="doc" v-if="doc"></InfoTable>
<div v-if="doc._source.content" class="content-div">{{ doc._source.content }}</div>
</b-card>
<div v-else>
<b-card>
<b-card-title>{{ $t("filePage.notFound") }}</b-card-title>
</b-card>
<div v-if="doc._source.content" class="content-div">{{ doc._source.content }}</div>
</b-card>
<div v-else>
<b-card>
<b-card-title>{{ $t("filePage.notFound") }}</b-card-title>
</b-card>
</div>
</div>
</div>
</template>
<script>
import Preloader from "@/components/Preloader.vue";
import InfoTable from "@/components/InfoTable.vue";
import Sist2Api from "@/Sist2Api";
import {ext} from "@/util";
import {ext, sid} from "@/util";
import Vue from "vue";
import sist2 from "@/Sist2Api";
import FullThumbnail from "@/components/FullThumbnail";
export default Vue.extend({
name: "FilePage",
components: {
FullThumbnail,
Preloader,
InfoTable
},
data() {
return {
loading: true,
found: false,
doc: null
}
},
methods: {
ext: ext,
onThumbnailClick() {
window.open(`/f/${this.doc.index}/${this.doc._id}`, "_blank");
name: "FilePage",
components: {
FullThumbnail,
Preloader,
InfoTable
},
findByCustomField(field, id) {
return {
query: {
bool: {
must: [
{
match: {
[field]: id
}
}
]
}
},
size: 1
}
data() {
return {
loading: true,
found: false,
doc: null
}
},
findById(id) {
return {
query: {
bool: {
must: [
{
match: {
"_id": id
}
}
]
}
methods: {
ext: ext,
sid: sid,
onThumbnailClick() {
window.open(`/f/${sid(this.doc)}`, "_blank");
},
size: 1
}
},
findByName(name) {
return {
query: {
bool: {
must: [
{
match: {
"name": name
}
}
]
}
findByCustomField(field, id) {
return {
query: {
bool: {
must: [
{
match: {
[field]: id
}
}
]
}
},
size: 1
}
},
size: 1
}
}
},
mounted() {
let query = null;
if (this.$route.query.byId) {
query = this.findById(this.$route.query.byId);
} else if (this.$route.query.byName) {
query = this.findByName(this.$route.query.byName);
} else if (this.$route.query.by && this.$route.query.q) {
query = this.findByCustomField(this.$route.query.by, this.$route.query.q)
}
if (query) {
Sist2Api.esQuery(query).then(result => {
if (result.hits.hits.length === 0) {
this.found = false;
} else {
this.doc = result.hits.hits[0];
this.found = true;
findById(id) {
return {
query: {
bool: {
must: [
{
match: {
"_id": id
}
}
]
}
},
size: 1
}
},
findByName(name) {
return {
query: {
bool: {
must: [
{
match: {
"name": name
}
}
]
}
},
size: 1
}
}
this.loading = false;
});
} else {
this.loading = false;
this.found = false;
},
mounted() {
let query = null;
if (this.$route.query.byId) {
query = this.findById(this.$route.query.byId);
} else if (this.$route.query.byName) {
query = this.findByName(this.$route.query.byName);
} else if (this.$route.query.by && this.$route.query.q) {
query = this.findByCustomField(this.$route.query.by, this.$route.query.q)
}
if (query) {
Sist2Api.esQuery(query).then(result => {
if (result.hits.hits.length === 0) {
this.found = false;
} else {
this.doc = result.hits.hits[0];
this.found = true;
}
this.loading = false;
});
} else {
this.loading = false;
this.found = false;
}
}
}
});
</script>
<style scoped>
.img-wrapper {
display: inline-block;
display: inline-block;
}
</style>

17
sist2-vue/src/views/SearchPage.vue
View File

@ -60,6 +60,7 @@
</template>
<script>
import {sid} from "@/util";
import Preloader from "@/components/Preloader.vue";
import {mapActions, mapGetters, mapMutations} from "vuex";
import SearchBar from "@/components/SearchBar.vue";
@ -92,7 +93,6 @@ export default Vue.extend({
SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
},
data: () => ({
loading: false,
uiLoading: true,
search: undefined,
docs: [],
@ -105,6 +105,9 @@ export default Vue.extend({
}),
computed: {
...mapGetters(["indices", "optDisplay"]),
hasEmbeddings() {
return Sist2Api.models().length > 0;
},
},
mounted() {
// Handle touch events
@ -172,12 +175,6 @@ export default Vue.extend({
setDateBoundsMax: "setDateBoundsMax",
setTags: "setTags",
}),
hasEmbeddings() {
if (!this.loading) {
return false;
}
return Sist2Api.models().some();
},
showErrorToast() {
this.$bvToast.toast(
this.$t("toast.esConnErr"),
@ -248,9 +245,9 @@ export default Vue.extend({
if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
hit._seq = await this.$store.dispatch("getKeySequence");
this.$store.commit("addLightboxSource", {
source: `f/${hit._source.index}/${hit._id}`,
thumbnail: hit._props.hasThumbnail
? `t/${hit._source.index}/${hit._id}`
source: `f/${sid(hit)}`,
thumbnail_count: hit._props.hasThumbnail
? `t/${sid(hit)}`
: null,
caption: {
component: LightboxCaption,

40
sist2-vue/tsconfig.json
View File

@ -1,40 +0,0 @@
{
"compilerOptions": {
"target": "esnext",
"module": "esnext",
"strict": false,
"jsx": "preserve",
"importHelpers": true,
"moduleResolution": "node",
"experimentalDecorators": true,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"sourceMap": false,
"baseUrl": ".",
"types": [
"webpack-env",
],
"paths": {
"@/*": [
"src/*"
]
},
"lib": [
"esnext",
"dom",
"dom.iterable",
"scripthost"
]
},
"include": [
"src/**/*.ts",
"src/**/*.tsx",
"src/**/*.vue",
"tests/**/*.ts",
"tests/**/*.tsx"
],
"exclude": [
"node_modules"
]
}

195
src/database/database.c
View File

@ -4,6 +4,7 @@
#include <string.h>
#include <pthread.h>
#include "src/util.h"
#include "src/parsing/mime.h"
#include <time.h>
@ -64,9 +65,11 @@ static int sep_rfind(const char *str) {
}
void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
#ifdef SIST_DEBUG
if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
sqlite3_result_error(ctx, "Invalid parameters", -1);
}
#endif
const char *value = (const char *) sqlite3_value_text(argv[0]);
@ -82,28 +85,27 @@ void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
}
void random_func(sqlite3_context *ctx, int argc, UNUSED(sqlite3_value **argv)) {
#ifdef SIST_DEBUG
if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
sqlite3_result_error(ctx, "Invalid parameters", -1);
}
#endif
char state_buf[128] = {0,};
struct random_data buf;
int result;
char state_buf[8] = {0,};
long seed = sqlite3_value_int64(argv[0]);
initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
initstate((int) seed, state_buf, sizeof(state_buf));
random_r(&buf, &result);
sqlite3_result_int(ctx, result);
sqlite3_result_int(ctx, (int) random());
}
void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
#ifdef SIST_DEBUG
if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
sqlite3_result_error(ctx, "Invalid parameters", -1);
}
#endif
database_ipc_ctx_t *ipc_ctx = sqlite3_user_data(ctx);
@ -146,6 +148,12 @@ void database_open(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA temp_store = memory;", NULL, NULL, NULL));
}
#ifdef SIST_DEBUG
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL));
#else
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA ignore_check_constraints = ON;", NULL, NULL, NULL));
#endif
if (db->type == INDEX_DATABASE) {
// Prepare statements;
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
@ -154,16 +162,15 @@ void database_open(database_t *db) {
&db->select_thumbnail_stmt, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"UPDATE document SET marked=1 WHERE id=? AND mtime=? RETURNING id",
"UPDATE marked SET marked=1 WHERE id=(SELECT ROWID FROM document WHERE path=?) AND mtime=? RETURNING id",
-1,
&db->mark_document_stmt, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"REPLACE INTO document_sidecar (id, json_data) VALUES (?,?)", -1,
&db->write_document_sidecar_stmt, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"REPLACE INTO document (id, mtime, size, json_data, version) VALUES (?, ?, ?, ?, (SELECT max(id) FROM version));",
"INSERT INTO document (path, parent, mime, mtime, size, thumbnail_count, json_data, version) "
"VALUES (?, (SELECT id FROM document WHERE path=?), ?, ?, ?, ?, ?, (SELECT max(id) FROM version)) "
"ON CONFLICT (path) DO UPDATE SET json_data=excluded.json_data "
"RETURNING id;",
-1,
&db->write_document_stmt, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
@ -173,7 +180,12 @@ void database_open(database_t *db) {
&db->write_thumbnail_stmt, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db, "SELECT json_data FROM document WHERE id=?", -1,
db->db, "SELECT json_set(json_data, "
"'$._id', CAST (doc.id AS TEXT),"
"'$.thumbnail', doc.thumbnail_count,"
"'$.mime', m.name,"
"'$.size', doc.size"
") FROM document doc LEFT JOIN mime m ON m.id=doc.mime WHERE doc.id=?", -1,
&db->get_document, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
@ -184,6 +196,12 @@ void database_open(database_t *db) {
db->db, "SELECT embedding FROM embedding WHERE id=? AND model_id=? AND start=0", -1,
&db->get_embedding, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
-1,
&db->write_tag_stmt, NULL));
// Create functions
sqlite3_create_function(
db->db,
@ -228,7 +246,7 @@ void database_open(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"DELETE FROM index_job WHERE id = (SELECT MIN(id) FROM index_job)"
" RETURNING doc_id,type,line;",
" RETURNING sid,type,line;",
-1, &db->pop_index_job_stmt, NULL
));
@ -243,7 +261,7 @@ void database_open(database_t *db) {
db->db, "INSERT INTO parse_job (filepath,mtime,st_size) VALUES (?,?,?);", -1,
&db->insert_parse_job_stmt, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db, "INSERT INTO index_job (doc_id,type,line) VALUES (?,?,?);", -1,
db->db, "INSERT INTO index_job (sid,type,line) VALUES (?,?,?);", -1,
&db->insert_index_job_stmt, NULL));
} else if (db->type == FTS_DATABASE) {
@ -294,6 +312,12 @@ void database_open(database_t *db) {
db->db, "SELECT mime, sum(count) FROM mime_index WHERE mime is not NULL GROUP BY mime", -1,
&db->fts_get_mimetypes, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"INSERT INTO tag (id, index_id, tag) VALUES (?,?,?) ON CONFLICT DO NOTHING;",
-1,
&db->fts_write_tag_stmt, NULL));
sqlite3_create_function(
db->db,
"random_seeded",
@ -340,13 +364,6 @@ void database_open(database_t *db) {
}
if (db->type == FTS_DATABASE || db->type == INDEX_DATABASE) {
// Tag table is the same schema for FTS database & index database
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
-1,
&db->write_tag_stmt, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db,
"DELETE FROM tag WHERE id=? AND tag=?;",
@ -376,8 +393,8 @@ void database_close(database_t *db, int optimize) {
db = NULL;
}
void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len) {
sqlite3_bind_text(db->select_thumbnail_stmt, 1, id, -1, SQLITE_STATIC);
void *database_read_thumbnail(database_t *db, int doc_id, int num, size_t *return_value_len) {
sqlite3_bind_int(db->select_thumbnail_stmt, 1, doc_id);
sqlite3_bind_int(db->select_thumbnail_stmt, 2, num);
int ret = sqlite3_step(db->select_thumbnail_stmt);
@ -410,7 +427,7 @@ void database_write_index_descriptor(database_t *db, index_descriptor_t *desc) {
sqlite3_prepare_v2(db->db, "INSERT INTO descriptor (id, version_major, version_minor, version_patch,"
" root, name, rewrite_url, timestamp) VALUES (?,?,?,?,?,?,?,?);", -1, &stmt, NULL);
sqlite3_bind_text(stmt, 1, desc->id, -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 1, desc->id);
sqlite3_bind_int(stmt, 2, desc->version_major);
sqlite3_bind_int(stmt, 3, desc->version_minor);
sqlite3_bind_int(stmt, 4, desc->version_patch);
@ -433,7 +450,7 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
const char *id = (char *) sqlite3_column_text(stmt, 0);
int id = sqlite3_column_int(stmt, 0);
int v_major = sqlite3_column_int(stmt, 1);
int v_minor = sqlite3_column_int(stmt, 2);
int v_patch = sqlite3_column_int(stmt, 3);
@ -443,7 +460,7 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
int timestamp = sqlite3_column_int(stmt, 7);
index_descriptor_t *desc = malloc(sizeof(index_descriptor_t));
strcpy(desc->id, id);
desc->id = id;
snprintf(desc->version, sizeof(desc->version), "%d.%d.%d", v_major, v_minor, v_patch);
desc->version_major = v_major;
desc->version_minor = v_minor;
@ -461,7 +478,8 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
database_iterator_t *database_create_delete_list_iterator(database_t *db) {
sqlite3_stmt *stmt;
sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list;", -1, &stmt, NULL);
sqlite3_prepare_v2(db->db, "SELECT doc.id FROM delete_list "
"INNER JOIN document doc ON doc.ROWID = delete_list.id;", -1, &stmt, NULL);
database_iterator_t *iter = malloc(sizeof(database_iterator_t));
@ -471,14 +489,11 @@ database_iterator_t *database_create_delete_list_iterator(database_t *db) {
return iter;
}
char *database_delete_list_iter(database_iterator_t *iter) {
int database_delete_list_iter(database_iterator_t *iter) {
int ret = sqlite3_step(iter->stmt);
if (ret == SQLITE_ROW) {
const char *id = (const char *) sqlite3_column_text(iter->stmt, 0);
char *id_heap = malloc(strlen(id) + 1);
strcpy(id_heap, id);
return id_heap;
return sqlite3_column_int(iter->stmt, 0);
}
if (ret != SQLITE_DONE) {
@ -491,7 +506,7 @@ char *database_delete_list_iter(database_iterator_t *iter) {
iter->stmt = NULL;
return NULL;
return 0;
}
database_iterator_t *database_create_document_iterator(database_t *db) {
@ -507,12 +522,16 @@ database_iterator_t *database_create_document_iterator(database_t *db) {
" '$._id', document.id, "
" '$.size', document.size, "
" '$.mtime', document.mtime, "
" '$.mime', mim.name,"
" '$.thumbnail', document.thumbnail_count, "
" '$.tag', json_group_array((SELECT tag FROM tag WHERE document.id = tag.id)))"
" ELSE"
" json_set(document.json_data,"
" '$._id', document.id,"
" '$.size', document.size,"
" '$.mtime', document.mtime,"
" '$.mime', mim.name,"
" '$.thumbnail', document.thumbnail_count, "
" '$.tag', json_group_array((SELECT tag FROM tag WHERE document.id = tag.id)),"
" '$.emb', json_group_object(m.path, json(emb_to_json(emb.embedding))),"
" '$.embedding', 1)"
@ -520,6 +539,7 @@ database_iterator_t *database_create_document_iterator(database_t *db) {
" FROM document"
" LEFT JOIN embedding emb ON document.id = emb.id"
" LEFT JOIN model m ON emb.model_id = m.id"
" LEFT JOIN mime mim ON mim.id = document.mime"
" GROUP BY document.id)"
" SELECT json_set(j, '$.index', (SELECT id FROM descriptor)) FROM doc",
-1, &stmt, NULL));
@ -573,43 +593,48 @@ cJSON *database_document_iter(database_iterator_t *iter) {
cJSON *database_incremental_scan_begin(database_t *db) {
LOG_DEBUG("database.c", "Preparing database for incremental scan");
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "UPDATE document SET marked=0;", NULL, NULL, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM marked;", NULL, NULL, NULL));
CRASH_IF_NOT_SQLITE_OK(
sqlite3_exec(db->db, "INSERT INTO marked SELECT ROWID, 0, mtime FROM document;", NULL, NULL, NULL));
}
cJSON *database_incremental_scan_end(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"DELETE FROM delete_list WHERE id IN (SELECT id FROM document WHERE marked=1);",
"DELETE FROM delete_list WHERE id IN (SELECT id FROM marked WHERE marked = 1);",
NULL, NULL, NULL
));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"DELETE FROM thumbnail WHERE id IN (SELECT id FROM document WHERE marked=0);",
"DELETE FROM thumbnail WHERE EXISTS ("
" SELECT document.id FROM document INNER JOIN marked m ON m.id = document.ROWID"
" WHERE marked=0 and document.id = thumbnail.id)",
NULL, NULL, NULL
));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"INSERT INTO delete_list (id) SELECT id FROM document WHERE marked=0 ON CONFLICT DO NOTHING;",
"INSERT INTO delete_list (id) "
"SELECT id FROM marked WHERE marked=0 ON CONFLICT DO NOTHING;",
NULL, NULL, NULL
));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"DELETE FROM document_sidecar WHERE id IN (SELECT id FROM document WHERE marked=0);",
"DELETE FROM document WHERE ROWID IN (SELECT id FROM marked WHERE marked=0);",
NULL, NULL, NULL
));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"DELETE FROM document WHERE marked=0;",
"DELETE FROM marked;",
NULL, NULL, NULL
));
}
int database_mark_document(database_t *db, const char *id, int mtime) {
sqlite3_bind_text(db->mark_document_stmt, 1, id, -1, SQLITE_STATIC);
int database_mark_document(database_t *db, const char *path, int mtime) {
sqlite3_bind_text(db->mark_document_stmt, 1, path, -1, SQLITE_STATIC);
sqlite3_bind_int(db->mark_document_stmt, 2, mtime);
pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
@ -631,31 +656,38 @@ int database_mark_document(database_t *db, const char *id, int mtime) {
CRASH_IF_STMT_FAIL(ret);
}
void database_write_document(database_t *db, document_t *doc, const char *json_data) {
sqlite3_bind_text(db->write_document_stmt, 1, doc->doc_id, -1, SQLITE_STATIC);
sqlite3_bind_int(db->write_document_stmt, 2, doc->mtime);
sqlite3_bind_int64(db->write_document_stmt, 3, (long) doc->size);
sqlite3_bind_text(db->write_document_stmt, 4, json_data, -1, SQLITE_STATIC);
int database_write_document(database_t *db, document_t *doc, const char *json_data) {
const char *rel_path = doc->filepath + ScanCtx.index.desc.root_len;
const char *parent_rel_path = doc->parent[0] != '\0'
? doc->parent + ScanCtx.index.desc.root_len
: NULL;
// path, parent, mtime, size, json_data
sqlite3_bind_text(db->write_document_stmt, 1, rel_path, -1, SQLITE_STATIC);
sqlite3_bind_text(db->write_document_stmt, 2, parent_rel_path, -1, SQLITE_STATIC);
sqlite3_bind_int64(db->write_document_stmt, 3, doc->mime);
sqlite3_bind_int(db->write_document_stmt, 4, doc->mtime);
sqlite3_bind_int64(db->write_document_stmt, 5, (long) doc->size);
sqlite3_bind_int(db->write_document_stmt, 6, doc->thumbnail_count);
if (json_data) {
sqlite3_bind_text(db->write_document_stmt, 7, json_data, -1, SQLITE_STATIC);
} else {
sqlite3_bind_null(db->write_document_stmt, 7);
}
pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_stmt));
int id = sqlite3_column_int(db->write_document_stmt, 0);
CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_stmt));
pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
return id;
}
void database_write_document_sidecar(database_t *db, const char *id, const char *json_data) {
sqlite3_bind_text(db->write_document_sidecar_stmt, 1, id, -1, SQLITE_STATIC);
sqlite3_bind_text(db->write_document_sidecar_stmt, 2, json_data, -1, SQLITE_STATIC);
pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_sidecar_stmt));
CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_sidecar_stmt));
pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
}
void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size) {
sqlite3_bind_text(db->write_thumbnail_stmt, 1, id, -1, SQLITE_STATIC);
void database_write_thumbnail(database_t *db, int doc_id, int num, void *data, size_t data_size) {
sqlite3_bind_int(db->write_thumbnail_stmt, 1, doc_id);
sqlite3_bind_int(db->write_thumbnail_stmt, 2, num);
sqlite3_bind_blob(db->write_thumbnail_stmt, 3, data, (int) data_size, SQLITE_STATIC);
@ -716,7 +748,7 @@ job_t *database_get_work(database_t *db, job_type_t job_type) {
} else {
job->bulk_line = malloc(sizeof(es_bulk_line_t));
}
strcpy(job->bulk_line->doc_id, (const char *) sqlite3_column_text(db->pop_index_job_stmt, 0));
strcpy(job->bulk_line->sid, (const char *) sqlite3_column_text(db->pop_index_job_stmt, 0));
job->bulk_line->type = sqlite3_column_int(db->pop_index_job_stmt, 1);
job->bulk_line->next = NULL;
@ -767,7 +799,7 @@ void database_add_work(database_t *db, job_t *job) {
} while (ret != SQLITE_DONE && ret != SQLITE_OK);
} else if (job->type == JOB_BULK_LINE) {
do {
sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->doc_id, -1, SQLITE_STATIC);
sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->sid, -1, SQLITE_STATIC);
sqlite3_bind_int(db->insert_index_job_stmt, 2, job->bulk_line->type);
if (job->bulk_line->type != ES_BULK_LINE_DELETE) {
sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC);
@ -808,24 +840,25 @@ void database_add_work(database_t *db, job_t *job) {
pthread_mutex_unlock(&db->ipc_ctx->mutex);
}
void database_write_tag(database_t *db, char *doc_id, char *tag) {
sqlite3_bind_text(db->write_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
sqlite3_bind_text(db->write_tag_stmt, 2, tag, -1, SQLITE_STATIC);
void database_write_tag(database_t *db, long sid, char *tag) {
sqlite3_bind_int64(db->write_tag_stmt, 1, sid);
sqlite3_bind_int(db->write_tag_stmt, 2, (int) (sid >> 32));
sqlite3_bind_text(db->write_tag_stmt, 3, tag, -1, SQLITE_STATIC);
CRASH_IF_STMT_FAIL(sqlite3_step(db->write_tag_stmt));
CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_tag_stmt));
}
void database_delete_tag(database_t *db, char *doc_id, char *tag) {
sqlite3_bind_text(db->delete_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
void database_delete_tag(database_t *db, long sid, char *tag) {
sqlite3_bind_int64(db->delete_tag_stmt, 1, sid);
sqlite3_bind_text(db->delete_tag_stmt, 2, tag, -1, SQLITE_STATIC);
CRASH_IF_STMT_FAIL(sqlite3_step(db->delete_tag_stmt));
CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->delete_tag_stmt));
}
cJSON *database_get_document(database_t *db, char *doc_id) {
sqlite3_bind_text(db->get_document, 1, doc_id, -1, SQLITE_STATIC);
cJSON *database_get_document(database_t *db, int doc_id) {
sqlite3_bind_int(db->get_document, 1, doc_id);
int ret = sqlite3_step(db->get_document);
CRASH_IF_STMT_FAIL(ret);
@ -833,7 +866,7 @@ cJSON *database_get_document(database_t *db, char *doc_id) {
cJSON *json;
if (ret == SQLITE_ROW) {
const char *json_str = sqlite3_column_text(db->get_document, 0);
const char *json_str = (char *) sqlite3_column_text(db->get_document, 0);
json = cJSON_Parse(json_str);
} else {
json = NULL;
@ -847,4 +880,24 @@ cJSON *database_get_document(database_t *db, char *doc_id) {
void database_increment_version(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db, "INSERT INTO version DEFAULT VALUES", NULL, NULL, NULL));
}
void database_sync_mime_table(database_t *db) {
unsigned int *cur = get_mime_ids();
sqlite3_stmt *stmt;
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare(
db->db,
"REPLACE INTO mime (id, name) VALUES (?,?)", -1, &stmt, NULL));
while (*cur != 0) {
sqlite3_bind_int64(stmt, 1, (long) *cur);
sqlite3_bind_text(stmt, 2, mime_get_mime_text(*cur), -1, NULL);
CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
sqlite3_reset(stmt);
cur += 1;
}
sqlite3_finalize(stmt);
}

32
src/database/database.h
View File

@ -81,7 +81,6 @@ typedef struct database {
sqlite3_stmt *mark_document_stmt;
sqlite3_stmt *write_document_stmt;
sqlite3_stmt *write_document_sidecar_stmt;
sqlite3_stmt *write_thumbnail_stmt;
sqlite3_stmt *get_document;
sqlite3_stmt *get_models;
@ -103,6 +102,7 @@ typedef struct database {
sqlite3_stmt *fts_get_document;
sqlite3_stmt *fts_suggest_tag;
sqlite3_stmt *fts_get_tags;
sqlite3_stmt *fts_write_tag_stmt;
sqlite3_stmt *fts_model_size;
@ -133,15 +133,15 @@ void database_close(database_t *, int optimize);
void database_increment_version(database_t *db);
void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size);
void database_write_thumbnail(database_t *db, int doc_id, int num, void *data, size_t data_size);
void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len);
void *database_read_thumbnail(database_t *db, int doc_id, int num, size_t *return_value_len);
void database_write_index_descriptor(database_t *db, index_descriptor_t *desc);
index_descriptor_t *database_read_index_descriptor(database_t *db);
void database_write_document(database_t *db, document_t *doc, const char *json_data);
int database_write_document(database_t *db, document_t *doc, const char *json_data);
database_iterator_t *database_create_document_iterator(database_t *db);
@ -154,10 +154,10 @@ cJSON *database_document_iter(database_iterator_t *);
database_iterator_t *database_create_delete_list_iterator(database_t *db);
char *database_delete_list_iter(database_iterator_t *iter);
int database_delete_list_iter(database_iterator_t *iter);
#define database_delete_list_iter_foreach(element, iter) \
for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
for (int (element) = database_delete_list_iter(iter); (element) != 0; (element) = database_delete_list_iter(iter))
cJSON *database_incremental_scan_begin(database_t *db);
@ -166,8 +166,6 @@ cJSON *database_incremental_scan_end(database_t *db);
int database_mark_document(database_t *db, const char *id, int mtime);
void database_write_document_sidecar(database_t *db, const char *id, const char *json_data);
database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold);
treemap_row_t database_treemap_iter(database_iterator_t *iter);
@ -206,7 +204,7 @@ void database_fts_index(database_t *db);
void database_fts_optimize(database_t *db);
cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
cJSON *database_fts_get_paths(database_t *db, int index_id, int depth_min, int depth_max, const char *prefix,
int suggest);
cJSON *database_fts_get_mimetypes(database_t *db);
@ -215,18 +213,20 @@ database_summary_stats_t database_fts_get_date_range(database_t *db);
cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
long size_max, long date_min, long date_max, int page_size,
char **index_ids, char **mime_types, char **tags, int sort_asc,
int *index_ids, char **mime_types, char **tags, int sort_asc,
fts_sort_t sort, int seed, char **after, int fetch_aggregations,
int highlight, int highlight_context_size, int model,
const float *embedding, int embedding_size);
void database_write_tag(database_t *db, char *doc_id, char *tag);
void database_write_tag(database_t *db, long sid, char *tag);
void database_delete_tag(database_t *db, char *doc_id, char *tag);
void database_fts_write_tag(database_t *db, long sid, char *tag);
void database_delete_tag(database_t *db, long sid, char *tag);
void database_fts_detach(database_t *db);
cJSON *database_fts_get_document(database_t *db, char *doc_id);
cJSON *database_fts_get_document(database_t *db, long sid);
database_summary_stats_t database_fts_sync_tags(database_t *db);
@ -234,7 +234,7 @@ cJSON *database_fts_suggest_tag(database_t *db, char *prefix);
cJSON *database_fts_get_tags(database_t *db);
cJSON *database_get_document(database_t *db, char *doc_id);
cJSON *database_get_document(database_t *db, int doc_id);
void cosine_sim_func(sqlite3_context *ctx, int argc, sqlite3_value **argv);
@ -242,6 +242,8 @@ cJSON *database_get_models(database_t *db);
int database_fts_get_model_size(database_t *db, int model_id);
cJSON *database_get_embedding(database_t *db, char *doc_id, int model_id);
cJSON *database_get_embedding(database_t *db, int doc_id, int model_id);
void database_sync_mime_table(database_t *db);
#endif

4
src/database/database_embeddings.c
View File

@ -69,9 +69,9 @@ cJSON *database_get_models(database_t *db) {
return json;
}
cJSON *database_get_embedding(database_t *db, char *doc_id, int model_id) {
cJSON *database_get_embedding(database_t *db, int doc_id, int model_id) {
sqlite3_bind_text(db->get_embedding, 1, doc_id, -1, SQLITE_STATIC);
sqlite3_bind_int(db->get_embedding, 1, doc_id);
sqlite3_bind_int(db->get_embedding, 2, model_id);
int ret = sqlite3_step(db->get_embedding);
CRASH_IF_STMT_FAIL(ret);

84
src/database/database_fts.c
View File

@ -42,21 +42,23 @@ void database_fts_index(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"WITH docs AS ("
" SELECT document.id as id, (SELECT id FROM descriptor) as index_id, size,"
" SELECT "
" ((SELECT id FROM descriptor) << 32) | document.id as id,"
" (SELECT id FROM descriptor) as index_id,"
" size,"
" document.json_data ->> 'name' as name,"
" document.json_data ->> 'path' as path,"
" mtime,"
" document.json_data ->> 'mime' as mime,"
" json_set(document.json_data, "
" '$._id',document.id,"
" '$.size',document.size, "
" '$.mtime',document.mtime)"
" m.name as mime,"
" thumbnail_count,"
" document.json_data"
" FROM document"
" LEFT JOIN mime m ON m.id=document.mime"
" )"
" INSERT"
" INTO fts.document_index (id, index_id, size, name, path, mtime, mime, json_data)"
" INTO fts.document_index (id, index_id, size, name, path, mtime, mime, thumbnail_count, json_data)"
" SELECT * FROM docs WHERE true"
" on conflict (id, index_id) do update set "
" on conflict (id) do update set "
" size=excluded.size, mtime=excluded.mtime, mime=excluded.mime, json_data=excluded.json_data;",
NULL, NULL, NULL));
@ -64,13 +66,14 @@ void database_fts_index(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"REPLACE INTO fts.embedding (id, model_id, start, end, embedding)"
" SELECT id, model_id, start, end, embedding FROM embedding", NULL, NULL, NULL));
"REPLACE INTO fts.model (id, size)"
" SELECT id, size FROM model", NULL, NULL, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"INSERT INTO fts.model (id, size)"
" SELECT id, size FROM model WHERE TRUE ON CONFLICT (id) DO NOTHING", NULL, NULL, NULL));
"REPLACE INTO fts.embedding (id, model_id, start, end, embedding)"
" SELECT (SELECT id FROM descriptor) << 32 | id, model_id, start, end, embedding FROM embedding "
" WHERE TRUE ON CONFLICT (id, model_id, start) DO NOTHING;", NULL, NULL, NULL));
// TODO: delete old embeddings
@ -172,7 +175,7 @@ void database_fts_optimize(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA fts.optimize;", NULL, NULL, NULL));
}
cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
cJSON *database_fts_get_paths(database_t *db, int index_id, int depth_min, int depth_max, const char *prefix,
int suggest) {
sqlite3_stmt *stmt;
@ -192,7 +195,7 @@ cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_mi
} else if (prefix) {
stmt = db->fts_search_paths_w_prefix;
if (index_id) {
sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 1, index_id);
} else {
sqlite3_bind_null(stmt, 1);
}
@ -207,7 +210,7 @@ cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_mi
} else {
stmt = db->fts_search_paths;
if (index_id) {
sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 1, index_id);
} else {
sqlite3_bind_null(stmt, 1);
}
@ -290,7 +293,6 @@ const char *date_where_clause(long date_min, long date_max) {
}
int array_length(char **arr) {
if (arr == NULL) {
return 0;
}
@ -301,6 +303,17 @@ int array_length(char **arr) {
return count;
}
int int_array_length(const int *arr) {
if (arr == NULL) {
return 0;
}
int count = -1;
while (arr[++count] != 0);
return count;
}
#define INDEX_ID_PARAM_OFFSET (10)
#define MIME_PARAM_OFFSET (INDEX_ID_PARAM_OFFSET + 1000)
@ -351,8 +364,8 @@ char *build_where_clause(const char *path_where, const char *size_where, const c
return where;
}
char *index_ids_where_clause(char **index_ids) {
int param_count = array_length(index_ids);
char *index_ids_where_clause(int *index_ids) {
int param_count = int_array_length(index_ids);
char *clause = malloc(13 + 2 + 6 * param_count);
@ -483,7 +496,7 @@ int database_fts_get_model_size(database_t *db, int model_id) {
cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
long size_max, long date_min, long date_max, int page_size,
char **index_ids, char **mime_types, char **tags, int sort_asc,
int *index_ids, char **mime_types, char **tags, int sort_asc,
fts_sort_t sort, int seed, char **after, int fetch_aggregations,
int highlight, int highlight_context_size, int model,
const float *embedding, int embedding_size) {
@ -524,13 +537,21 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
const char *json_object_sql;
if (highlight && query_where != NULL) {
json_object_sql = "json_set(json_remove(doc.json_data, '$.content'),"
"'$._id', CAST(doc.id AS TEXT),"
"'$.index', doc.index_id,"
"'$.thumbnail', doc.thumbnail_count,"
"'$.mime', doc.mime,"
"'$.size', doc.size,"
"'$.embedding', (CASE WHEN emb.id IS NOT NULL THEN 1 ELSE 0 END),"
"'$._highlight.name', snippet(search, 0, '<mark>', '</mark>', '', ?6),"
"'$._highlight.content', snippet(search, 1, '<mark>', '</mark>', '', ?6))";
} else {
json_object_sql = "json_set(json_remove(doc.json_data, '$.content'),"
"'$._id', CAST(doc.id AS TEXT),"
"'$.index', doc.index_id,"
"'$.thumbnail', doc.thumbnail_count,"
"'$.mime', doc.mime,"
"'$.size', doc.size,"
"'$.embedding', (CASE WHEN emb.id IS NOT NULL THEN 1 ELSE 0 END))";
}
@ -592,7 +613,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
if (index_ids) {
array_foreach(index_ids) {
sqlite3_bind_text(stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i]);
}
}
if (mime_types) {
@ -692,7 +713,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
if (index_ids) {
array_foreach(index_ids) {
sqlite3_bind_text(agg_stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
sqlite3_bind_int(agg_stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i]);
}
}
if (mime_types) {
@ -764,19 +785,20 @@ database_summary_stats_t database_fts_sync_tags(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"DELETE FROM fts.tag WHERE"
" (id, tag) NOT IN (SELECT id, tag FROM tag)",
" (id, index_id, tag) NOT IN (SELECT ((SELECT id FROM descriptor) << 32) | id, (SELECT id FROM descriptor), tag FROM tag)"
" AND index_id = (SELECT id FROM descriptor)",
NULL, NULL, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"INSERT INTO fts.tag (id, tag) "
" SELECT id, tag FROM tag "
" WHERE (id, tag) NOT IN (SELECT * FROM fts.tag)",
"INSERT INTO fts.tag (id, index_id, tag) "
" SELECT (((SELECT id FROM descriptor) << 32) | id) as sid, (SELECT id FROM descriptor), tag FROM tag "
" WHERE (sid, tag) NOT IN (SELECT id, tag FROM fts.tag)",
NULL, NULL, NULL));
}
cJSON *database_fts_get_document(database_t *db, char *doc_id) {
sqlite3_bind_text(db->fts_get_document, 1, doc_id, -1, NULL);
cJSON *database_fts_get_document(database_t *db, long sid) {
sqlite3_bind_int64(db->fts_get_document, 1, sid);
int ret = sqlite3_step(db->fts_get_document);
cJSON *json = NULL;
@ -844,3 +866,11 @@ cJSON *database_fts_get_tags(database_t *db) {
return json;
}
void database_fts_write_tag(database_t *db, long sid, char *tag) {
sqlite3_bind_int64(db->fts_write_tag_stmt, 1, sid);
sqlite3_bind_int(db->fts_write_tag_stmt, 2, (int) (sid >> 32));
sqlite3_bind_text(db->fts_write_tag_stmt, 3, tag, -1, SQLITE_STATIC);
CRASH_IF_STMT_FAIL(sqlite3_step(db->fts_write_tag_stmt));
CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->fts_write_tag_stmt));
}

108
src/database/database_schema.c
View File

@ -1,57 +1,63 @@
#ifdef SIST_DEBUG
#define STRICT " STRICT"
#else
#define STRICT ""
#endif
const char *FtsDatabaseSchema =
"CREATE TABLE IF NOT EXISTS document_index ("
" id TEXT NOT NULL,"
" index_id TEXT NOT NULL,"
" id INTEGER PRIMARY KEY,"
" index_id INTEGER NOT NULL,"
" size INTEGER NOT NULL,"
" name TEXT NOT NULL,"
" path TEXT NOT NULL,"
" mtime INTEGER NOT NULL,"
" mime TEXT,"
" json_data TEXT NOT NULL,"
" PRIMARY KEY (id, index_id)"
");"
" thumbnail_count INTEGER NOT NULL,"
" json_data TEXT NOT NULL"
")"STRICT";"
""
"CREATE TABLE IF NOT EXISTS stats ("
" mtime_min INTEGER,"
" mtime_max INTEGER"
");"
")"STRICT";"
""
"CREATE TABLE IF NOT EXISTS path_index ("
" path TEXT,"
" index_id TEXT,"
" index_id INTEGER,"
" count INTEGER NOT NULL,"
" depth INTEGER NOT NULL,"
" PRIMARY KEY (path, index_id)"
");"
")"STRICT";"
""
"CREATE TABLE IF NOT EXISTS mime_index ("
" index_id TEXT,"
" index_id INTEGER,"
" mime TEXT,"
" count INT,"
" count INTEGER,"
" PRIMARY KEY(index_id, mime)"
");"
")"STRICT";"
""
"CREATE TABLE IF NOT EXISTS tag ("
" id TEXT NOT NULL,"
" id INTEGER NOT NULL,"
" index_id INTEGER NOT NULL,"
" tag TEXT NOT NULL,"
" PRIMARY KEY (id, tag)"
");"
")"STRICT";"
"CREATE INDEX IF NOT EXISTS tag_tag_idx ON tag(tag);"
"CREATE INDEX IF NOT EXISTS tag_id_idx ON tag(id);"
""
"CREATE TABLE IF NOT EXISTS embedding ("
" id TEXT REFERENCES document(id),"
" id INTEGER REFERENCES document_index(id),"
" model_id INTEGER NOT NULL REFERENCES model(id),"
" start INTEGER NOT NULL,"
" end INTEGER,"
" embedding BLOB NOT NULL,"
" PRIMARY KEY (id, model_id, start)"
");"
")"STRICT";"
""
"CREATE TABLE IF NOT EXISTS model ("
" id INTEGER PRIMARY KEY CHECK (id > 0 AND id < 1000),"
" size INTEGER NOT NULL"
");"
")"STRICT";"
""
"CREATE TRIGGER IF NOT EXISTS tag_write_trigger"
" AFTER INSERT ON tag"
@ -71,7 +77,7 @@ const char *FtsDatabaseSchema =
""
"CREATE VIEW IF NOT EXISTS document_view (id, name, content, title)"
" AS"
" SELECT rowid,"
" SELECT id,"
" json_data->>'name',"
" json_data->>'content',"
" json_data->>'title'"
@ -94,18 +100,18 @@ const char *IpcDatabaseSchema =
" filepath TEXT NOT NULL,"
" mtime INTEGER NOT NULL,"
" st_size INTEGER NOT NULL"
");"
")"STRICT";"
""
"CREATE TABLE index_job ("
" id INTEGER PRIMARY KEY,"
" doc_id TEXT NOT NULL CHECK ( length(doc_id) = 32 ),"
" sid TEXT NOT NULL,"
" type INTEGER NOT NULL,"
" line TEXT"
");";
")"STRICT";";
const char *IndexDatabaseSchema =
"CREATE TABLE thumbnail ("
" id TEXT NOT NULL CHECK ( length(id) = 32 ),"
" id INTEGER REFERENCES document(id),"
" num INTEGER NOT NULL,"
" data BLOB NOT NULL,"
" PRIMARY KEY(id, num)"
@ -114,34 +120,46 @@ const char *IndexDatabaseSchema =
"CREATE TABLE version ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" date TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP)"
");"
")"STRICT";"
""
"CREATE TABLE mime("
" id INTEGER PRIMARY KEY,"
" name TEXT"
")"STRICT";"
"CREATE UNIQUE INDEX mime_name_idx ON mime(name);"
""
"CREATE TABLE document ("
" id TEXT PRIMARY KEY CHECK ( length(id) = 32 ),"
" marked INTEGER NOT NULL DEFAULT (1),"
" id INTEGER PRIMARY KEY,"
" parent INTEGER REFERENCES document(id),"
" mime INTEGER REFERENCES mime(id),"
" path TEXT NOT NULL,"
" version INTEGER NOT NULL REFERENCES version(id),"
" mtime INTEGER NOT NULL,"
" size INTEGER NOT NULL,"
" json_data TEXT NOT NULL CHECK ( json_valid(json_data) )"
") WITHOUT ROWID;"
" thumbnail_count INTEGER NOT NULL,"
" json_data TEXT CHECK ( json_data IS NULL OR json_valid(json_data) )"
")"STRICT";"
"CREATE UNIQUE INDEX document_path_idx ON document(path);"
"CREATE TABLE marked ("
" id INTEGER PRIMARY KEY,"
" marked INTEGER NOT NULL,"
" mtime INTEGER NOT NULL"
")"STRICT";"
""
"CREATE INDEX marked_marked ON marked(marked);"
""
"CREATE TABLE delete_list ("
" id TEXT PRIMARY KEY CHECK ( length(id) = 32 )"
") WITHOUT ROWID;"
" id INTEGER PRIMARY KEY"
")"STRICT";"
""
"CREATE TABLE tag ("
" id TEXT NOT NULL,"
" id INTEGER NOT NULL REFERENCES document(id),"
" tag TEXT NOT NULL,"
" PRIMARY KEY (id, tag)"
");"
""
"CREATE TABLE document_sidecar ("
" id TEXT PRIMARY KEY NOT NULL,"
" json_data TEXT NOT NULL"
") WITHOUT ROWID;"
")"STRICT";"
""
"CREATE TABLE descriptor ("
" id TEXT NOT NULL,"
" id INTEGER PRIMARY KEY,"
" version_major INTEGER NOT NULL,"
" version_minor INTEGER NOT NULL,"
" version_patch INTEGER NOT NULL,"
@ -149,37 +167,37 @@ const char *IndexDatabaseSchema =
" name TEXT NOT NULL,"
" rewrite_url TEXT,"
" timestamp INTEGER NOT NULL"
");"
")"STRICT";"
""
"CREATE TABLE stats_treemap ("
" path TEXT NOT NULL,"
" size INTEGER NOT NULL"
");"
")"STRICT";"
""
"CREATE TABLE stats_size_agg ("
" bucket INTEGER NOT NULL,"
" count INTEGER NOT NULL"
");"
")"STRICT";"
""
"CREATE TABLE stats_date_agg ("
" bucket INTEGER NOT NULL,"
" count INTEGER NOT NULL"
");"
")"STRICT";"
""
"CREATE TABLE stats_mime_agg ("
" mime TEXT NOT NULL,"
" size INTEGER NOT NULL,"
" count INTEGER NOT NULL"
");"
")"STRICT";"
""
"CREATE TABLE embedding ("
" id TEXT REFERENCES document(id),"
" id INTEGER REFERENCES document(id),"
" model_id INTEGER NOT NULL references model(id),"
" start INTEGER NOT NULL,"
" end INTEGER,"
" embedding BLOB NOT NULL,"
" PRIMARY KEY (id, model_id, start)"
");"
")"STRICT";"
""
"CREATE TABLE model ("
" id INTEGER PRIMARY KEY CHECK (id > 0 AND id < 1000),"
@ -188,5 +206,5 @@ const char *IndexDatabaseSchema =
" path TEXT NOT NULL UNIQUE,"
" size INTEGER NOT NULL,"
" type TEXT NOT NULL CHECK ( type IN ('flat', 'nested') )"
");";
")"STRICT";";

8
src/database/database_stats.c
View File

@ -98,10 +98,10 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
// mime aggregation
sqlite3_prepare_v2(db->db, "INSERT INTO stats_mime_agg"
" SELECT"
" (json_data->>'mime') as bucket,"
" m.name as bucket,"
" sum(size),"
" count(*)"
" FROM document"
" FROM document INNER JOIN mime m ON m.id=document.mime"
" WHERE bucket IS NOT NULL"
" GROUP BY bucket", -1, &stmt, NULL);
CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
@ -117,8 +117,8 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
// flat map
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db,
"INSERT INTO tm (path, size) SELECT json_data->>'path' as path, sum(size)"
" FROM document WHERE json_data->>'parent' IS NULL GROUP BY path;",
"INSERT INTO tm (path, size) SELECT path, sum(size)"
" FROM document WHERE parent IS NULL GROUP BY path;",
NULL, NULL, NULL));
// Merge up

20
src/index/elastic.c
View File

@ -47,7 +47,7 @@ void elastic_cleanup() {
destroy_indexer(Indexer);
}
void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
void print_json(cJSON *document, const char id_str[SIST_SID_LEN]) {
cJSON *line = cJSON_CreateObject();
@ -64,12 +64,12 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
cJSON_Delete(line);
}
void delete_document(const char *document_id) {
void delete_document(const char *sid) {
es_bulk_line_t bulk_line;
bulk_line.type = ES_BULK_LINE_DELETE;
bulk_line.next = NULL;
strcpy(bulk_line.doc_id, document_id);
strcpy(bulk_line.sid, sid);
tpool_add_work(IndexCtx.pool, &(job_t) {
.type = JOB_BULK_LINE,
@ -78,14 +78,14 @@ void delete_document(const char *document_id) {
}
void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) {
void index_json(cJSON *document, const char doc_id[SIST_SID_LEN]) {
char *json = cJSON_PrintUnformatted(document);
size_t json_len = strlen(json);
es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t) + json_len + 2);
bulk_line->type = ES_BULK_LINE_INDEX;
memcpy(bulk_line->line, json, json_len);
strcpy(bulk_line->doc_id, doc_id);
strcpy(bulk_line->sid, doc_id);
*(bulk_line->line + json_len) = '\n';
*(bulk_line->line + json_len + 1) = '\0';
bulk_line->next = NULL;
@ -124,13 +124,13 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len, int legacy) {
snprintf(
action_str, sizeof(action_str),
"{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
line->doc_id, Indexer->es_index
line->sid, Indexer->es_index
);
} else {
snprintf(
action_str, sizeof(action_str),
"{\"index\":{\"_id\":\"%s\",\"_index\":\"%s\"}}\n",
line->doc_id, Indexer->es_index
line->sid, Indexer->es_index
);
}
@ -148,7 +148,7 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len, int legacy) {
snprintf(
action_str, sizeof(action_str),
"{\"delete\":{\"_id\":\"%s\",\"_index\":\"%s\"}}\n",
line->doc_id, Indexer->es_index
line->sid, Indexer->es_index
);
size_t action_str_len = strlen(action_str);
@ -236,7 +236,7 @@ void _elastic_flush(int max) {
if (r->status_code == 413) {
if (max <= 1) {
LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->doc_id);
LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->sid);
free_response(r);
free(buf);
free_queue(1);
@ -348,7 +348,7 @@ es_indexer_t *create_indexer(const char *url, const char *index) {
return indexer;
}
void finish_indexer(char *index_id) {
void finish_indexer(int index_id) {
char url[4096];

10
src/index/elastic.h
View File

@ -8,7 +8,7 @@
typedef struct es_bulk_line {
struct es_bulk_line *next;
char doc_id[SIST_DOC_ID_LEN];
char sid[SIST_SID_LEN];
int type;
char line[0];
} es_bulk_line_t;
@ -44,16 +44,16 @@ typedef struct es_indexer es_indexer_t;
void elastic_index_line(es_bulk_line_t *line);
void print_json(cJSON *document, const char index_id_str[SIST_INDEX_ID_LEN]);
void print_json(cJSON *document, const char doc_id[SIST_SID_LEN]);
void index_json(cJSON *document, const char doc_id[SIST_INDEX_ID_LEN]);
void index_json(cJSON *document, const char doc_id[SIST_SID_LEN]);
void delete_document(const char *document_id);
void delete_document(const char *sid);
es_indexer_t *create_indexer(const char *url, const char *index);
void elastic_cleanup();
void finish_indexer(char *index_id);
void finish_indexer(int index_id);
void elastic_init(int force_reset, const char* user_mappings, const char* user_settings);

51
src/io/serialize.c
View File

@ -32,8 +32,6 @@ char *get_meta_key_text(enum metakey meta_key) {
return "title";
case MetaFontName:
return "font_name";
case MetaParent:
return "parent";
case MetaExifMake:
return "exif_make";
case MetaExifDescription:
@ -58,8 +56,6 @@ char *get_meta_key_text(enum metakey meta_key) {
return "author";
case MetaModifiedBy:
return "modified_by";
case MetaThumbnail:
return "thumbnail";
case MetaPages:
return "pages";
case MetaExifGpsLongitudeRef:
@ -81,21 +77,23 @@ char *get_meta_key_text(enum metakey meta_key) {
}
}
char *build_json_string(document_t *doc) {
typedef struct {
meta_line_t *meta_head;
meta_line_t *meta_tail;
} linked_list_t;
void write_document(document_t *doc) {
linked_list_t thumbnails_to_write = {.meta_head = NULL, .meta_tail = NULL};
cJSON *json = cJSON_CreateObject();
int buffer_size_guess = 8192;
const char *mime_text = mime_get_mime_text(doc->mime);
if (mime_text == NULL) {
cJSON_AddNullToObject(json, "mime");
} else {
cJSON_AddStringToObject(json, "mime", mime_text);
}
// Ignore root directory in the file path
doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
doc->base = (short) (doc->base - ScanCtx.index.desc.root_len);
char *filepath = doc->filepath + ScanCtx.index.desc.root_len;
char filepath[PATH_MAX * 3];
strcpy(filepath, doc->filepath + ScanCtx.index.desc.root_len);
cJSON_AddStringToObject(json, "extension", filepath + doc->ext);
@ -125,7 +123,6 @@ char *build_json_string(document_t *doc) {
while (meta != NULL) {
switch (meta->key) {
case MetaThumbnail:
case MetaPages:
case MetaWidth:
case MetaHeight:
@ -143,7 +140,6 @@ char *build_json_string(document_t *doc) {
case MetaAlbumArtist:
case MetaGenre:
case MetaFontName:
case MetaParent:
case MetaExifMake:
case MetaExifDescription:
case MetaExifSoftware:
@ -168,6 +164,11 @@ char *build_json_string(document_t *doc) {
buffer_size_guess += (int) strlen(meta->str_val);
break;
}
case MetaThumbnail: {
// Keep a list of thumbnails to write after we know what the sid is
APPEND_THUMBNAIL(&thumbnails_to_write, meta->str_val, meta->size);
break;
}
default:
LOG_FATALF("serialize.c", "Invalid meta key: %x %s", meta->key, get_meta_key_text(meta->key));
}
@ -180,13 +181,19 @@ char *build_json_string(document_t *doc) {
char *json_str = cJSON_PrintBuffered(json, buffer_size_guess, FALSE);
cJSON_Delete(json);
return json_str;
}
void write_document(document_t *doc) {
char *json_str = build_json_string(doc);
database_write_document(ProcData.index_db, doc, json_str);
int doc_id = database_write_document(ProcData.index_db, doc, json_str);
free(doc);
free(json_str);
// Write thumbnails
meta = thumbnails_to_write.meta_head;
int index_num = 0;
while (meta != NULL) {
database_write_thumbnail(ProcData.index_db, doc_id, index_num, meta->str_val, meta->size);
meta_line_t *tmp = meta;
meta = meta->next;
free(tmp);
index_num += 1;
}
}

39
src/main.c
View File

@ -39,7 +39,7 @@ void database_scan_begin(scan_args_t *args) {
index_descriptor_t *original_desc = database_read_index_descriptor(db);
// copy original index id
strcpy(desc->id, original_desc->id);
desc->id = original_desc->id;
if (original_desc->version_major != VersionMajor) {
LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc->version, Version);
@ -67,7 +67,7 @@ void database_scan_begin(scan_args_t *args) {
desc->version_patch = VersionPatch;
// generate new index id based on timestamp
md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id);
desc->id = (int) ScanCtx.index.desc.timestamp;
database_initialize(db);
database_open(db);
@ -75,14 +75,11 @@ void database_scan_begin(scan_args_t *args) {
}
database_increment_version(db);
database_sync_mime_table(db);
database_close(db, FALSE);
}
void write_thumbnail_callback(char *key, int num, void *buf, size_t buf_len) {
database_write_thumbnail(ProcData.index_db, key, num, buf, buf_len);
}
void log_callback(const char *filepath, int level, char *str) {
if (level == LEVEL_FATAL) {
sist_log(filepath, level, str);
@ -140,7 +137,6 @@ void initialize_scan_context(scan_args_t *args) {
// Comic
ScanCtx.comic_ctx.log = log_callback;
ScanCtx.comic_ctx.logf = logf_callback;
ScanCtx.comic_ctx.store = write_thumbnail_callback;
ScanCtx.comic_ctx.enable_tn = args->tn_count > 0;
ScanCtx.comic_ctx.tn_size = args->tn_size;
ScanCtx.comic_ctx.tn_qscale = args->tn_quality;
@ -157,7 +153,6 @@ void initialize_scan_context(scan_args_t *args) {
}
ScanCtx.ebook_ctx.log = log_callback;
ScanCtx.ebook_ctx.logf = logf_callback;
ScanCtx.ebook_ctx.store = write_thumbnail_callback;
ScanCtx.ebook_ctx.fast_epub_parse = args->fast_epub;
ScanCtx.ebook_ctx.tn_qscale = args->tn_quality;
@ -165,7 +160,6 @@ void initialize_scan_context(scan_args_t *args) {
ScanCtx.font_ctx.enable_tn = args->tn_count > 0;
ScanCtx.font_ctx.log = log_callback;
ScanCtx.font_ctx.logf = logf_callback;
ScanCtx.font_ctx.store = write_thumbnail_callback;
// Media
ScanCtx.media_ctx.tn_qscale = args->tn_quality;
@ -173,7 +167,6 @@ void initialize_scan_context(scan_args_t *args) {
ScanCtx.media_ctx.tn_count = args->tn_count;
ScanCtx.media_ctx.log = log_callback;
ScanCtx.media_ctx.logf = logf_callback;
ScanCtx.media_ctx.store = write_thumbnail_callback;
ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer_mib * 1024 * 1024;
ScanCtx.media_ctx.read_subtitles = args->read_subtitles;
ScanCtx.media_ctx.read_subtitles = args->tn_count;
@ -189,13 +182,11 @@ void initialize_scan_context(scan_args_t *args) {
ScanCtx.ooxml_ctx.content_size = args->content_size;
ScanCtx.ooxml_ctx.log = log_callback;
ScanCtx.ooxml_ctx.logf = logf_callback;
ScanCtx.ooxml_ctx.store = write_thumbnail_callback;
// MOBI
ScanCtx.mobi_ctx.content_size = args->content_size;
ScanCtx.mobi_ctx.log = log_callback;
ScanCtx.mobi_ctx.logf = logf_callback;
ScanCtx.mobi_ctx.store = write_thumbnail_callback;
ScanCtx.mobi_ctx.enable_tn = args->tn_count > 0;
ScanCtx.mobi_ctx.tn_size = args->tn_size;
ScanCtx.mobi_ctx.tn_qscale = args->tn_quality;
@ -209,7 +200,6 @@ void initialize_scan_context(scan_args_t *args) {
ScanCtx.msdoc_ctx.content_size = args->content_size;
ScanCtx.msdoc_ctx.log = log_callback;
ScanCtx.msdoc_ctx.logf = logf_callback;
ScanCtx.msdoc_ctx.store = write_thumbnail_callback;
ScanCtx.msdoc_ctx.msdoc_mime = mime_get_mime_by_string("application/msword");
ScanCtx.threads = args->threads;
@ -228,7 +218,6 @@ void initialize_scan_context(scan_args_t *args) {
ScanCtx.raw_ctx.tn_size = args->tn_size;
ScanCtx.raw_ctx.log = log_callback;
ScanCtx.raw_ctx.logf = logf_callback;
ScanCtx.raw_ctx.store = write_thumbnail_callback;
// Wpd
ScanCtx.wpd_ctx.content_size = args->content_size;
@ -316,16 +305,15 @@ void sist2_index(index_args_t *args) {
database_open(db);
database_iterator_t *iterator = database_create_document_iterator(db);
database_document_iter_foreach(json, iterator) {
char doc_id[SIST_DOC_ID_LEN];
strcpy(doc_id, cJSON_GetObjectItem(json, "_id")->valuestring);
char sid[SIST_SID_LEN];
int doc_id = cJSON_GetObjectItem(json, "_id")->valueint;
cJSON_DeleteItemFromObject(json, "_id");
// TODO: delete tag if empty
format_sid(sid, desc->id, doc_id);
if (args->print) {
print_json(json, doc_id);
print_json(json, sid);
} else {
index_json(json, doc_id);
index_json(json, sid);
cnt += 1;
}
cJSON_Delete(json);
@ -334,10 +322,12 @@ void sist2_index(index_args_t *args) {
free(iterator);
if (!args->print) {
char sid[SIST_SID_LEN];
database_iterator_t *del_iter = database_create_delete_list_iterator(db);
database_delete_list_iter_foreach(id, del_iter) {
delete_document(id);
free(id);
database_delete_list_iter_foreach(doc_id, del_iter) {
format_sid(sid, desc->id, doc_id);
delete_document(sid);
}
free(del_iter);
}
@ -533,7 +523,8 @@ int main(int argc, const char *argv[]) {
OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings."),
OPT_GROUP("sqlite-index options"),
OPT_STRING(0, "search-index", &common_search_index, "Path to search index. Will be created if it does not exist yet."),
OPT_STRING(0, "search-index", &common_search_index,
"Path to search index. Will be created if it does not exist yet."),
OPT_GROUP("Web options"),
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT: http://localhost:9200"),

2
src/parsing/mime.h
View File

@ -61,4 +61,6 @@ unsigned int mime_get_mime_by_ext(const char *ext);
unsigned int mime_get_mime_by_string(const char *str);
unsigned int* get_mime_ids();
#endif

6
src/parsing/mime_generated.c
View File

@ -365,7 +365,6 @@ model_vnd_gdl=65893,
model_vnd_gs_gdl=65894,
model_vrml=65895,
model_x_pov=65896,
sist2_sidecar=2,
text_PGP=590185,
text_asp=590186,
text_css=590187,
@ -909,7 +908,6 @@ case image_x_sony_arw: return "image/x-sony-arw";
case image_x_sony_sr2: return "image/x-sony-sr2";
case image_x_sony_srf: return "image/x-sony-srf";
case image_x_epson_erf: return "image/x-epson-erf";
case sist2_sidecar: return "sist2/sidecar";
default: return NULL;}}
unsigned int mime_extension_lookup(unsigned long extension_crc32) {switch (extension_crc32) {
case 2495639202:return application_x_matlab_data;
@ -1293,7 +1291,6 @@ case 1698465774:return image_x_sony_arw;
case 2083014127:return image_x_sony_sr2;
case 271503362:return image_x_sony_srf;
case 142938048:return image_x_epson_erf;
case 287571459:return sist2_sidecar;
default: return 0;}}
unsigned int mime_name_lookup(unsigned long mime_crc32) {switch (mime_crc32) {
case 3272851765: return application_x_matlab_data;
@ -1747,6 +1744,7 @@ case 3060720351: return image_x_sony_arw;
case 2944016606: return image_x_sony_sr2;
case 3279729971: return image_x_sony_srf;
case 1665206815: return image_x_epson_erf;
case 521139448: return sist2_sidecar;
default: return 0;}}
unsigned int mime_ids[] = {655530,655363,655364,655365,655366,655362,655361,655367,655368,655369,655370,655371,655372 | 0x40000000,655373,655374,655375,655376 | 0x08000000,655377,655378,655379,655380,655382,655381,655383,655384,655390,655385,655386,655387,655388,655389,655391,655392,655393,655394,655395 | 0x40000000,655396,655397,655398,655399,655400,655401,655402,655403,655404,655405,655406,655407,655408,655411,655412,655413,655414,655415,655416,655417,655418,655419 | 0x20000000,655421,655422,655423,655424,655425,655426,655427,655428,655429,655430,655431,655432 | 0x04000000,655433 | 0x04000000,655434 | 0x04000000,655435,655436,655437,655438,655439,655440,655441,655442,655443,655444,655445,655446 | 0x10000000,655447,655448,655449 | 0x10000000,655450,655451,655452,655453,655454,655455,655456,655457,655458,655459,655461 | 0x08000000,655460,655462,655463,655464,655465,655466,655467,655468,655469,655470,655471,655472,655473,655474,655475,655476,655477,655478,655479,655480,1,655481,655482,655483,655484,655485,655486,655487,655488,655489 | 0x20000000,655490,655491,655492,655493,655494,655495,655496,655497,655498,655499,655500,655501,655502,655503,655504,655505,655506,655507,655508,655509,655510,655511,655512,655513,655514,655515,655516,655517,655519,655518 | 0x08000000,655521,655520,655522 | 0x08000000,655523 | 0x08000000,655524 | 0x08000000,655525,655526,655527,655528,655529,655531,655532,655533,655534,655535,655599,655536 | 0x02000000,655409 | 0x02000000,655540,655537,655538,655539,655541,655542,655543,655544,655545,655546,655547,655548,655549,655550,655552,655551,655553,655554,655555,655556,655557,655558,655559,655560,655561,655562 | 0x10000000,655563,655564,655565,655566,655567,655569,655568,655570,655571,655572,655573,655574,655575,655576,655577,655578 | 0x10000000,655579,655580,655581,655583,655582,655584,655585,655586,655587,655588,655589,655590,655591,655592,655593,655594,655595 | 0x08000000,655596,655597 | 0x08000000,655600 | 0x10000000,655601,458994 | 0x80000000,458995,458996,458998,458997,458999,459000,459001,459002,459003,459004,459005,459006,459007,459008,459009,459010,459011,459012,459013,459014,459015,459016,459017,459018,459030,459019,459020,459021,459022,459023,459025,459024,459026,459027,459029 | 0x80000000,459028 | 0x80000000,327959 | 0x20000000,327960 | 0x20000000,327962 | 0x20000000,327961 | 0x20000000,524571,524572,524573,524574,524575,524576,524577,524578,524579,524580,524581,524582,524583,524584 | 0x80000000,524585 | 0x80000000,524586,524587 | 0x80000000,524588 | 0x80000000,524589,524590,524591,524592,524593,524594,524595,524596,524597,524599,524602,524603,524605,524606,524608,524610,524611,524612 | 0x80000000,524613,524614,524619,524620,524624,524626,524627,524628,524629,524630,524631,524636,524637,524638,524639 | 0x80000000,524640 | 0x80000000,524641,196962,196963,65892,65893,65894,65895,65896,590186,590187,590189 | 0x01000000,590190,590191,590192,590185,590193,590231,590188,655410,590194,590195,590196,590197,590198,590199,590200,590201,590203,590202,590204,590205,590206,590207,590208,590209,590210,590211,590212,590213,590214,590215,590216,590217,590219,590220,590244 | 0x01000000,590218,590222,590221,590223,590224,590225,590226,590227,590228,590229,590230,590232,590233,590234,590235 | 0x01000000,590236,590237,590238,590239,590240,590241,590242,590243,393638,393639,393640,393637,393641,393642,393643,393644,393645,393646,393647,393648,393649,393650,393651,393652,393653,393654,393655,393656,393657 | 0x80000000,393658,393659,393660,393661,393662,393663,393664,393665,721346,655598,655420,524622 | 0x00800000,524621 | 0x00800000,524609 | 0x00800000,524623 | 0x00800000,524598 | 0x00800000,524600 | 0x00800000,524601 | 0x00800000,524604 | 0x00800000,524615 | 0x00800000,524616 | 0x00800000,524617 | 0x00800000,524618 | 0x00800000,524625 | 0x00800000,524632 | 0x00800000,524633 | 0x00800000,524634 | 0x00800000,524635 | 0x00800000,524607 | 0x00800000,0};
unsigned int* get_mime_ids() { return mime_ids; }
#endif

31
src/parsing/parse.c
View File

@ -4,10 +4,8 @@
#include "src/ctx.h"
#include "mime.h"
#include "src/io/serialize.h"
#include "src/parsing/sidecar.h"
#include "src/parsing/fs_util.h"
#include "src/parsing/magic_util.h"
#include <pthread.h>
#define MIN_VIDEO_SIZE (1024 * 64)
@ -27,7 +25,6 @@ typedef enum {
FILETYPE_OOXML,
FILETYPE_COMIC,
FILETYPE_MOBI,
FILETYPE_SIST2_SIDECAR,
FILETYPE_MSDOC,
FILETYPE_JSON,
FILETYPE_NDJSON,
@ -63,8 +60,6 @@ file_type_t get_file_type(unsigned int mime, size_t size, const char *filepath)
return FILETYPE_COMIC;
} else if (IS_MOBI(mime)) {
return FILETYPE_MOBI;
} else if (mime == MIME_SIST2_SIDECAR) {
return FILETYPE_SIST2_SIDECAR;
} else if (is_msdoc(&ScanCtx.msdoc_ctx, mime)) {
return FILETYPE_MSDOC;
} else if (is_json(&ScanCtx.json_ctx, mime)) {
@ -157,7 +152,8 @@ void parse(parse_job_t *job) {
doc->size = job->vfile.st_size;
doc->mtime = MAX(job->vfile.mtime, 0);
doc->mime = get_mime(job);
generate_doc_id(doc->filepath + ScanCtx.index.desc.root_len, doc->doc_id);
doc->thumbnail_count = 0;
strcpy(doc->parent, job->parent);
if (doc->mime == GET_MIME_ERROR_FATAL) {
CLOSE_FILE(job->vfile)
@ -165,16 +161,12 @@ void parse(parse_job_t *job) {
return;
}
if (database_mark_document(ProcData.index_db, doc->doc_id, doc->mtime)) {
if (database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime)) {
CLOSE_FILE(job->vfile)
free(doc);
return;
}
if (LogCtx.very_verbose) {
LOG_DEBUGF(job->filepath, "Starting parse job {%s}", doc->doc_id);
}
switch (get_file_type(doc->mime, doc->size, doc->filepath)) {
case FILETYPE_RAW:
parse_raw(&ScanCtx.raw_ctx, &job->vfile, doc);
@ -195,6 +187,10 @@ void parse(parse_job_t *job) {
parse_font(&ScanCtx.font_ctx, &job->vfile, doc);
break;
case FILETYPE_ARCHIVE:
// Insert the document now so that the children documents can link to an existing ID
database_write_document(ProcData.index_db, doc, NULL);
parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc, ScanCtx.exclude, ScanCtx.exclude_extra);
break;
case FILETYPE_OOXML:
@ -206,11 +202,6 @@ void parse(parse_job_t *job) {
case FILETYPE_MOBI:
parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, doc);
break;
case FILETYPE_SIST2_SIDECAR:
parse_sidecar(&job->vfile, doc);
CLOSE_FILE(job->vfile)
free(doc);
return;
case FILETYPE_MSDOC:
parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc);
break;
@ -225,14 +216,6 @@ void parse(parse_job_t *job) {
break;
}
//Parent meta
if (job->parent[0] != '\0') {
meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + SIST_INDEX_ID_LEN);
meta_parent->key = MetaParent;
strcpy(meta_parent->str_val, job->parent);
APPEND_META((doc), meta_parent);
}
CLOSE_FILE(job->vfile)
if (job->vfile.has_checksum) {

40
src/parsing/sidecar.c
View File

@ -1,40 +0,0 @@
#include "sidecar.h"
#include "src/ctx.h"
void parse_sidecar(vfile_t *vfile, document_t *doc) {
LOG_DEBUGF("sidecar.c", "Parsing sidecar file %s", vfile->filepath);
size_t size;
char *buf = read_all(vfile, &size);
if (buf == NULL) {
LOG_ERRORF("sidecar.c", "Read error for %s", vfile->filepath);
return;
}
buf = realloc(buf, size + 1);
*(buf + size) = '\0';
cJSON *json = cJSON_Parse(buf);
if (json == NULL) {
LOG_ERRORF("sidecar.c", "Could not parse JSON sidecar %s", vfile->filepath);
return;
}
char *json_str = cJSON_PrintUnformatted(json);
char assoc_doc_id[SIST_DOC_ID_LEN];
char rel_path[PATH_MAX];
size_t rel_path_len = doc->ext - 1 - ScanCtx.index.desc.root_len;
memcpy(rel_path, vfile->filepath + ScanCtx.index.desc.root_len, rel_path_len);
*(rel_path + rel_path_len) = '\0';
generate_doc_id(rel_path, assoc_doc_id);
database_write_document_sidecar(ProcData.index_db, assoc_doc_id, json_str);
cJSON_Delete(json);
free(json_str);
free(buf);
}

8
src/parsing/sidecar.h
View File

@ -1,8 +0,0 @@
#ifndef SIST2_SIDECAR_H
#define SIST2_SIDECAR_H
#include "src/sist.h"
void parse_sidecar(vfile_t *vfile, document_t *doc);
#endif

20
src/sist.h
View File

@ -3,19 +3,19 @@
#define _GNU_SOURCE
#ifndef FALSE
#define FALSE (0)
#ifndef FALSE
#define FALSE (0)
#define BOOL int
#endif
#ifndef TRUE
#define TRUE (!FALSE)
#ifndef TRUE
#define TRUE (!FALSE)
#endif
#undef MAX
#undef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#undef MIN
#undef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#ifndef PATH_MAX
@ -23,7 +23,7 @@
#endif
#undef ABS
#define ABS(a) (((a) < 0) ? -(a) : (a))
#define ABS(a) (((a) < 0) ? -(a) : (a))
#define UNUSED(x) __attribute__((__unused__)) x
@ -51,11 +51,11 @@
#include <ctype.h>
#include "git_hash.h"
#define VERSION "3.2.1"
#define VERSION "3.3.0"
static const char *const Version = VERSION;
static const int VersionMajor = 3;
static const int VersionMinor = 2;
static const int VersionPatch = 1;
static const int VersionMinor = 3;
static const int VersionPatch = 0;
#ifndef SIST_PLATFORM
#define SIST_PLATFORM unknown

9
src/types.h
View File

@ -4,7 +4,7 @@
typedef struct database database_t;
typedef struct index_descriptor {
char id[SIST_INDEX_ID_LEN];
int id;
char version[64];
int version_major;
int version_minor;
@ -24,4 +24,11 @@ typedef struct index_t {
char path[PATH_MAX];
} index_t;
typedef struct {
int doc_id;
int index_id;
long sid_int64;
char sid_str[SIST_SID_LEN];
} sist_id_t;
#endif

47
src/util.h
View File

@ -7,6 +7,7 @@
#include "third-party/utf8.h/utf8.h"
#include "libscan/scan.h"
#include "types.h"
#include <openssl/evp.h>
@ -18,7 +19,8 @@ dyn_buffer_t url_escape(char *str);
extern int PrintingProgressBar;
void progress_bar_print_json(size_t done, size_t count, size_t tn_size, size_t index_size, int waiting);
void progress_bar_print_json(size_t done, size_t count, size_t tn_size, size_t index_size, int waiting);
void progress_bar_print(double percentage, size_t tn_size, size_t index_size);
const char *find_file_in_paths(const char **paths, const char *filename);
@ -87,24 +89,6 @@ static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
*s = '\0';
}
static void md5_hexdigest(const void *data, size_t size, char *output) {
EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL);
EVP_DigestUpdate(md_ctx, data, size);
unsigned char digest[MD5_DIGEST_LENGTH];
EVP_DigestFinal_ex(md_ctx, digest, NULL);
EVP_MD_CTX_free(md_ctx);
buf2hex(digest, MD5_DIGEST_LENGTH, output);
}
__always_inline
static void generate_doc_id(const char *rel_path, char *doc_id) {
md5_hexdigest(rel_path, strlen(rel_path), doc_id);
}
#define MILLISECOND 1000
struct timespec timespec_add(struct timespec ts1, long usec);
@ -125,6 +109,29 @@ struct timespec timespec_add(struct timespec ts1, long usec);
} while (0)
#define array_foreach(arr) \
for (int i = 0; (arr)[i] != NULL; i++)
for (int i = 0; (arr)[i] != 0; i++)
#define format_sid(out, index_id, doc_id) \
sprintf((out), "%08x.%08x", (index_id), (doc_id))
static int parse_sid(sist_id_t *sid, const char doc_sid_str[SIST_SID_LEN]) {
if (doc_sid_str[8] != '.') {
return FALSE;
}
char tmp[9];
memcpy(tmp, doc_sid_str, 8);
sid->index_id = (int) strtol(tmp, NULL, 16);
memcpy(tmp, doc_sid_str + 9, 8);
sid->doc_id = (int) strtol(tmp, NULL, 16);
memcpy(sid->sid_str, doc_sid_str, SIST_SID_LEN - 1);
*(sid->sid_str + SIST_SID_LEN - 1) = '\0';
sid->sid_int64 = ((long) sid->index_id << 32) | sid->doc_id;
return TRUE;
}
#endif

155
src/web/serve.c
View File

@ -48,30 +48,24 @@ void get_embedding(struct mg_connection *nc, struct mg_http_message *hm) {
WebCtx.es_version->major, WebCtx.es_version->minor, WebCtx.es_version->patch);
}
if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2 + 4) {
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
sist_id_t sid;
if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
LOG_DEBUGF("serve.c", "Invalid embedding path: %.*s", (int) hm->uri.len, hm->uri.ptr);
HTTP_REPLY_NOT_FOUND
return;
}
char doc_id[SIST_DOC_ID_LEN];
char index_id[SIST_INDEX_ID_LEN];
int model_id = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
memcpy(index_id, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
*(index_id + SIST_INDEX_ID_LEN - 1) = '\0';
memcpy(doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN);
*(doc_id + SIST_DOC_ID_LEN - 1) = '\0';
int model_id = (int) strtol(hm->uri.ptr + SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 3, NULL, 10);
database_t *db = web_get_database(index_id);
database_t *db = web_get_database(sid.index_id);
if (db == NULL) {
LOG_DEBUGF("serve.c", "Could not get database for index: %s", index_id);
LOG_DEBUGF("serve.c", "Could not get database for index: %s", sid.index_id);
HTTP_REPLY_NOT_FOUND
return;
}
cJSON *json = database_get_embedding(db, doc_id, model_id);
cJSON *json = database_get_embedding(db, sid.doc_id, model_id);
if (json == NULL) {
HTTP_REPLY_NOT_FOUND
@ -84,17 +78,19 @@ void get_embedding(struct mg_connection *nc, struct mg_http_message *hm) {
void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
if (hm->uri.len != SIST_INDEX_ID_LEN + 7) {
if (hm->uri.len != 17) {
HTTP_REPLY_NOT_FOUND
return;
}
char arg_index_id[SIST_INDEX_ID_LEN];
char index_id_str[9];
char arg_stat_type[5];
memcpy(arg_index_id, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
*(arg_index_id + SIST_INDEX_ID_LEN - 1) = '\0';
memcpy(arg_stat_type, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, 4);
memcpy(index_id_str, hm->uri.ptr + 3, 8);
*(index_id_str + 8) = '\0';
int index_id = (int)strtol(index_id_str, NULL, 16);
memcpy(arg_stat_type, hm->uri.ptr + 3 + 9, 4);
*(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';
database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type);
@ -103,9 +99,9 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
return;
}
database_t *db = web_get_database(arg_index_id);
database_t *db = web_get_database(index_id);
if (db == NULL) {
LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index_id);
LOG_DEBUGF("serve.c", "Could not get database for index: %d", index_id);
HTTP_REPLY_NOT_FOUND
return;
}
@ -152,19 +148,19 @@ void serve_chunk_vendors_css(struct mg_connection *nc, struct mg_http_message *h
web_serve_asset_chunk_vendors_css(nc);
}
void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, const char *arg_index,
const char *arg_doc_id, int arg_num) {
void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, int index_id,
int doc_id, int arg_num) {
database_t *db = web_get_database(arg_index);
database_t *db = web_get_database(index_id);
if (db == NULL) {
LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index);
LOG_DEBUGF("serve.c", "Could not get database for index: %d", index_id);
HTTP_REPLY_NOT_FOUND
return;
}
size_t data_len = 0;
void *data = database_read_thumbnail(db, arg_doc_id, arg_num, &data_len);
void *data = database_read_thumbnail(db, doc_id, arg_num, &data_len);
if (data_len != 0) {
web_send_headers(
@ -181,44 +177,29 @@ void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, const
}
void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) {
if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2 + 5) {
sist_id_t sid;
if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
HTTP_REPLY_NOT_FOUND
return;
}
char arg_doc_id[SIST_DOC_ID_LEN];
char arg_index[SIST_INDEX_ID_LEN];
char arg_num[5] = {0};
int num = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
*(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN);
*(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0';
memcpy(arg_num, hm->uri.ptr + SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 3, 4);
int num = (int) strtol(arg_num, NULL, 10);
serve_thumbnail(nc, hm, arg_index, arg_doc_id, num);
serve_thumbnail(nc, hm, sid.index_id, sid.doc_id, num);
}
void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
sist_id_t sid;
if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2) {
if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
HTTP_REPLY_NOT_FOUND
return;
}
char arg_doc_id[SIST_DOC_ID_LEN];
char arg_index[SIST_INDEX_ID_LEN];
memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
*(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN);
*(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0';
serve_thumbnail(nc, hm, arg_index, arg_doc_id, 0);
serve_thumbnail(nc, hm, sid.index_id, sid.doc_id, 0);
}
void search(struct mg_connection *nc, struct mg_http_message *hm) {
@ -382,7 +363,7 @@ void index_info(struct mg_connection *nc) {
cJSON *idx_json = cJSON_CreateObject();
cJSON_AddStringToObject(idx_json, "name", idx->desc.name);
cJSON_AddStringToObject(idx_json, "version", idx->desc.version);
cJSON_AddStringToObject(idx_json, "id", idx->desc.id);
cJSON_AddNumberToObject(idx_json, "id", idx->desc.id);
cJSON_AddStringToObject(idx_json, "rewriteUrl", idx->desc.rewrite_url);
cJSON_AddNumberToObject(idx_json, "timestamp", (double) idx->desc.timestamp);
cJSON_AddItemToArray(arr, idx_json);
@ -405,15 +386,14 @@ void index_info(struct mg_connection *nc) {
cJSON_Delete(json);
}
cJSON *get_root_document_by_id(const char *index_id, const char *doc_id) {
cJSON *get_root_document_by_id(int index_id, int doc_id) {
database_t *db = web_get_database(index_id);
if (!db) {
return NULL;
}
char next_id[SIST_DOC_ID_LEN];
strcpy(next_id, doc_id);
int next_id = doc_id;
while (TRUE) {
cJSON *doc = database_get_document(db, next_id);
@ -423,38 +403,31 @@ cJSON *get_root_document_by_id(const char *index_id, const char *doc_id) {
}
cJSON *parent = cJSON_GetObjectItem(doc, "parent");
if (parent == NULL || cJSON_IsNull(parent)) {
if (parent == NULL || !cJSON_IsNumber(parent)) {
return doc;
}
strcpy(next_id, parent->valuestring);
cJSON_Delete(parent);
next_id = parent->valueint;
cJSON_Delete(doc);
}
}
void file(struct mg_connection *nc, struct mg_http_message *hm) {
sist_id_t sid;
if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2) {
if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr);
HTTP_REPLY_NOT_FOUND
return;
}
char arg_doc_id[SIST_DOC_ID_LEN];
char arg_index[SIST_INDEX_ID_LEN];
memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
*(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN);
*(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0';
index_t *idx = web_get_index_by_id(arg_index);
index_t *idx = web_get_index_by_id(sid.index_id);
if (idx == NULL) {
HTTP_REPLY_NOT_FOUND
return;
}
cJSON *source = get_root_document_by_id(arg_index, arg_doc_id);
cJSON *source = get_root_document_by_id(sid.index_id, sid.doc_id);
if (strlen(idx->desc.rewrite_url) == 0) {
serve_file_from_disk(source, idx, nc, hm);
@ -478,7 +451,6 @@ void status(struct mg_connection *nc) {
typedef struct {
char *name;
int delete;
char *doc_id;
} tag_req_t;
tag_req_t *parse_tag_request(cJSON *json) {
@ -501,20 +473,14 @@ tag_req_t *parse_tag_request(cJSON *json) {
return NULL;
}
cJSON *arg_doc_id = cJSON_GetObjectItem(json, "doc_id");
if (arg_doc_id == NULL || !cJSON_IsString(arg_doc_id)) {
return NULL;
}
tag_req_t *req = malloc(sizeof(tag_req_t));
req->delete = arg_delete->valueint;
req->name = arg_name->valuestring;
req->doc_id = arg_doc_id->valuestring;
return req;
}
subreq_ctx_t *elastic_delete_tag(const tag_req_t *req) {
subreq_ctx_t *elastic_delete_tag(const char* sid, const tag_req_t *req) {
char *buf = malloc(sizeof(char) * 8192);
snprintf(buf, 8192,
"{"
@ -529,12 +495,12 @@ subreq_ctx_t *elastic_delete_tag(const tag_req_t *req) {
);
char url[4096];
snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, req->doc_id);
snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, sid);
return web_post_async(url, buf, WebCtx.es_insecure_ssl);
}
subreq_ctx_t *elastic_write_tag(const tag_req_t *req) {
subreq_ctx_t *elastic_write_tag(const char* sid, const tag_req_t *req) {
char *buf = malloc(sizeof(char) * 8192);
snprintf(buf, 8192,
"{"
@ -549,21 +515,18 @@ subreq_ctx_t *elastic_write_tag(const tag_req_t *req) {
);
char url[4096];
snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, req->doc_id);
snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, sid);
return web_post_async(url, buf, WebCtx.es_insecure_ssl);
}
void tag(struct mg_connection *nc, struct mg_http_message *hm) {
if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
sist_id_t sid;
if (hm->uri.len != 22 || !parse_sid(&sid, hm->uri.ptr + 5)) {
LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr);
HTTP_REPLY_NOT_FOUND
return;
}
char arg_index[SIST_INDEX_ID_LEN];
memcpy(arg_index, hm->uri.ptr + 5, SIST_INDEX_ID_LEN);
*(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
char *body = malloc(hm->body.len + 1);
memcpy(body, hm->body.ptr, hm->body.len);
*(body + hm->body.len) = '\0';
@ -575,36 +538,36 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
return;
}
database_t *db = web_get_database(arg_index);
database_t *db = web_get_database(sid.index_id);
if (db == NULL) {
LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index);
LOG_DEBUGF("serve.c", "Could not get database for index: %d", sid.index_id);
HTTP_REPLY_NOT_FOUND
return;
}
tag_req_t *req = parse_tag_request(json);
if (req == NULL) {
LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index);
LOG_DEBUG("serve.c", "Could not parse tag request");
cJSON_Delete(json);
HTTP_REPLY_BAD_REQUEST
return;
}
if (req->delete) {
database_delete_tag(db, req->doc_id, req->name);
database_delete_tag(db, sid.doc_id, req->name);
if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
database_delete_tag(WebCtx.search_db, req->doc_id, req->name);
database_delete_tag(WebCtx.search_db, sid.sid_int64, req->name);
HTTP_REPLY_OK
} else {
nc->fn_data = elastic_delete_tag(req);
nc->fn_data = elastic_delete_tag(sid.sid_str, req);
}
} else {
database_write_tag(db, req->doc_id, req->name);
database_write_tag(db, sid.doc_id, req->name);
if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
database_write_tag(WebCtx.search_db, req->doc_id, req->name);
database_fts_write_tag(WebCtx.search_db, sid.sid_int64, req->name);
HTTP_REPLY_OK
} else {
nc->fn_data = elastic_write_tag(req);
nc->fn_data = elastic_write_tag(sid.sid_str, req);
}
}
@ -739,11 +702,11 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
if (mg_http_match_uri(hm, "/status")) {
status(nc);
} else if (mg_http_match_uri(hm, "/f/*/*")) {
} else if (mg_http_match_uri(hm, "/f/*")) {
file(nc, hm);
} else if (mg_http_match_uri(hm, "/t/*/*/*")) {
thumbnail_with_num(nc, hm);
} else if (mg_http_match_uri(hm, "/t/*/*")) {
thumbnail_with_num(nc, hm);
} else if (mg_http_match_uri(hm, "/t/*")) {
thumbnail(nc, hm);
} else if (mg_http_match_uri(hm, "/s/*/*")) {
stats_files(nc, hm);
@ -752,7 +715,7 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
return;
}
tag(nc, hm);
} else if (mg_http_match_uri(hm, "/e/*/*/*")) {
} else if (mg_http_match_uri(hm, "/e/*/*")) {
get_embedding(nc, hm);
return;
} else {

49
src/web/web_fts.c
View File

@ -3,7 +3,7 @@
#include "src/web/web_util.h"
typedef struct {
char *index_id;
int index_id;
char *prefix;
int min_depth;
int max_depth;
@ -23,7 +23,7 @@ typedef struct {
double date_min;
double date_max;
int page_size;
char **index_ids;
int *index_ids;
char **mime_types;
char **tags;
int sort_asc;
@ -108,7 +108,7 @@ static json_value get_json_bool(cJSON *object, const char *name) {
return (json_value) {item, FALSE};
}
static json_value get_json_float_array(cJSON *object, const char *name) {
static json_value get_json_number_array(cJSON *object, const char *name) {
cJSON *item = cJSON_GetObjectItem(object, name);
if (item == NULL || cJSON_IsNull(item)) {
return (json_value) {NULL, FALSE};
@ -147,7 +147,6 @@ static json_value get_json_array(cJSON *object, const char *name) {
}
char **json_array_to_c_array(cJSON *json) {
cJSON *element;
char **arr = calloc(cJSON_GetArraySize(json) + 1, sizeof(char *));
int i = 0;
@ -158,6 +157,17 @@ char **json_array_to_c_array(cJSON *json) {
return arr;
}
int *json_number_array_to_c_array(cJSON *json) {
cJSON *element;
int *arr = calloc(cJSON_GetArraySize(json) + 1, sizeof(int));
int i = 0;
cJSON_ArrayForEach(element, json) {
arr[i++] = (int) element->valuedouble;
}
return arr;
}
#define DEFAULT_HIGHLIGHT_CONTEXT_SIZE 20
fts_search_req_t *get_search_req(struct mg_http_message *hm) {
@ -184,11 +194,11 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
(req_seed = get_json_number(json, "seed")).invalid ||
(req_fetch_aggregations = get_json_bool(json, "fetchAggregations")).invalid ||
(req_sort_asc = get_json_bool(json, "sortAsc")).invalid ||
(req_index_ids = get_json_array(json, "indexIds")).invalid ||
(req_index_ids = get_json_number_array(json, "indexIds")).invalid ||
(req_mime_types = get_json_array(json, "mimeTypes")).invalid ||
(req_highlight = get_json_bool(json, "highlight")).invalid ||
(req_highlight_context_size = get_json_number(json, "highlightContextSize")).invalid ||
(req_embedding = get_json_float_array(json, "embedding")).invalid ||
(req_embedding = get_json_number_array(json, "embedding")).invalid ||
(req_model = get_json_number(json, "model")).invalid ||
(req_tags = get_json_array(json, "tags")).invalid) {
cJSON_Delete(json);
@ -251,7 +261,7 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
req->date_max = req_date_max.val ? req_date_max.val->valuedouble : 0;
req->page_size = (int) req_page_size.val->valuedouble;
req->sort_asc = req_sort_asc.val ? req_sort_asc.val->valueint : TRUE;
req->index_ids = req_index_ids.val ? json_array_to_c_array(req_index_ids.val) : NULL;
req->index_ids = req_index_ids.val ? json_number_array_to_c_array(req_index_ids.val) : NULL;
req->after = req_after.val ? json_array_to_c_array(req_after.val) : NULL;
req->mime_types = req_mime_types.val ? json_array_to_c_array(req_mime_types.val) : NULL;
req->tags = req_tags.val ? json_array_to_c_array(req_tags.val) : NULL;
@ -282,7 +292,9 @@ void destroy_search_req(fts_search_req_t *req) {
free(req->query);
free(req->path);
destroy_array(req->index_ids);
if (req->index_ids) {
free(req->index_ids);
}
destroy_array(req->mime_types);
destroy_array(req->tags);
@ -303,7 +315,7 @@ fts_search_paths_req_t *get_search_paths_req(struct mg_http_message *hm) {
json_value req_index_id, req_min_depth, req_max_depth, req_prefix;
if (!cJSON_IsObject(json) ||
(req_index_id = get_json_string(json, "indexId")).invalid ||
(req_index_id = get_json_number(json, "indexId")).invalid ||
(req_prefix = get_json_string(json, "prefix")).invalid ||
(req_min_depth = get_json_number(json, "minDepth")).val == NULL ||
(req_max_depth = get_json_number(json, "maxDepth")).val == NULL) {
@ -313,19 +325,16 @@ fts_search_paths_req_t *get_search_paths_req(struct mg_http_message *hm) {
fts_search_paths_req_t *req = malloc(sizeof(fts_search_paths_req_t));
req->index_id = req_index_id.val ? strdup(req_index_id.val->valuestring) : NULL;
req->index_id = req_index_id.val ? req_index_id.val->valueint : 0;
req->prefix = req_prefix.val ? strdup(req_prefix.val->valuestring) : NULL;
req->min_depth = req_min_depth.val->valueint;
req->max_depth = req_max_depth.val->valueint;
req->prefix = req_prefix.val ? strdup(req_prefix.val->valuestring) : NULL;
cJSON_Delete(json);
return req;
}
void destroy_search_paths_req(fts_search_paths_req_t *req) {
if (req->index_id) {
free(req->index_id);
}
if (req->prefix) {
free(req->prefix);
}
@ -398,11 +407,15 @@ void fts_search(struct mg_connection *nc, struct mg_http_message *hm) {
void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm) {
char doc_id[SIST_DOC_ID_LEN];
memcpy(doc_id, hm->uri.ptr + 7, SIST_INDEX_ID_LEN);
*(doc_id + SIST_INDEX_ID_LEN - 1) = '\0';
sist_id_t sid;
cJSON *json = database_fts_get_document(WebCtx.search_db, doc_id);
if (hm->uri.len != 24 || !parse_sid(&sid, hm->uri.ptr + 7)) {
LOG_DEBUGF("serve.c", "Invalid /fts/d/ path: %.*s", (int) hm->uri.len, hm->uri.ptr);
HTTP_REPLY_NOT_FOUND
return;
}
cJSON *json = database_fts_get_document(WebCtx.search_db, sid.sid_int64);
if (!json) {
HTTP_REPLY_NOT_FOUND

6
src/web/web_util.c
View File

@ -32,16 +32,16 @@ void web_serve_asset_chunk_vendors_css(struct mg_connection *nc) {
mg_send(nc, chunk_vendors_css, sizeof(chunk_vendors_css));
}
index_t *web_get_index_by_id(const char *index_id) {
index_t *web_get_index_by_id(int index_id) {
for (int i = WebCtx.index_count; i >= 0; i--) {
if (strncmp(index_id, WebCtx.indices[i].desc.id, SIST_INDEX_ID_LEN) == 0) {
if (index_id == WebCtx.indices[i].desc.id) {
return &WebCtx.indices[i];
}
}
return NULL;
}
database_t *web_get_database(const char *index_id) {
database_t *web_get_database(int index_id) {
index_t *idx = web_get_index_by_id(index_id);
if (idx != NULL) {
return idx->db;

4
src/web/web_util.h
View File

@ -10,9 +10,9 @@
// See https://web.dev/coop-coep/
#define HTTP_CROSS_ORIGIN_HEADERS "Cross-Origin-Embedder-Policy: require-corp\r\nCross-Origin-Opener-Policy: same-origin\r\n"
index_t *web_get_index_by_id(const char *index_id);
index_t *web_get_index_by_id(int index_id);
database_t *web_get_database(const char *index_id);
database_t *web_get_database(int index_id);
__always_inline
static char *web_address_to_string(struct mg_addr *addr) {

2
third-party/libscan/libscan/arc/arc.c vendored
View File

@ -191,7 +191,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
sub_job->vfile.logf = ctx->logf;
sub_job->vfile.has_checksum = FALSE;
sub_job->vfile.calculate_checksum = f->calculate_checksum;
strcpy(sub_job->parent, doc->doc_id);
strcpy(sub_job->parent, doc->filepath);
while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
struct stat entry_stat = *archive_entry_stat(entry);

1
third-party/libscan/libscan/arc/arc.h vendored
View File

@ -20,7 +20,6 @@ typedef struct {
parse_callback_t parse;
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
char passphrase[4096];
} scan_arc_ctx_t;

1
third-party/libscan/libscan/comic/comic.c vendored
View File

@ -54,7 +54,6 @@ void parse_comic(scan_comic_ctx_t *ctx, vfile_t *f, document_t *doc) {
.max_media_buffer = 0,
.log = ctx->log,
.logf = ctx->logf,
.store = ctx->store,
};
ret = store_image_thumbnail(&media_ctx, buf, entry_size, doc, file_path);

1
third-party/libscan/libscan/comic/comic.h vendored
View File

@ -7,7 +7,6 @@
typedef struct {
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
int enable_tn;
int tn_size;

4
third-party/libscan/libscan/ebook/ebook.c vendored
View File

@ -160,8 +160,8 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
av_init_packet(&thumbnail_packet);
avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
APPEND_LONG_META(doc, MetaThumbnail, 1);
ctx->store(doc->doc_id, 0, (char *) thumbnail_packet.data, thumbnail_packet.size);
doc->thumbnail_count = 1;
APPEND_THUMBNAIL(doc, (char *) thumbnail_packet.data, thumbnail_packet.size);
free(samples);
av_packet_unref(&thumbnail_packet);

1
third-party/libscan/libscan/ebook/ebook.h vendored
View File

@ -12,7 +12,6 @@ typedef struct {
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
int fast_epub_parse;
int tn_qscale;
} scan_ebook_ctx_t;

4
third-party/libscan/libscan/font/font.c vendored
View File

@ -231,8 +231,8 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) {
dyn_buffer_t bmp_data = dyn_buffer_create();
bmp_format(&bmp_data, dimensions, bitmap);
APPEND_LONG_META(doc, MetaThumbnail, 1);
ctx->store(doc->doc_id, 0, bmp_data.buf, bmp_data.cur);
doc->thumbnail_count = 1;
APPEND_THUMBNAIL(doc, bmp_data.buf, bmp_data.cur);
dyn_buffer_destroy(&bmp_data);
free(bitmap);

1
third-party/libscan/libscan/font/font.h vendored
View File

@ -8,7 +8,6 @@ typedef struct {
int enable_tn;
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
} scan_font_ctx_t;
void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc);

1
third-party/libscan/libscan/json/json.h vendored
View File

@ -8,7 +8,6 @@ typedef struct {
long content_size;
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
unsigned int json_mime;
unsigned int ndjson_mime;
} scan_json_ctx_t;

19
third-party/libscan/libscan/macros.h vendored
View File

@ -37,15 +37,22 @@
meta_long->long_val = value; \
APPEND_META(doc, meta_long);}} while(0)
#define APPEND_THUMBNAIL(doc, data, data_len) do{ \
{meta_line_t *meta_tn = malloc(sizeof(meta_line_t) + (data_len)); \
meta_tn->key = MetaThumbnail; \
meta_tn->size = data_len; \
memcpy(meta_tn->str_val, data, data_len); \
APPEND_META(doc, meta_tn);}} while(0)
#define APPEND_META(doc, meta) do {\
meta->next = NULL;\
if (doc->meta_head == NULL) {\
doc->meta_head = meta;\
doc->meta_tail = doc->meta_head;\
(meta)->next = NULL;\
if ((doc)->meta_head == NULL) {\
(doc)->meta_head = meta;\
(doc)->meta_tail = (doc)->meta_head;\
} else {\
doc->meta_tail->next = meta;\
doc->meta_tail = meta;\
(doc)->meta_tail->next = meta;\
(doc)->meta_tail = meta;\
}}while(0)
#define APPEND_UTF8_META(doc, keyname, str) \

22
third-party/libscan/libscan/media/media.c vendored
View File

@ -466,7 +466,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
if (scaled_frame == STORE_AS_IS) {
return_value = SAVE_THUMBNAIL_OK;
ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
APPEND_THUMBNAIL(doc, frame_and_packet->packet->data, frame_and_packet->packet->size);
} else {
// Encode frame
AVCodecContext *thumbnail_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
@ -477,9 +477,9 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
AVPacket *thumbnail_packet = av_packet_alloc();
avcodec_receive_packet(thumbnail_encoder, thumbnail_packet);
// Save thumbnail
// Save thumbnail_count
if (thumbnail_index == 0) {
ctx->store(doc->doc_id, 0, thumbnail_packet->data, thumbnail_packet->size);
APPEND_THUMBNAIL(doc, thumbnail_packet->data, thumbnail_packet->size);
return_value = SAVE_THUMBNAIL_OK;
} else if (thumbnail_index > 1) {
@ -487,7 +487,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
// I figure out a better fix.
thumbnail_index -= 1;
ctx->store(doc->doc_id, thumbnail_index, thumbnail_packet->data, thumbnail_packet->size);
APPEND_THUMBNAIL(doc, thumbnail_packet->data, thumbnail_packet->size);
return_value = SAVE_THUMBNAIL_OK;
} else {
@ -584,7 +584,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
int thumbnails_to_generate = (IS_VIDEO(pFormatCtx) && stream->codecpar->codec_id != AV_CODEC_ID_GIF &&
video_duration_in_seconds >= 15)
// Limit to ~1 thumbnail every 7s
// Limit to ~1 thumbnail_count every 7s
? MAX(MIN(ctx->tn_count, video_duration_in_seconds / 7 + 1), 1) + 1
: 1;
@ -610,7 +610,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
}
if (number_of_thumbnails_generated > 0) {
APPEND_LONG_META(doc, MetaThumbnail, number_of_thumbnails_generated);
doc->thumbnail_count = number_of_thumbnails_generated;
}
avcodec_free_context(&decoder);
@ -859,8 +859,8 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
}
if (scaled_frame == STORE_AS_IS) {
APPEND_LONG_META(doc, MetaThumbnail, 1);
ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
doc->thumbnail_count = 1;
APPEND_THUMBNAIL(doc, frame_and_packet->packet->data, frame_and_packet->packet->size);
} else {
// Encode frame to jpeg
AVCodecContext *jpeg_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
@ -871,9 +871,9 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
AVPacket *jpeg_packet = av_packet_alloc();
avcodec_receive_packet(jpeg_encoder, jpeg_packet);
// Save thumbnail
APPEND_LONG_META(doc, MetaThumbnail, 1);
ctx->store(doc->doc_id, 0, jpeg_packet->data, jpeg_packet->size);
// Save thumbnail_count
doc->thumbnail_count = 1;
APPEND_THUMBNAIL(doc, jpeg_packet->data, jpeg_packet->size);
av_packet_free(&jpeg_packet);
avcodec_free_context(&jpeg_encoder);

1
third-party/libscan/libscan/media/media.h vendored
View File

@ -13,7 +13,6 @@
typedef struct {
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
int tn_size;
int tn_qscale;

1
third-party/libscan/libscan/mobi/scan_mobi.c vendored
View File

@ -31,7 +31,6 @@ int store_cover(scan_mobi_ctx_t *ctx, document_t *doc, MOBIData *m) {
.max_media_buffer = 0,
.log = ctx->log,
.logf = ctx->logf,
.store = ctx->store,
};
store_image_thumbnail(&media_ctx, record->data, record->size, doc, "img.jpg");

1
third-party/libscan/libscan/mobi/scan_mobi.h vendored
View File

@ -7,7 +7,6 @@ typedef struct {
long content_size;
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
int tn_qscale;
int tn_size;

1
third-party/libscan/libscan/msdoc/msdoc.h vendored
View File

@ -7,7 +7,6 @@ typedef struct {
long content_size;
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
unsigned int msdoc_mime;
} scan_msdoc_ctx_t;

5
third-party/libscan/libscan/ooxml/ooxml.c vendored
View File

@ -190,8 +190,7 @@ void read_thumbnail(scan_ooxml_ctx_t *ctx, document_t *doc, struct archive *a, s
char *buf = malloc(entry_size);
archive_read_data(a, buf, entry_size);
APPEND_LONG_META(doc, MetaThumbnail, 1);
ctx->store(doc->doc_id, 1, buf, entry_size);
doc->thumbnail_count = 1;
free(buf);
}
@ -238,7 +237,7 @@ void parse_ooxml(scan_ooxml_ctx_t *ctx, vfile_t *f, document_t *doc) {
if (read_doc_props(ctx, a, doc) != 0) {
break;
}
} else if (ctx->enable_tn && strcmp(path, "docProps/thumbnail.jpeg") == 0) {
} else if (ctx->enable_tn && strcmp(path, "docProps/thumbnail_count.jpeg") == 0) {
read_thumbnail(ctx, doc, a, entry);
}
}

1
third-party/libscan/libscan/ooxml/ooxml.h vendored
View File

@ -9,7 +9,6 @@ typedef struct {
long content_size;
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
} scan_ooxml_ctx_t;
void parse_ooxml(scan_ooxml_ctx_t *ctx, vfile_t *f, document_t *doc);

5
third-party/libscan/libscan/raw/raw.c vendored
View File

@ -13,7 +13,6 @@ int store_thumbnail_jpeg(scan_raw_ctx_t *ctx, libraw_thumbnail_t img, document_t
.read_subtitles = FALSE,
.tn_count = 1,
.max_media_buffer = 0,
.store = ctx->store,
.log = ctx->log,
.logf = ctx->logf,
.tn_size = ctx->tn_size,
@ -84,8 +83,8 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
av_init_packet(&thumbnail_packet);
avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
APPEND_LONG_META(doc, MetaThumbnail, 1);
ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) thumbnail_packet.data, thumbnail_packet.size);
doc->thumbnail_count = 1;
APPEND_THUMBNAIL(doc, (char *) thumbnail_packet.data, thumbnail_packet.size);
av_packet_unref(&thumbnail_packet);
av_free(*scaled_frame->data);

1
third-party/libscan/libscan/raw/raw.h vendored
View File

@ -6,7 +6,6 @@
typedef struct {
log_callback_t log;
logf_callback_t logf;
store_callback_t store;
int enable_tn;
int tn_size;

17
third-party/libscan/libscan/scan.h vendored
View File

@ -18,8 +18,6 @@
#define UNUSED(x) __attribute__((__unused__)) x
typedef void (*store_callback_t)(char *key, int num, void *buf, size_t buf_len);
typedef void (*logf_callback_t)(const char *filepath, int level, char *format, ...);
typedef void (*log_callback_t)(const char *filepath, int level, char *str);
@ -50,8 +48,8 @@ typedef int scan_code_t;
#define CTX_LOG_FATALF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_FATAL, fmt, __VA_ARGS__); exit(-1)
#define CTX_LOG_FATAL(filepath, str) ctx->log(filepath, LEVEL_FATAL, str); exit(-1)
#define SIST_DOC_ID_LEN MD5_STR_LENGTH
#define SIST_INDEX_ID_LEN MD5_STR_LENGTH
// 0000000.000000000
#define SIST_SID_LEN 18
#define EBOOK_LOCKS 0
@ -66,7 +64,6 @@ enum metakey {
MetaGenre,
MetaTitle,
MetaFontName,
MetaParent,
MetaExifMake,
MetaExifDescription,
MetaExifSoftware,
@ -79,7 +76,6 @@ enum metakey {
MetaExifDateTime,
MetaAuthor,
MetaModifiedBy,
MetaThumbnail,
MetaChecksum,
// Number
@ -96,11 +92,15 @@ enum metakey {
MetaExifGpsLatitudeRef,
MetaExifGpsLatitudeDec,
MetaExifGpsLongitudeDec,
// other
MetaThumbnail,
};
typedef struct meta_line {
struct meta_line *next;
enum metakey key;
size_t size;
union {
char str_val[0];
unsigned long long_val;
@ -109,7 +109,6 @@ typedef struct meta_line {
typedef struct document {
char doc_id[SIST_DOC_ID_LEN];
unsigned long size;
unsigned int mime;
int mtime;
@ -117,7 +116,9 @@ typedef struct document {
int ext;
meta_line_t *meta_head;
meta_line_t *meta_tail;
int thumbnail_count;
char filepath[PATH_MAX * 2 + 1];
char parent[PATH_MAX * 2 + 1];
} document_t;
typedef struct vfile vfile_t;
@ -166,7 +167,7 @@ typedef struct {
int base;
int ext;
struct vfile vfile;
char parent[SIST_DOC_ID_LEN];
char parent[PATH_MAX * 2 + 1];
char filepath[PATH_MAX * 2 + 1];
} parse_job_t;

2
third-party/libscan/libscan/util.h vendored
View File

@ -8,7 +8,7 @@
#include "macros.h"
#include <openssl/evp.h>
#define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
#define STR_STARTS_WITH_CONSTANT(x, y) ((x) != NULL && (y) != NULL && strncmp(y, x, sizeof(y) - 1) == 0)
#define TEXT_BUF_FULL (-1)
#define INITIAL_BUF_SIZE (1024 * 16)

6
third-party/libscan/test/main.cpp vendored
View File

@ -526,7 +526,7 @@ TEST(MediaVideo, Vid3Mp4) {
ASSERT_EQ(get_meta(&doc, MetaMediaBitrate)->long_val, 825169);
ASSERT_EQ(get_meta(&doc, MetaMediaDuration)->long_val, 10);
//TODO: Check that thumbnail was generated correctly
//TODO: Check that thumbnail_count was generated correctly
cleanup(&doc, &f);
}
@ -541,7 +541,7 @@ TEST(MediaVideo, Vid3Ogv) {
ASSERT_EQ(get_meta(&doc, MetaMediaBitrate)->long_val, 590261);
ASSERT_EQ(get_meta(&doc, MetaMediaDuration)->long_val, 10);
//TODO: Check that thumbnail was generated correctly
//TODO: Check that thumbnail_count was generated correctly
cleanup(&doc, &f);
}
@ -556,7 +556,7 @@ TEST(MediaVideo, Vid3Webm) {
ASSERT_EQ(get_meta(&doc, MetaMediaBitrate)->long_val, 343153);
ASSERT_EQ(get_meta(&doc, MetaMediaDuration)->long_val, 10);
//TODO: Check that thumbnail was generated correctly
//TODO: Check that thumbnail_count was generated correctly
cleanup(&doc, &f);
}