simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-12-11 14:38:54 +00:00
Code Issues Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
simon987
2023-04-25 08:49:50 -04:00
parent 35cfd3b3b1
commit 1cfceba518
9 changed files with 223 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/database/database.c
View File

@@ -74,6 +74,8 @@ void database_initialize(database_t *db) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IndexDatabaseSchema, NULL, NULL, NULL));
} else if (db->type == IPC_CONSUMER_DATABASE || db->type == IPC_PRODUCER_DATABASE) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IpcDatabaseSchema, NULL, NULL, NULL));
} else if (db->type == FTS_DATABASE) {
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, FtsDatabaseSchema, NULL, NULL, NULL));
}
sqlite3_close(db->db);
@@ -479,28 +481,6 @@ void database_write_thumbnail(database_t *db, const char *id, int num, void *dat
}
//void database_create_fts_index(database_t *db, database_t *fts_db) {
// // In a separate file,
//
// // use database_initialize() to create FTS schema
// // if --force-reset, then truncate the tables first
//
// /*
// * create/append fts table
// *
// * create/append scalar index table with
// * id,index,size,mtime,mime
// *
// * create/append path index table with
// * index,path,depth
// *
// * content table is a view with SELECT UNION for all attached tables
// * random_seed column
// */
//
// // INSERT INTO ft(ft) VALUES('optimize');
//}
job_t *database_get_work(database_t *db, job_type_t job_type) {
job_t *job;

13
src/database/database.h
View File

@@ -10,6 +10,7 @@ typedef struct index_descriptor index_descriptor_t;
extern const char *IpcDatabaseSchema;
extern const char *IndexDatabaseSchema;
extern const char *FtsDatabaseSchema;
typedef enum {
INDEX_DATABASE,
@@ -86,8 +87,6 @@ typedef struct {
long size;
} treemap_row_t;
static treemap_row_t null_treemap_row = {0, 0, 0};
database_t *database_create(const char *filename, database_type_t type);
@@ -116,7 +115,7 @@ cJSON *database_document_iter(database_iterator_t *);
database_iterator_t *database_create_delete_list_iterator(database_t *db);
char * database_delete_list_iter(database_iterator_t *iter);
char *database_delete_list_iter(database_iterator_t *iter);
#define database_delete_list_iter_foreach(element, iter) \
for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
@@ -160,8 +159,14 @@ cJSON *database_get_stats(database_t *db, database_stat_type_d type);
#define CRASH_IF_NOT_SQLITE_OK(x) do { \
int return_value = x; \
if (return_value != SQLITE_OK) { \
LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \
LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
} \
} while (0)
void database_fts_attach(database_t *db, const char *fts_database_path);
void database_fts_index(database_t *db);
void database_fts_optimize(database_t *db);
#endif //SIST2_DATABASE_H

88
src/database/database_fts.c Normal file
View File

@@ -0,0 +1,88 @@
#include "database.h"
#include "src/ctx.h"
void database_fts_attach(database_t *db, const char *fts_database_path) {
LOG_DEBUGF("database_fts.c", "Attaching to %s", fts_database_path);
sqlite3_stmt *stmt;
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db, "ATTACH DATABASE ? AS fts"
"", -1, &stmt, NULL));
sqlite3_bind_text(stmt, 1, fts_database_path, -1, SQLITE_STATIC);
CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
sqlite3_finalize(stmt);
}
void database_fts_index(database_t *db) {
LOG_INFO("database_fts.c", "Creating content table.");
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"WITH docs AS (SELECT document.id as id,\n"
" (SELECT id FROM descriptor) as index_id,\n"
" size,\n"
" document.json_data ->> 'path' as path,\n"
" length(document.json_data->>'path') - length(REPLACE(document.json_data->>'path', '/', '')) as path_depth,\n"
" document.json_data ->> 'mime' as mime,\n"
" mtime,\n"
" CASE\n"
" WHEN sc.json_data IS NULL THEN CASE\n"
" WHEN t.tag IS NULL THEN json_set(\n"
" document.json_data, '$._id',\n"
" document.id, '$.size',\n"
" document.size, '$.mtime',\n"
" document.mtime)\n"
" ELSE json_set(document.json_data, '$._id',\n"
" document.id, '$.size',\n"
" document.size, '$.mtime',\n"
" document.mtime, '$.tag',\n"
" json_group_array(t.tag)) END\n"
" ELSE CASE\n"
" WHEN t.tag IS NULL THEN json_patch(\n"
" json_set(document.json_data, '$._id', document.id, '$.size',\n"
" document.size, '$.mtime', document.mtime),\n"
" sc.json_data)\n"
" ELSE json_set(json_patch(document.json_data, sc.json_data), '$._id',\n"
" document.id, '$.size', document.size, '$.mtime',\n"
" document.mtime, '$.tag',\n"
" json_group_array(t.tag)) END END as json_data\n"
" FROM document\n"
" LEFT JOIN document_sidecar sc ON document.id = sc.id\n"
" LEFT JOIN tag t ON document.id = t.id\n"
" GROUP BY document.id)\n"
"INSERT\n"
"INTO fts.document_index (id, index_id, size, path, path_depth, mtime, mime, json_data)\n"
"SELECT *\n"
"FROM docs\n"
"WHERE true\n"
"on conflict (id, index_id) do update set size=excluded.size,\n"
" mtime=excluded.mtime,\n"
" json_data=excluded.json_data;",
NULL, NULL, NULL));
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"DELETE\n"
"FROM fts.document_index\n"
"WHERE id IN (SELECT id FROM delete_list)\n"
" AND index_id = (SELECT id FROM descriptor);",
NULL, NULL, NULL
));
}
void database_fts_optimize(database_t *db) {
LOG_INFO("database_fts.c", "Optimizing search index.");
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
db->db,
"INSERT INTO search(search) VALUES('optimize');",
NULL, NULL, NULL));
LOG_DEBUG("database_fts.c", "Optimized fts5 table.");
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA fts.optimize;", NULL, NULL, NULL));
LOG_DEBUG("database_fts.c", "optimized indices.");
}

42
src/database/database_schema.c
View File

@@ -1,3 +1,45 @@
const char *FtsDatabaseSchema =
"CREATE TABLE IF NOT EXISTS document_index ("
" id TEXT NOT NULL,"
" index_id TEXT NOT NULL,"
" size INTEGER NOT NULL,"
" path TEXT NOT NULL,"
" path_depth INT NOT NULL,"
" mtime INTEGER NOT NULL,"
" mime TEXT NOT NULL,"
" json_data TEXT NOT NULL,"
" PRIMARY KEY (id, index_id)"
");"
""
"CREATE VIEW IF NOT EXISTS document_view (rowid, name, content)"
" AS"
" SELECT rowid,"
" json_data->>'name',"
" json_data->>'content'"
" FROM document_index;"
""
"CREATE INDEX IF NOT EXISTS document_index_size_idx ON document_index (size);"
"CREATE INDEX IF NOT EXISTS document_index_mtime_idx ON document_index (mtime);"
"CREATE INDEX IF NOT EXISTS document_index_mime_idx ON document_index (mime);"
"CREATE INDEX IF NOT EXISTS document_index_path_idx ON document_index (path);"
"CREATE INDEX IF NOT EXISTS document_index_path_depth_idx ON document_index (path_depth);"
""
"CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 ("
" name,"
" content,"
" content='document_view'"
");"
""
"CREATE TRIGGER IF NOT EXISTS on_insert AFTER INSERT ON document_index BEGIN"
" INSERT INTO search(rowid, name, content) VALUES (new.rowid, new.json_data->>'name', new.json_data->>'content');"
"END;"
"CREATE TRIGGER IF NOT EXISTS on_delete AFTER DELETE ON document_index BEGIN"
" INSERT INTO search(search, name, content) VALUES('delete', old.json_data->>'name', old.json_data->>'content');"
"END;"
"CREATE TRIGGER IF NOT EXISTS on_update AFTER UPDATE ON document_index BEGIN"
" INSERT INTO search(search, rowid, name, content) VALUES('delete', old.rowid, old.json_data->>'name', old.json_data->>'content');"
" INSERT INTO search(rowid, name, content) VALUES (new.rowid, new.json_data->>'name', new.json_data->>'content');"
"END;";
const char *IpcDatabaseSchema =
"CREATE TABLE parse_job ("