Static build (scan only)

CMakeLists.txt

@@ -1,9 +1,13 @@
 cmake_minimum_required(VERSION 3.7)
 set(CMAKE_C_STANDARD 11)
 
+option(WITH_SIST2 "Build main executable" ON)
+option(WITH_SIST2_SCAN "Build scan executable" ON)
+
 project(sist2 C)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules")
 
+if (WITH_SIST2)
     add_executable(
             sist2
             src/main.c
@@ -32,7 +36,39 @@ add_executable(
             # LMDB
             lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
             lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
-        src/cli.c src/cli.h)
+            src/cli.c src/cli.h
+    )
+endif ()
+
+if (WITH_SIST2_SCAN)
+    add_executable(
+            sist2_scan
+            src/main.c
+            src/sist.h
+            src/io/walk.h src/io/walk.c
+            src/parsing/media.h src/parsing/media.c
+            src/parsing/pdf.h src/parsing/pdf.c
+            src/io/store.h src/io/store.c
+            src/tpool.h src/tpool.c
+            src/parsing/parse.h src/parsing/parse.c
+            src/io/serialize.h src/io/serialize.c
+            src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
+            src/parsing/text.h src/parsing/text.c
+            src/util.c src/util.h
+            src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
+
+            # argparse
+            argparse/argparse.h argparse/argparse.c
+
+            # cJSON
+            cJSON/cJSON.h cJSON/cJSON.c
+
+            # LMDB
+            lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
+            lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
+            src/cli.c src/cli.h
+    )
+endif ()
 
 find_package(PkgConfig REQUIRED)
 set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
@@ -46,41 +82,38 @@ pkg_check_modules(GLIB REQUIRED glib-2.0)
 pkg_check_modules(GOBJECT REQUIRED gobject-2.0)
 pkg_check_modules(UUID REQUIRED uuid)
 
-include_directories(${LIBMAGIC_INCLUDE_DIRS})
-link_directories(${LIBMAGIC_LIBRARY_DIRS})
-add_definitions(${LIBMAGIC_CFLAGS_OTHER})
-
-link_directories(${UUID_LIBRARY_DIRS})
-include_directories(${UUID_INCLUDE_DIRS})
-add_definitions(${UUID_CFLAGS_OTHER})
-
-include_directories(${GLIB_INCLUDE_DIRS})
-link_directories(${GLIB_LIBRARY_DIRS})
-add_definitions(${GLIB_CFLAGS_OTHER})
-
-include_directories(${GOBJECT_INCLUDE_DIRS})
-link_directories(${GOBJECT_LIBRARY_DIRS})
-add_definitions(${GOBJECT_CFLAGS_OTHER})
-
-link_directories(${FFMPEG_LIBRARY_DIRS})
-include_directories(${FFMPEG_INCLUDE_DIRS})
-
-include_directories(${OPENSSL_INCLUDE_DIR})
-link_directories(${OPENSSL_CRYPTO_LIBRARY})
 
 list(REMOVE_ITEM GLIB_LIBRARIES pcre)
 list(REMOVE_ITEM GOBJECT_LIBRARIES pcre)
 list(REMOVE_ITEM UUID_LIBRARIES pcre)
 
-include_directories(${FREETYPE_INCLUDE_DIRS})
-add_definitions(${FREETYPE_CFLAGS_OTHER})
-
-include_directories(
+if (WITH_SIST2)
+    target_include_directories(
+            sist2 PUBLIC
+            ${LIBMAGIC_INCLUDE_DIRS}
+            ${GOBJECT_INCLUDE_DIRS}
+            ${OPENSSL_INCLUDE_DIR}
+            ${FFMPEG_INCLUDE_DIRS}
+            ${GLIB_INCLUDE_DIRS}
+            ${OPENSSL_CRYPTO_LIBRARY}
+            ${FREETYPE_INCLUDE_DIRS}
+            ${UUID_INCLUDE_DIRS}
             ${PROJECT_SOURCE_DIR}/
             ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
             ${PROJECT_SOURCE_DIR}/lib/onion/src/
             ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
     )
+    target_link_directories(
+            sist2 PUBLIC
+            ${UUID_LIBRARY_DIRS}
+            ${FFMPEG_LIBRARY_DIRS}
+    )
+
+    add_definitions(${LIBMAGIC_CFLAGS_OTHER})
+    add_definitions(${UUID_CFLAGS_OTHER})
+    add_definitions(${GLIB_CFLAGS_OTHER})
+    add_definitions(${GOBJECT_CFLAGS_OTHER})
+    add_definitions(${FREETYPE_CFLAGS_OTHER})
 
     target_compile_options(sist2
             PRIVATE
@@ -120,8 +153,80 @@ TARGET_LINK_LIBRARIES(
             magic
     )
 
+endif ()
+
+if (WITH_SIST2_SCAN)
+
+    set_target_properties(
+            sist2_scan
+            PROPERTIES COMPILE_DEFINITIONS SIST_SCAN_ONLY
+    )
+    set_target_properties(
+            sist2_scan
+            PROPERTIES
+            COMPILE_DEFINITIONS SIST_SCAN_ONLY
+            LINK_FLAGS -static
+    )
+    target_include_directories(
+            sist2_scan PUBLIC
+            ${LIBMAGIC_INCLUDE_DIRS}
+            ${GOBJECT_INCLUDE_DIRS}
+            ${OPENSSL_INCLUDE_DIR}
+            ${FFMPEG_INCLUDE_DIRS}
+            ${GLIB_INCLUDE_DIRS}
+            ${UUID_INCLUDE_DIRS}
+            ${FREETYPE_INCLUDE_DIRS}
+            ${PROJECT_SOURCE_DIR}/
+            ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
+            ${PROJECT_SOURCE_DIR}/lib/onion/src/
+            ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
+    )
+    target_link_directories(
+            sist2_scan PUBLIC
+            ${UUID_LIBRARY_DIRS}
+            ${FFMPEG_LIBRARY_DIRS}
+            ${OPENSSL_CRYPTO_LIBRARY}
+    )
+    target_compile_options(sist2_scan
+            PRIVATE
+            -O3
+            #        -march=native
+            -fno-stack-protector
+            -fomit-frame-pointer
+            )
+    TARGET_LINK_LIBRARIES(
+            sist2_scan
+
+            ${GLIB_LIBRARIES}
+            ${GOBJECT_LIBRARIES}
+            ${UUID_LIBRARIES}
+
+            # ffmpeg
+            ${PROJECT_SOURCE_DIR}/lib/libavcodec.a
+            ${PROJECT_SOURCE_DIR}/lib/libavformat.a
+            ${PROJECT_SOURCE_DIR}/lib/libavutil.a
+            ${PROJECT_SOURCE_DIR}/lib/libswscale.a
+            ${PROJECT_SOURCE_DIR}/lib/libswresample.a
+
+            # mupdf
+            ${PROJECT_SOURCE_DIR}/lib/libmupdf.a
+            ${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
+
+            ${PROJECT_SOURCE_DIR}/lib/libbz2.a
+            ${PROJECT_SOURCE_DIR}/lib/libmagic.a
+
+            pthread
+            m
+    )
+endif ()
+
 add_custom_target(
         before_sist2
         COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/before_build.sh
 )
+
+IF (WITH_SIST2)
     add_dependencies(sist2 before_sist2)
+else ()
+    add_dependencies(sist2_scan before_sist2)
+endif ()

scripts/before_build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 rm -rf index.sist2/
 

scripts/get_static_libs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 cd lib
 
 cd mupdf
@@ -36,4 +36,22 @@ make -j 4
 cd ../..
 
 mv onion/build/src/onion/libonion_static.a .
+
+#bzip2
+git clone https://github.com/enthought/bzip2-1.0.6
+cd bzip2-1.0.6
+make -j 4
+cd ..
+mv bzip2-1.0.6/libbz2.a .
+
+# magic
+git clone https://github.com/threatstack/libmagic
+cd libmagic
+./autogen.sh
+./configure --enable-static --disable-shared
+make -j 4
+cd ..
+mv libmagic/src/.libs/libmagic.a .
+
+
 cd ..

scripts/get_static_libs_freebsd.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+cd lib
+
+# mupdf
+cd mupdf
+HAVE_X11=no HAVE_GLUT=no gmake -j 4
+cd ..
+
+mv mupdf/build/release/libmupdf.a .
+mv mupdf/build/release/libmupdf-third.a .
+
+# ffmpeg
+cd ffmpeg
+./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay \
+ --disable-ffprobe --disable-doc\
+ --disable-manpages --disable-postproc --disable-avfilter \
+ --disable-alsa --disable-lzma --disable-xlib --disable-debug\
+ --disable-vdpau --disable-vaapi --disable-sdl2 --disable-network
+gmake -j 4
+cd ..
+
+mv ffmpeg/libavcodec/libavcodec.a .
+mv ffmpeg/libavformat/libavformat.a .
+mv ffmpeg/libavutil/libavutil.a .
+mv ffmpeg/libswresample/libswresample.a .
+mv ffmpeg/libswscale/libswscale.a .
+
+#bzip2
+git clone https://github.com/enthought/bzip2-1.0.6
+cd bzip2-1.0.6
+make -j 4
+cd ..
+mv bzip2-1.0.6/libbz2.a .
+
+# magic
+git clone https://github.com/threatstack/libmagic
+cd libmagic
+./autogen.sh
+./configure --enable-static --disable-shared
+make -j 4
+cd ..
+mv libmagic/src/.libs/libmagic.a .
+
+cd ..

src/cli.c

@@ -17,16 +17,6 @@ scan_args_t *scan_args_create() {
     return args;
 }
 
-index_args_t *index_args_create() {
-    index_args_t *args = calloc(sizeof(index_args_t), 1);
-    return args;
-}
-
-web_args_t *web_args_create() {
-    web_args_t *args = calloc(sizeof(web_args_t), 1);
-    return args;
-}
-
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");
@@ -100,6 +90,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
+#ifndef SIST_SCAN_ONLY
 int index_args_validate(index_args_t *args, int argc, const char **argv) {
 
     if (argc < 2) {
@@ -153,3 +144,14 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
+index_args_t *index_args_create() {
+    index_args_t *args = calloc(sizeof(index_args_t), 1);
+    return args;
+}
+
+web_args_t *web_args_create() {
+    web_args_t *args = calloc(sizeof(web_args_t), 1);
+    return args;
+}
+#endif
+

src/cli.h

@@ -15,6 +15,10 @@ typedef struct scan_args {
     char *path;
 } scan_args_t;
 
+scan_args_t *scan_args_create();
+int scan_args_validate(scan_args_t *args, int argc, const char **argv);
+
+#ifndef SIST_SCAN_ONLY
 typedef struct index_args {
     char *es_url;
     const char *index_path;
@@ -30,12 +34,11 @@ typedef struct web_args {
     const char **indices;
 } web_args_t;
 
-scan_args_t *scan_args_create();
 index_args_t *index_args_create();
 web_args_t *web_args_create();
 
-int scan_args_validate(scan_args_t *args, int argc, const char **argv);
 int index_args_validate(index_args_t *args, int argc, const char **argv);
 int web_args_validate(web_args_t *args, int argc, const char **argv);
+#endif
 
 #endif

src/ctx.h

@@ -26,6 +26,7 @@ struct {
 } ScanCtx;
 
 
+#ifndef SIST_SCAN_ONLY
 struct {
     char *es_url;
 } IndexCtx;
@@ -35,6 +36,7 @@ struct {
     int index_count;
     struct index_t indices[16];
 } WebCtx;
+#endif
 
 
 #endif

src/io/serialize.c

@@ -115,8 +115,8 @@ void write_document(document_t *doc) {
 
     if (IndexFd == -1) {
         char dstfile[PATH_MAX];
-        pid_t tid = syscall(SYS_gettid);
-        snprintf(dstfile, PATH_MAX, "%s_index_%d", ScanCtx.index.path, tid);
+        pthread_t self = pthread_self();
+        snprintf(dstfile, PATH_MAX, "%s_index_%lu", ScanCtx.index.path, self);
         IndexFd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR);
 
         if (IndexFd == -1) {

src/main.c

@@ -1,7 +1,12 @@
 #include "sist.h"
 #include "ctx.h"
 
+#ifndef SIST_SCAN_ONLY
 #define DESCRIPTION "Lightning-fast file system indexer and search tool."
+#else
+#define DESCRIPTION "Lightning-fast file system indexer and search tool. (SCAN ONLY)"
+#endif
+
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
@@ -14,7 +19,9 @@ static const char *const usage[] = {
 };
 
 void global_init() {
+    #ifndef SIST_SCAN_ONLY
     curl_global_init(CURL_GLOBAL_NOTHING);
+    #endif
     av_log_set_level(AV_LOG_QUIET);
 }
 
@@ -23,7 +30,7 @@ void init_dir(const char *dirpath) {
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
 
     uuid_t uuid;
-    uuid_generate_time_safe(uuid);
+    uuid_generate(uuid);
     uuid_unparse(uuid, ScanCtx.index.desc.uuid);
     time(&ScanCtx.index.desc.timestamp);
     strcpy(ScanCtx.index.desc.version, Version);
@@ -116,6 +123,7 @@ void sist2_scan(scan_args_t *args) {
     store_destroy(ScanCtx.index.store);
 }
 
+#ifndef SIST_SCAN_ONLY
 void sist2_index(index_args_t *args) {
 
     IndexCtx.es_url = args->es_url;
@@ -186,6 +194,7 @@ void sist2_web(web_args_t *args) {
 
     serve(args->bind, args->port);
 }
+#endif
 
 
 int main(int argc, const char *argv[]) {
@@ -193,8 +202,10 @@ int main(int argc, const char *argv[]) {
     global_init();
 
     scan_args_t *scan_args = scan_args_create();
+    #ifndef SIST_SCAN_ONLY
     index_args_t *index_args = index_args_create();
     web_args_t *web_args = web_args_create();
+    #endif
 
     char * common_es_url = NULL;
 
@@ -214,6 +225,7 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."),
             OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: (name of the directory)"),
 
+            #ifndef SIST_SCAN_ONLY
             OPT_GROUP("Index options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
@@ -224,6 +236,7 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
             OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
             OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
+            #endif
 
             OPT_END(),
     };
@@ -233,8 +246,10 @@ int main(int argc, const char *argv[]) {
     argparse_describe(&argparse, DESCRIPTION, EPILOG);
     argc = argparse_parse(&argparse, argc, argv);
 
+    #ifndef SIST_SCAN_ONLY
     web_args->es_url = common_es_url;
     index_args->es_url = common_es_url;
+    #endif
 
     if (argc == 0) {
         argparse_usage(&argparse);
@@ -247,7 +262,10 @@ int main(int argc, const char *argv[]) {
         }
         sist2_scan(scan_args);
 
-    } else if (strcmp(argv[0], "index") == 0) {
+    }
+
+    #ifndef SIST_SCAN_ONLY
+    else if (strcmp(argv[0], "index") == 0) {
 
         int err = index_args_validate(index_args, argc, argv);
         if (err != 0) {
@@ -263,7 +281,9 @@ int main(int argc, const char *argv[]) {
         }
         sist2_web(web_args);
 
-    } else {
+    }
+    #endif
+    else {
         fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
         argparse_usage(&argparse);
         return 1;

src/parsing/parse.c

@@ -57,7 +57,7 @@ void parse(void *arg) {
     doc.ino = job->info.st_ino;
     doc.mtime = job->info.st_mtim.tv_sec;
 
-    uuid_generate_time_safe(doc.uuid);
+    uuid_generate(doc.uuid);
     char *buf[PARSE_BUF_SIZE];
 
     if (job->info.st_size == 0) {

src/sist.h

@@ -25,19 +25,20 @@
 #include <pthread.h>
 #include <sys/stat.h>
 #include <wordexp.h>
+
+#ifndef SIST_SCAN_ONLY
 #include <onion/onion.h>
 #include <onion/handler.h>
 #include <onion/block.h>
 #include <onion/shortcuts.h>
 #include <curl/curl.h>
-
+#endif
 
 #include "cJSON/cJSON.h"
 
 #include "types.h"
 #include "tpool.h"
 #include "util.h"
-#include "src/index/elastic.h"
 #include "io/store.h"
 #include "io/serialize.h"
 #include "io/walk.h"
@@ -47,9 +48,13 @@
 #include "parsing/pdf.h"
 #include "parsing/media.h"
 #include "parsing/font.h"
+#include "cli.h"
+
+#ifndef SIST_SCAN_ONLY
+#include "src/index/elastic.h"
 #include "index/web.h"
 #include "web/serve.h"
-#include "cli.h"
+#endif
 
 ;
 

src/util.c

@@ -1,6 +1,133 @@
-#define _GNU_SOURCE
 #include "util.h"
 
+dyn_buffer_t dyn_buffer_create() {
+    dyn_buffer_t buf;
+
+    buf.size = INITIAL_BUF_SIZE;
+    buf.cur = 0;
+    buf.buf = malloc(INITIAL_BUF_SIZE);
+
+    return buf;
+}
+
+void grow_buffer(dyn_buffer_t *buf, size_t size) {
+    if (buf->cur + size > buf->size) {
+        do {
+            buf->size *= 2;
+        } while (buf->cur + size > buf->size);
+
+        buf->buf = realloc(buf->buf, buf->size);
+    }
+}
+
+void grow_buffer_small(dyn_buffer_t *buf) {
+    if (buf->cur + sizeof(long) > buf->size) {
+        buf->size *= 2;
+        buf->buf = realloc(buf->buf, buf->size);
+    }
+}
+
+void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size) {
+    grow_buffer(buf, size);
+
+    memcpy(buf->buf + buf->cur, data, size);
+    buf->cur += size;
+}
+
+void dyn_buffer_write_char(dyn_buffer_t *buf, char c) {
+    grow_buffer_small(buf);
+
+    *(buf->buf + buf->cur) = c;
+    buf->cur += sizeof(c);
+}
+
+void dyn_buffer_write_str(dyn_buffer_t *buf, char *str) {
+    dyn_buffer_write(buf, str, strlen(str));
+    dyn_buffer_write_char(buf, '\0');
+}
+
+void dyn_buffer_write_int(dyn_buffer_t *buf, int d) {
+    grow_buffer_small(buf);
+
+    *(int *) (buf->buf + buf->cur) = d;
+    buf->cur += sizeof(int);
+}
+
+void dyn_buffer_write_short(dyn_buffer_t *buf, short s) {
+    grow_buffer_small(buf);
+
+    *(short *) (buf->buf + buf->cur) = s;
+    buf->cur += sizeof(short);
+}
+
+void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l) {
+    grow_buffer_small(buf);
+
+    *(unsigned long *) (buf->buf + buf->cur) = l;
+    buf->cur += sizeof(unsigned long);
+}
+
+void dyn_buffer_destroy(dyn_buffer_t *buf) {
+    free(buf->buf);
+}
+
+void text_buffer_destroy(text_buffer_t *buf) {
+    dyn_buffer_destroy(&buf->dyn_buffer);
+}
+
+text_buffer_t text_buffer_create(int max_size) {
+    text_buffer_t text_buf;
+
+    text_buf.dyn_buffer = dyn_buffer_create();
+    text_buf.max_size = max_size;
+    text_buf.last_char_was_whitespace = FALSE;
+
+    return text_buf;
+}
+
+void text_buffer_terminate_string(text_buffer_t *buf) {
+    dyn_buffer_write_char(&buf->dyn_buffer, '\0');
+}
+
+int text_buffer_append_char(text_buffer_t *buf, int c) {
+
+    if (SHOULD_IGNORE_CHAR(c)) {
+        if (!buf->last_char_was_whitespace) {
+            dyn_buffer_write_char(&buf->dyn_buffer, ' ');
+            buf->last_char_was_whitespace = TRUE;
+
+            if (buf->dyn_buffer.cur >= buf->max_size) {
+                return TEXT_BUF_FULL;
+            }
+        }
+    } else {
+        buf->last_char_was_whitespace = FALSE;
+        dyn_buffer_write_char(&buf->dyn_buffer, (char) c);
+
+        if (buf->dyn_buffer.cur >= buf->max_size) {
+            return TEXT_BUF_FULL;
+        }
+    }
+
+    return 0;
+}
+
+void incremental_put(GHashTable *table, unsigned long inode_no, int mtime) {
+    g_hash_table_insert(table, (gpointer) inode_no, GINT_TO_POINTER(mtime));
+}
+
+int incremental_get(GHashTable *table, unsigned long inode_no) {
+    if (table != NULL) {
+        return GPOINTER_TO_INT(g_hash_table_lookup(table, (gpointer) inode_no));
+    } else {
+        return 0;
+    }
+}
+
+int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no) {
+    g_hash_table_insert(table, GINT_TO_POINTER(inode_no), GINT_TO_POINTER(1));
+}
+
 
 #define PBSTR "========================================"
 #define PBWIDTH 40
@@ -27,7 +154,7 @@ char *abspath(const char *path) {
     wordexp_t w;
     wordexp(path, &w, 0);
 
-    char *abs = canonicalize_file_name(w.we_wordv[0]);
+    char *abs = realpath(w.we_wordv[0], NULL);
     if (abs == NULL) {
         return NULL;
     }

src/util.h

@@ -21,132 +21,6 @@ typedef struct text_buffer {
     dyn_buffer_t dyn_buffer;
 } text_buffer_t;
 
-__always_inline
-dyn_buffer_t dyn_buffer_create() {
-    dyn_buffer_t buf;
-
-    buf.size = INITIAL_BUF_SIZE;
-    buf.cur = 0;
-    buf.buf = malloc(INITIAL_BUF_SIZE);
-
-    return buf;
-}
-
-__always_inline
-void grow_buffer(dyn_buffer_t *buf, size_t size) {
-    if (buf->cur + size > buf->size) {
-        do {
-            buf->size *= 2;
-        } while (buf->cur + size > buf->size);
-
-        buf->buf = realloc(buf->buf, buf->size);
-    }
-}
-
-__always_inline
-void grow_buffer_small(dyn_buffer_t *buf) {
-    if (buf->cur + sizeof(long) > buf->size) {
-        buf->size *= 2;
-        buf->buf = realloc(buf->buf, buf->size);
-    }
-}
-
-__always_inline
-void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size) {
-    grow_buffer(buf, size);
-
-    memcpy(buf->buf + buf->cur, data, size);
-    buf->cur += size;
-}
-
-__always_inline
-void dyn_buffer_write_char(dyn_buffer_t *buf, char c) {
-    grow_buffer_small(buf);
-
-    *(buf->buf + buf->cur) = c;
-    buf->cur += sizeof(c);
-}
-
-__always_inline
-void dyn_buffer_write_str(dyn_buffer_t *buf, char *str) {
-    dyn_buffer_write(buf, str, strlen(str));
-    dyn_buffer_write_char(buf, '\0');
-}
-
-__always_inline
-void dyn_buffer_write_int(dyn_buffer_t *buf, int d) {
-    grow_buffer_small(buf);
-
-    *(int *) (buf->buf + buf->cur) = d;
-    buf->cur += sizeof(int);
-}
-
-__always_inline
-void dyn_buffer_write_short(dyn_buffer_t *buf, short s) {
-    grow_buffer_small(buf);
-
-    *(short *) (buf->buf + buf->cur) = s;
-    buf->cur += sizeof(short);
-}
-
-__always_inline
-void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l) {
-    grow_buffer_small(buf);
-
-    *(unsigned long *) (buf->buf + buf->cur) = l;
-    buf->cur += sizeof(unsigned long);
-}
-
-__always_inline
-void dyn_buffer_destroy(dyn_buffer_t *buf) {
-    free(buf->buf);
-}
-
-__always_inline
-void text_buffer_destroy(text_buffer_t *buf) {
-    dyn_buffer_destroy(&buf->dyn_buffer);
-}
-
-__always_inline
-text_buffer_t text_buffer_create(int max_size) {
-    text_buffer_t text_buf;
-
-    text_buf.dyn_buffer = dyn_buffer_create();
-    text_buf.max_size = max_size;
-    text_buf.last_char_was_whitespace = FALSE;
-
-    return text_buf;
-}
-
-__always_inline
-void text_buffer_terminate_string(text_buffer_t *buf) {
-    dyn_buffer_write_char(&buf->dyn_buffer, '\0');
-}
-
-__always_inline
-int text_buffer_append_char(text_buffer_t *buf, int c) {
-
-    if (SHOULD_IGNORE_CHAR(c)) {
-        if (!buf->last_char_was_whitespace) {
-            dyn_buffer_write_char(&buf->dyn_buffer, ' ');
-            buf->last_char_was_whitespace = TRUE;
-
-            if (buf->dyn_buffer.cur >= buf->max_size) {
-                return TEXT_BUF_FULL;
-            }
-        }
-    } else {
-        buf->last_char_was_whitespace = FALSE;
-        dyn_buffer_write_char(&buf->dyn_buffer, (char) c);
-
-        if (buf->dyn_buffer.cur >= buf->max_size) {
-            return TEXT_BUF_FULL;
-        }
-    }
-
-    return 0;
-}
-
 char *abspath(const char * path);
 char *expandpath(const char *path);
 dyn_buffer_t url_escape(char *str);
@@ -154,27 +28,41 @@ dyn_buffer_t url_escape(char *str);
 void progress_bar_print(double percentage, size_t tn_size, size_t index_size);
 
 
-__always_inline
-void incremental_put(GHashTable *table, unsigned long inode_no, int mtime) {
-    g_hash_table_insert(table, (gpointer) inode_no, GINT_TO_POINTER(mtime));
-}
-
-__always_inline
-int incremental_get(GHashTable *table, unsigned long inode_no) {
-    if (table != NULL) {
-        return GPOINTER_TO_INT(g_hash_table_lookup(table, (gpointer) inode_no));
-    } else {
-        return 0;
-    }
-}
-
-__always_inline
-int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no) {
-    g_hash_table_insert(table, GINT_TO_POINTER(inode_no), GINT_TO_POINTER(1));
-}
-
 GHashTable *incremental_get_table();
 
+dyn_buffer_t dyn_buffer_create();
+
+void grow_buffer(dyn_buffer_t *buf, size_t size);
+
+void grow_buffer_small(dyn_buffer_t *buf);
+
+void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size);
+
+void dyn_buffer_write_char(dyn_buffer_t *buf, char c);
+
+void dyn_buffer_write_str(dyn_buffer_t *buf, char *str);
+
+void dyn_buffer_write_int(dyn_buffer_t *buf, int d);
+
+void dyn_buffer_write_short(dyn_buffer_t *buf, short s);
+
+void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l);
+
+void dyn_buffer_destroy(dyn_buffer_t *buf);
+
+void text_buffer_destroy(text_buffer_t *buf);
+
+text_buffer_t text_buffer_create(int max_size);
+
+void text_buffer_terminate_string(text_buffer_t *buf);
+
+int text_buffer_append_char(text_buffer_t *buf, int c);
+
+void incremental_put(GHashTable *table, unsigned long inode_no, int mtime);
+
+int incremental_get(GHashTable *table, unsigned long inode_no);
+int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no);
+
 
 
 #endif

web/js/search.js

@@ -29,7 +29,7 @@ $.jsonPost("i").then(resp => {
     resp["indices"].forEach(idx => {
         $("#indices").append($("<option>")
             .attr("value", idx.id)
-            .attr("selected", true)
+            .attr("selected", !idx.name.includes("(nsfw)"))
             .append(idx.name)
         );
         selectedIndices.push(idx.id);