Fix #81

Update USAGE.md
Log user script task, add async arg
2025-12-12 15:08:53 +00:00 · 2020-08-03 20:09:07 -04:00 · 2020-08-03 19:48:49 -04:00 · 2020-08-03 19:44:43 -04:00 · 2020-08-03 19:15:12 -04:00 · 2020-07-26 11:52:48 -04:00
127 changed files with 5395 additions and 5184 deletions
--- a/.github/ISSUE_TEMPLATE/issue-template.md
+++ b/.github/ISSUE_TEMPLATE/issue-template.md
@@ -0,0 +1,18 @@
+---
+name: Issue template
+about: General
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+sist2 version:
+
+Platform (Linux or Docker):
+
+Elasticsearch version:
+
+Command with arguments:  `ex: "scan ~/Documents -o ./i2 --threads 3 -q 1.0`
+
+If the issue is related to the `scan` module, please attach the files necessary to reproduce the error or email them to me[at]simon987.net.
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ bundle.js
 *.a
 vgcore.*
 build/
+third-party/
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,45 +1,6 @@
-[submodule "argparse"]
-	path = argparse
+[submodule "third-party/libscan"]
+	path = third-party/libscan
+	url = https://github.com/simon987/libscan
+[submodule "third-party/argparse"]
+	path = third-party/argparse
 	url = https://github.com/cofyc/argparse
-[submodule "cJSON"]
-	path = cJSON
-	url = https://github.com/DaveGamble/cJSON
-[submodule "lmdb"]
-	path = lmdb
-	url = https://github.com/LMDB/lmdb
-[submodule "utf8.h"]
-	path = utf8.h
-	url = https://github.com/sheredom/utf8.h
-[submodule "lib/bzip2-1.0.6"]
-	path = lib/bzip2-1.0.6
-	url = https://github.com/enthought/bzip2-1.0.6
-[submodule "lib/libmagic"]
-	path = lib/libmagic
-	url = https://github.com/threatstack/libmagic
-[submodule "lib/harfbuzz"]
-	path = lib/harfbuzz
-	url = https://github.com/harfbuzz/harfbuzz
-[submodule "lib/openjpeg"]
-	path = lib/openjpeg
-	url = https://github.com/uclouvain/openjpeg
-[submodule "lib/ffmpeg"]
-	path = lib/ffmpeg
-	url = https://git.ffmpeg.org/ffmpeg.git
-[submodule "lib/onion"]
-	path = lib/onion
-	url = https://github.com/davidmoreno/onion
-[submodule "lib/mupdf"]
-	path = lib/mupdf
-	url = git://git.ghostscript.com/mupdf.git
-[submodule "lib/tesseract"]
-	path = lib/tesseract
-	url = https://github.com/tesseract-ocr/tesseract
-[submodule "lib/leptonica"]
-	path = lib/leptonica
-	url = https://github.com/danbloomberg/leptonica
-[submodule "lib/libtiff"]
-	path = lib/libtiff
-	url = https://gitlab.com/libtiff/libtiff
-[submodule "lib/libpng"]
-	path = lib/libpng
-	url = https://github.com/glennrp/libpng
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,85 +2,55 @@ cmake_minimum_required(VERSION 3.7)
 set(CMAKE_C_STANDARD 11)

 project(sist2 C)
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules")

 option(SIST_DEBUG "Build a debug executable" on)

+add_subdirectory(third-party/libscan)
+set(ARGPARSE_SHARED off)
+add_subdirectory(third-party/argparse)
+
 add_executable(
        sist2
        src/main.c
        src/sist.h
        src/io/walk.h src/io/walk.c
-        src/parsing/media.h src/parsing/media.c
-        src/parsing/pdf.h src/parsing/pdf.c
        src/io/store.h src/io/store.c
        src/tpool.h src/tpool.c
        src/parsing/parse.h src/parsing/parse.c
        src/io/serialize.h src/io/serialize.c
        src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
-        src/parsing/text.h src/parsing/text.c
        src/index/web.c src/index/web.h
        src/web/serve.c src/web/serve.h
-        src/web/auth_basic.h src/web/auth_basic.c
        src/index/elastic.c src/index/elastic.h
        src/util.c src/util.h
-        src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
-        src/parsing/arc.c src/parsing/arc.h
-        src/parsing/doc.c src/parsing/doc.h
+        src/ctx.h src/types.h
        src/log.c src/log.h
-        src/parsing/cbr.h src/parsing/cbr.c

        # argparse
-        argparse/argparse.h argparse/argparse.c
+        third-party/argparse/argparse.h third-party/argparse/argparse.c

-        # cJSON
-        cJSON/cJSON.h cJSON/cJSON.c
-
-        # LMDB
-        lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
-        lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
        src/cli.c src/cli.h
+        src/stats.c src/stats.h src/ctx.c)

-        # utf8.h
-        utf8.h/utf8.h
-)
+target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
+set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)

-find_package(PkgConfig REQUIRED)
-set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
+find_package(lmdb CONFIG REQUIRED)
+find_package(cJSON CONFIG REQUIRED)
+find_package(unofficial-glib CONFIG REQUIRED)
+find_package(unofficial-mongoose CONFIG REQUIRED)
+find_library(UUID_LIB NAMES uuid)
+find_package(CURL CONFIG REQUIRED)

-find_package(Freetype REQUIRED)
+#find_package(OpenSSL REQUIRED)

-pkg_check_modules(GLIB REQUIRED glib-2.0)
-pkg_check_modules(GOBJECT REQUIRED gobject-2.0)
-pkg_check_modules(UUID REQUIRED uuid)
-
-add_definitions(${UUID_CFLAGS_OTHER})
-add_definitions(${GLIB_CFLAGS_OTHER})
-add_definitions(${GOBJECT_CFLAGS_OTHER})
-add_definitions(${FREETYPE_CFLAGS_OTHER})
-
-list(REMOVE_ITEM GLIB_LIBRARIES pcre)
-list(REMOVE_ITEM GOBJECT_LIBRARIES pcre)
-list(REMOVE_ITEM UUID_LIBRARIES pcre)

 target_include_directories(
        sist2 PUBLIC
-        ${GOBJECT_INCLUDE_DIRS}
-        ${GLIB_INCLUDE_DIRS}
-        ${PROJECT_SOURCE_DIR}/lib/ffmpeg/
-        ${FREETYPE_INCLUDE_DIRS}
-        ${UUID_INCLUDE_DIRS}
-        ${PROJECT_SOURCE_DIR}/
-        ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
-        ${PROJECT_SOURCE_DIR}/lib/onion/src/
-        ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
-        ${PROJECT_SOURCE_DIR}/include/
-        /usr/include/libxml2/
-        ${PROJECT_SOURCE_DIR}/lib/tesseract/include/
-)
-target_link_directories(
-        sist2 PUBLIC
-        ${UUID_LIBRARY_DIRS}
+        ${CMAKE_SOURCE_DIR}/third-party/onion/src/
+        ${CMAKE_SOURCE_DIR}/third-party/utf8.h/
+        ${CMAKE_SOURCE_DIR}/third-party/libscan/
+        ${CMAKE_SOURCE_DIR}/
 )

 target_compile_options(
@@ -97,11 +67,13 @@ if (SIST_DEBUG)
            -fstack-protector
            -fno-omit-frame-pointer
            -fsanitize=address
+            -O2
    )
    target_link_options(
            sist2
            PRIVATE
            -fsanitize=address
+#            -static
    )
    set_target_properties(
            sist2
@@ -109,6 +81,7 @@ if (SIST_DEBUG)
            OUTPUT_NAME sist2_debug
    )
 else ()
+#    set(VCPKG_BUILD_TYPE release)
    target_compile_options(
            sist2
            PRIVATE
@@ -118,53 +91,29 @@ else ()
    )
 endif ()

-TARGET_LINK_LIBRARIES(
+add_dependencies(
+        sist2
+        scan
+        argparse
+)
+
+target_link_libraries(
        sist2

-        ${GLIB_LIBRARIES}
-        ${GOBJECT_LIBRARIES}
-        ${UUID_LIBRARIES}
-
-        # ffmpeg
-        ${PROJECT_SOURCE_DIR}/lib/libavcodec.a
-        ${PROJECT_SOURCE_DIR}/lib/libavformat.a
-        ${PROJECT_SOURCE_DIR}/lib/libavutil.a
-        ${PROJECT_SOURCE_DIR}/lib/libswscale.a
-        ${PROJECT_SOURCE_DIR}/lib/libswresample.a
-
-        # mupdf
-        ${PROJECT_SOURCE_DIR}/lib/libmupdf.a
-        ${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
-
-        # onion
-        ${PROJECT_SOURCE_DIR}/lib/libonion_static.a
+        z
+        lmdb
+        cjson
+        argparse
+        unofficial::glib::glib
+        unofficial::mongoose::mongoose
+#        OpenSSL::SSL OpenSSL::Crypto
+        CURL::libcurl

+        ${UUID_LIB}
        pthread
-
-        m
-        bz2
-#        ${PROJECT_SOURCE_DIR}/lib/libmagic.a
        magic
-        ${PROJECT_SOURCE_DIR}/lib/libharfbuzz.a
-        ${PROJECT_SOURCE_DIR}/lib/libopenjp2.a
-        freetype
-        archive

-        xml2
-
-        ${PROJECT_SOURCE_DIR}/lib/libtesseract.a
-        ${PROJECT_SOURCE_DIR}/lib/liblept.a
-        ${PROJECT_SOURCE_DIR}/lib/libtiff.a
-        ${PROJECT_SOURCE_DIR}/lib/libpng16.a
-        stdc++
-
-        # curl
-        ${PROJECT_SOURCE_DIR}/lib/libcurl.a
-        ${PROJECT_SOURCE_DIR}/lib/libcrypto.a
-        ${PROJECT_SOURCE_DIR}/lib/libssl.a
-        dl
-
-        pcre
+        scan
 )

 add_custom_target(
--- a/CMakeModules/FindFFmpeg.cmake
+++ b/CMakeModules/FindFFmpeg.cmake
@@ -1,80 +0,0 @@
-# - Try to find ffmpeg libraries (libavcodec, libavformat and libavutil)
-# Once done this will define
-#
-# FFMPEG_FOUND - system has ffmpeg or libav
-# FFMPEG_INCLUDE_DIR - the ffmpeg include directory
-# FFMPEG_LIBRARIES - Link these to use ffmpeg
-# FFMPEG_LIBAVCODEC
-# FFMPEG_LIBAVFORMAT
-# FFMPEG_LIBAVUTIL
-#
-# Copyright (c) 2008 Andreas Schneider <mail@cynapses.org>
-# Modified for other libraries by Lasse Kärkkäinen <tronic>
-# Modified for Hedgewars by Stepik777
-#
-# Redistribution and use is allowed according to the terms of the New
-# BSD license.
-#
-
-if (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
-# in cache already
-set(FFMPEG_FOUND TRUE)
-else (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
-# use pkg-config to get the directories and then use these values
-# in the FIND_PATH() and FIND_LIBRARY() calls
-find_package(PkgConfig)
-if (PKG_CONFIG_FOUND)
-pkg_check_modules(_FFMPEG_AVCODEC libavcodec)
-pkg_check_modules(_FFMPEG_AVFORMAT libavformat)
-pkg_check_modules(_FFMPEG_AVUTIL libavutil)
-endif (PKG_CONFIG_FOUND)
-
-find_path(FFMPEG_AVCODEC_INCLUDE_DIR
-NAMES libavcodec/avcodec.h
-PATHS ${_FFMPEG_AVCODEC_INCLUDE_DIRS} /usr/include /usr/local/include /opt/local/include /sw/include
-PATH_SUFFIXES ffmpeg libav
-)
-
-find_library(FFMPEG_LIBAVCODEC
-NAMES avcodec
-PATHS ${_FFMPEG_AVCODEC_LIBRARY_DIRS} /usr/lib /usr/local/lib /opt/local/lib /sw/lib
-)
-
-find_library(FFMPEG_LIBAVFORMAT
-NAMES avformat
-PATHS ${_FFMPEG_AVFORMAT_LIBRARY_DIRS} /usr/lib /usr/local/lib /opt/local/lib /sw/lib
-)
-
-find_library(FFMPEG_LIBAVUTIL
-NAMES avutil
-PATHS ${_FFMPEG_AVUTIL_LIBRARY_DIRS} /usr/lib /usr/local/lib /opt/local/lib /sw/lib
-)
-
-if (FFMPEG_LIBAVCODEC AND FFMPEG_LIBAVFORMAT)
-set(FFMPEG_FOUND TRUE)
-endif()
-
-if (FFMPEG_FOUND)
-set(FFMPEG_INCLUDE_DIR ${FFMPEG_AVCODEC_INCLUDE_DIR})
-
-set(FFMPEG_LIBRARIES
-${FFMPEG_LIBAVCODEC}
-${FFMPEG_LIBAVFORMAT}
-${FFMPEG_LIBAVUTIL}
-)
-
-endif (FFMPEG_FOUND)
-
-if (FFMPEG_FOUND)
-if (NOT FFMPEG_FIND_QUIETLY)
-message(STATUS "Found FFMPEG or Libav: ${FFMPEG_LIBRARIES}, ${FFMPEG_INCLUDE_DIR}")
-endif (NOT FFMPEG_FIND_QUIETLY)
-else (FFMPEG_FOUND)
-if (FFMPEG_FIND_REQUIRED)
-message(FATAL_ERROR "Could not find libavcodec or libavformat or libavutil")
-endif (FFMPEG_FIND_REQUIRED)
-endif (FFMPEG_FOUND)
-
-endif (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
-
-
--- a/CMakeModules/FindLibMagic.cmake
+++ b/CMakeModules/FindLibMagic.cmake
@@ -1,100 +0,0 @@
-
-#-------------------------------------------------------------------------------
-# Copyright (c) 2013-2013, Lars Baehren <lbaehren@gmail.com>
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#  * Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#-------------------------------------------------------------------------------
-
-# - Check for the presence of LIBMAGIC
-#
-# The following variables are set when LIBMAGIC is found:
-#  LIBMAGIC_FOUND      = Set to true, if all components of LIBMAGIC have been
-#                        found.
-#  LIBMAGIC_INCLUDES   = Include path for the header files of LIBMAGIC
-#  LIBMAGIC_LIBRARIES  = Link these to use LIBMAGIC
-#  LIBMAGIC_LFLAGS     = Linker flags (optional)
-
-if (NOT LIBMAGIC_FOUND)
-
-  if (NOT LIBMAGIC_ROOT_DIR)
-    set (LIBMAGIC_ROOT_DIR ${CMAKE_INSTALL_PREFIX})
-  endif (NOT LIBMAGIC_ROOT_DIR)
-
-  ##____________________________________________________________________________
-  ## Check for the header files
-
-  find_path (LIBMAGIC_FILE_H
-    NAMES file/file.h
-    HINTS ${LIBMAGIC_ROOT_DIR} ${CMAKE_INSTALL_PREFIX}
-    PATH_SUFFIXES include
-    )
-  if (LIBMAGIC_FILE_H)
-    list (APPEND LIBMAGIC_INCLUDES ${LIBMAGIC_FILE_H})
-  endif (LIBMAGIC_FILE_H)
-
-  find_path (LIBMAGIC_MAGIC_H
-    NAMES magic.h
-    HINTS ${LIBMAGIC_ROOT_DIR} ${CMAKE_INSTALL_PREFIX}
-    PATH_SUFFIXES include include/linux
-    )
-  if (LIBMAGIC_MAGIC_H)
-    list (APPEND LIBMAGIC_INCLUDES ${LIBMAGIC_MAGIC_H})
-  endif (LIBMAGIC_MAGIC_H)
-
-  list (REMOVE_DUPLICATES LIBMAGIC_INCLUDES)
-
-  ##____________________________________________________________________________
-  ## Check for the library
-
-  find_library (LIBMAGIC_LIBRARIES magic
-    HINTS ${LIBMAGIC_ROOT_DIR} ${CMAKE_INSTALL_PREFIX}
-    PATH_SUFFIXES lib
-    )
-
-  ##____________________________________________________________________________
-  ## Actions taken when all components have been found
-
-  #find_package_handle_standard_args (LIBMAGIC DEFAULT_MSG LIBMAGIC_LIBRARIES LIBMAGIC_INCLUDES)
-
-  if (LIBMAGIC_FOUND)
-    if (NOT LIBMAGIC_FIND_QUIETLY)
-      message (STATUS "Found components for LIBMAGIC")
-      message (STATUS "LIBMAGIC_ROOT_DIR  = ${LIBMAGIC_ROOT_DIR}")
-      message (STATUS "LIBMAGIC_INCLUDES  = ${LIBMAGIC_INCLUDES}")
-      message (STATUS "LIBMAGIC_LIBRARIES = ${LIBMAGIC_LIBRARIES}")
-    endif (NOT LIBMAGIC_FIND_QUIETLY)
-  else (LIBMAGIC_FOUND)
-    if (LIBMAGIC_FIND_REQUIRED)
-      message (FATAL_ERROR "Could not find LIBMAGIC!")
-    endif (LIBMAGIC_FIND_REQUIRED)
-  endif (LIBMAGIC_FOUND)
-
-  ##____________________________________________________________________________
-  ## Mark advanced variables
-
-  mark_as_advanced (
-    LIBMAGIC_ROOT_DIR
-    LIBMAGIC_INCLUDES
-    LIBMAGIC_LIBRARIES
-    )
-
-endif (NOT LIBMAGIC_FOUND)
--- a/CMakeModules/FindOpenSSL.cmake
+++ b/CMakeModules/FindOpenSSL.cmake
@@ -1,478 +0,0 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
-macro(_OpenSSL_test_and_find_dependencies ssl_library crypto_library)
-  if((CMAKE_SYSTEM_NAME STREQUAL "Linux") AND
-     (("${ssl_library}" MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$") OR
-      ("${crypto_library}" MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$")))
-    set(_OpenSSL_has_dependencies TRUE)
-    find_package(Threads)
-  else()
-    set(_OpenSSL_has_dependencies FALSE)
-  endif()
-endmacro()
-
-function(_OpenSSL_add_dependencies libraries_var library)
-  if(CMAKE_THREAD_LIBS_INIT)
-    list(APPEND ${libraries_var} ${CMAKE_THREAD_LIBS_INIT})
-  endif()
-  list(APPEND ${libraries_var} ${CMAKE_DL_LIBS})
-  set(${libraries_var} ${${libraries_var}} PARENT_SCOPE)
-endfunction()
-
-function(_OpenSSL_target_add_dependencies target)
-  if(_OpenSSL_has_dependencies)
-    set_property( TARGET ${target} APPEND PROPERTY INTERFACE_LINK_LIBRARIES Threads::Threads )
-    set_property( TARGET ${target} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS} )
-  endif()
-endfunction()
-
-if (UNIX)
-  find_package(PkgConfig QUIET)
-  pkg_check_modules(_OPENSSL QUIET openssl)
-endif ()
-
-# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES
-if(OPENSSL_USE_STATIC_LIBS)
-  set(_openssl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-  if(WIN32)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
-  else()
-    set(CMAKE_FIND_LIBRARY_SUFFIXES .a )
-  endif()
-endif()
-
-if (WIN32)
-  # http://www.slproweb.com/products/Win32OpenSSL.html
-  set(_OPENSSL_ROOT_HINTS
-    ${OPENSSL_ROOT_DIR}
-    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\OpenSSL (32-bit)_is1;Inno Setup: App Path]"
-    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\OpenSSL (64-bit)_is1;Inno Setup: App Path]"
-    ENV OPENSSL_ROOT_DIR
-    )
-  file(TO_CMAKE_PATH "$ENV{PROGRAMFILES}" _programfiles)
-  set(_OPENSSL_ROOT_PATHS
-    "${_programfiles}/OpenSSL"
-    "${_programfiles}/OpenSSL-Win32"
-    "${_programfiles}/OpenSSL-Win64"
-    "C:/OpenSSL/"
-    "C:/OpenSSL-Win32/"
-    "C:/OpenSSL-Win64/"
-    )
-  unset(_programfiles)
-else ()
-  set(_OPENSSL_ROOT_HINTS
-    ${OPENSSL_ROOT_DIR}
-    ENV OPENSSL_ROOT_DIR
-    )
-endif ()
-
-set(_OPENSSL_ROOT_HINTS_AND_PATHS
-    HINTS ${_OPENSSL_ROOT_HINTS}
-    PATHS ${_OPENSSL_ROOT_PATHS}
-    )
-
-find_path(OPENSSL_INCLUDE_DIR
-  NAMES
-    openssl/ssl.h
-  ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-  HINTS
-    ${_OPENSSL_INCLUDEDIR}
-  PATH_SUFFIXES
-    include
-)
-
-if(WIN32 AND NOT CYGWIN)
-  if(MSVC)
-    # /MD and /MDd are the standard values - if someone wants to use
-    # others, the libnames have to change here too
-    # use also ssl and ssleay32 in debug as fallback for openssl < 0.9.8b
-    # enable OPENSSL_MSVC_STATIC_RT to get the libs build /MT (Multithreaded no-DLL)
-    # In Visual C++ naming convention each of these four kinds of Windows libraries has it's standard suffix:
-    #   * MD for dynamic-release
-    #   * MDd for dynamic-debug
-    #   * MT for static-release
-    #   * MTd for static-debug
-
-    # Implementation details:
-    # We are using the libraries located in the VC subdir instead of the parent directory even though :
-    # libeay32MD.lib is identical to ../libeay32.lib, and
-    # ssleay32MD.lib is identical to ../ssleay32.lib
-    # enable OPENSSL_USE_STATIC_LIBS to use the static libs located in lib/VC/static
-
-    if (OPENSSL_MSVC_STATIC_RT)
-      set(_OPENSSL_MSVC_RT_MODE "MT")
-    else ()
-      set(_OPENSSL_MSVC_RT_MODE "MD")
-    endif ()
-
-    # Since OpenSSL 1.1, lib names are like libcrypto32MTd.lib and libssl32MTd.lib
-    if( "${CMAKE_SIZEOF_VOID_P}" STREQUAL "8" )
-        set(_OPENSSL_MSVC_ARCH_SUFFIX "64")
-    else()
-        set(_OPENSSL_MSVC_ARCH_SUFFIX "32")
-    endif()
-
-    if(OPENSSL_USE_STATIC_LIBS)
-      set(_OPENSSL_PATH_SUFFIXES
-        "lib/VC/static"
-        "VC/static"
-        "lib"
-        )
-    else()
-      set(_OPENSSL_PATH_SUFFIXES
-        "lib/VC"
-        "VC"
-        "lib"
-        )
-    endif ()
-
-    find_library(LIB_EAY_DEBUG
-      NAMES
-        libcrypto${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}d
-        libcrypto${_OPENSSL_MSVC_RT_MODE}d
-        libcryptod
-        libeay32${_OPENSSL_MSVC_RT_MODE}d
-        libeay32d
-        cryptod
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    find_library(LIB_EAY_RELEASE
-      NAMES
-        libcrypto${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}
-        libcrypto${_OPENSSL_MSVC_RT_MODE}
-        libcrypto
-        libeay32${_OPENSSL_MSVC_RT_MODE}
-        libeay32
-        crypto
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    find_library(SSL_EAY_DEBUG
-      NAMES
-        libssl${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}d
-        libssl${_OPENSSL_MSVC_RT_MODE}d
-        libssld
-        ssleay32${_OPENSSL_MSVC_RT_MODE}d
-        ssleay32d
-        ssld
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    find_library(SSL_EAY_RELEASE
-      NAMES
-        libssl${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}
-        libssl${_OPENSSL_MSVC_RT_MODE}
-        libssl
-        ssleay32${_OPENSSL_MSVC_RT_MODE}
-        ssleay32
-        ssl
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    set(LIB_EAY_LIBRARY_DEBUG "${LIB_EAY_DEBUG}")
-    set(LIB_EAY_LIBRARY_RELEASE "${LIB_EAY_RELEASE}")
-    set(SSL_EAY_LIBRARY_DEBUG "${SSL_EAY_DEBUG}")
-    set(SSL_EAY_LIBRARY_RELEASE "${SSL_EAY_RELEASE}")
-
-    include(${CMAKE_CURRENT_LIST_DIR}/SelectLibraryConfigurations.cmake)
-    select_library_configurations(LIB_EAY)
-    select_library_configurations(SSL_EAY)
-
-    mark_as_advanced(LIB_EAY_LIBRARY_DEBUG LIB_EAY_LIBRARY_RELEASE
-                     SSL_EAY_LIBRARY_DEBUG SSL_EAY_LIBRARY_RELEASE)
-    set(OPENSSL_SSL_LIBRARY ${SSL_EAY_LIBRARY} )
-    set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY_LIBRARY} )
-  elseif(MINGW)
-    # same player, for MinGW
-    set(LIB_EAY_NAMES crypto libeay32)
-    set(SSL_EAY_NAMES ssl ssleay32)
-    find_library(LIB_EAY
-      NAMES
-        ${LIB_EAY_NAMES}
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        "lib/MinGW"
-        "lib"
-    )
-
-    find_library(SSL_EAY
-      NAMES
-        ${SSL_EAY_NAMES}
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        "lib/MinGW"
-        "lib"
-    )
-
-    mark_as_advanced(SSL_EAY LIB_EAY)
-    set(OPENSSL_SSL_LIBRARY ${SSL_EAY} )
-    set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY} )
-    unset(LIB_EAY_NAMES)
-    unset(SSL_EAY_NAMES)
-  else()
-    # Not sure what to pick for -say- intel, let's use the toplevel ones and hope someone report issues:
-    find_library(LIB_EAY
-      NAMES
-        libcrypto
-        libeay32
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      HINTS
-        ${_OPENSSL_LIBDIR}
-      PATH_SUFFIXES
-        lib
-    )
-
-    find_library(SSL_EAY
-      NAMES
-        libssl
-        ssleay32
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      HINTS
-        ${_OPENSSL_LIBDIR}
-      PATH_SUFFIXES
-        lib
-    )
-
-    mark_as_advanced(SSL_EAY LIB_EAY)
-    set(OPENSSL_SSL_LIBRARY ${SSL_EAY} )
-    set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY} )
-  endif()
-else()
-
-  find_library(OPENSSL_SSL_LIBRARY
-    NAMES
-      ssl
-      ssleay32
-      ssleay32MD
-    NAMES_PER_DIR
-    ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-    HINTS
-      ${_OPENSSL_LIBDIR}
-    PATH_SUFFIXES
-      lib
-  )
-
-  find_library(OPENSSL_CRYPTO_LIBRARY
-    NAMES
-      crypto
-    NAMES_PER_DIR
-    ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-    HINTS
-      ${_OPENSSL_LIBDIR}
-    PATH_SUFFIXES
-      lib
-  )
-
-  mark_as_advanced(OPENSSL_CRYPTO_LIBRARY OPENSSL_SSL_LIBRARY)
-
-endif()
-
-# compat defines
-set(OPENSSL_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY})
-set(OPENSSL_CRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY})
-_OpenSSL_test_and_find_dependencies("${OPENSSL_SSL_LIBRARY}" "${OPENSSL_CRYPTO_LIBRARY}")
-if(_OpenSSL_has_dependencies)
-  _OpenSSL_add_dependencies( OPENSSL_SSL_LIBRARIES "${OPENSSL_SSL_LIBRARY}" )
-  _OpenSSL_add_dependencies( OPENSSL_CRYPTO_LIBRARIES "${OPENSSL_CRYPTO_LIBRARY}" )
-endif()
-
-function(from_hex HEX DEC)
-  string(TOUPPER "${HEX}" HEX)
-  set(_res 0)
-  string(LENGTH "${HEX}" _strlen)
-
-  while (_strlen GREATER 0)
-    math(EXPR _res "${_res} * 16")
-    string(SUBSTRING "${HEX}" 0 1 NIBBLE)
-    string(SUBSTRING "${HEX}" 1 -1 HEX)
-    if (NIBBLE STREQUAL "A")
-      math(EXPR _res "${_res} + 10")
-    elseif (NIBBLE STREQUAL "B")
-      math(EXPR _res "${_res} + 11")
-    elseif (NIBBLE STREQUAL "C")
-      math(EXPR _res "${_res} + 12")
-    elseif (NIBBLE STREQUAL "D")
-      math(EXPR _res "${_res} + 13")
-    elseif (NIBBLE STREQUAL "E")
-      math(EXPR _res "${_res} + 14")
-    elseif (NIBBLE STREQUAL "F")
-      math(EXPR _res "${_res} + 15")
-    else()
-      math(EXPR _res "${_res} + ${NIBBLE}")
-    endif()
-
-    string(LENGTH "${HEX}" _strlen)
-  endwhile()
-
-  set(${DEC} ${_res} PARENT_SCOPE)
-endfunction()
-
-if(OPENSSL_INCLUDE_DIR AND EXISTS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h")
-  file(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h" openssl_version_str
-       REGEX "^#[\t ]*define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])+.*")
-
-  if(openssl_version_str)
-    # The version number is encoded as 0xMNNFFPPS: major minor fix patch status
-    # The status gives if this is a developer or prerelease and is ignored here.
-    # Major, minor, and fix directly translate into the version numbers shown in
-    # the string. The patch field translates to the single character suffix that
-    # indicates the bug fix state, which 00 -> nothing, 01 -> a, 02 -> b and so
-    # on.
-
-    string(REGEX REPLACE "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F]).*$"
-           "\\1;\\2;\\3;\\4;\\5" OPENSSL_VERSION_LIST "${openssl_version_str}")
-    list(GET OPENSSL_VERSION_LIST 0 OPENSSL_VERSION_MAJOR)
-    list(GET OPENSSL_VERSION_LIST 1 OPENSSL_VERSION_MINOR)
-    from_hex("${OPENSSL_VERSION_MINOR}" OPENSSL_VERSION_MINOR)
-    list(GET OPENSSL_VERSION_LIST 2 OPENSSL_VERSION_FIX)
-    from_hex("${OPENSSL_VERSION_FIX}" OPENSSL_VERSION_FIX)
-    list(GET OPENSSL_VERSION_LIST 3 OPENSSL_VERSION_PATCH)
-
-    if (NOT OPENSSL_VERSION_PATCH STREQUAL "00")
-      from_hex("${OPENSSL_VERSION_PATCH}" _tmp)
-      # 96 is the ASCII code of 'a' minus 1
-      math(EXPR OPENSSL_VERSION_PATCH_ASCII "${_tmp} + 96")
-      unset(_tmp)
-      # Once anyone knows how OpenSSL would call the patch versions beyond 'z'
-      # this should be updated to handle that, too. This has not happened yet
-      # so it is simply ignored here for now.
-      string(ASCII "${OPENSSL_VERSION_PATCH_ASCII}" OPENSSL_VERSION_PATCH_STRING)
-    endif ()
-
-    set(OPENSSL_VERSION "${OPENSSL_VERSION_MAJOR}.${OPENSSL_VERSION_MINOR}.${OPENSSL_VERSION_FIX}${OPENSSL_VERSION_PATCH_STRING}")
-  endif ()
-endif ()
-
-set(OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARIES} )
-list(REMOVE_DUPLICATES OPENSSL_LIBRARIES)
-
-foreach(_comp IN LISTS OpenSSL_FIND_COMPONENTS)
-  if(_comp STREQUAL "Crypto")
-    if(EXISTS "${OPENSSL_INCLUDE_DIR}" AND
-        (EXISTS "${OPENSSL_CRYPTO_LIBRARY}" OR
-        EXISTS "${LIB_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${LIB_EAY_LIBRARY_RELEASE}")
-    )
-      set(OpenSSL_${_comp}_FOUND TRUE)
-    else()
-      set(OpenSSL_${_comp}_FOUND FALSE)
-    endif()
-  elseif(_comp STREQUAL "SSL")
-    if(EXISTS "${OPENSSL_INCLUDE_DIR}" AND
-        (EXISTS "${OPENSSL_SSL_LIBRARY}" OR
-        EXISTS "${SSL_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${SSL_EAY_LIBRARY_RELEASE}")
-    )
-      set(OpenSSL_${_comp}_FOUND TRUE)
-    else()
-      set(OpenSSL_${_comp}_FOUND FALSE)
-    endif()
-  else()
-    message(WARNING "${_comp} is not a valid OpenSSL component")
-    set(OpenSSL_${_comp}_FOUND FALSE)
-  endif()
-endforeach()
-unset(_comp)
-
-include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
-find_package_handle_standard_args(OpenSSL
-  REQUIRED_VARS
-    OPENSSL_CRYPTO_LIBRARY
-    OPENSSL_INCLUDE_DIR
-  VERSION_VAR
-    OPENSSL_VERSION
-  HANDLE_COMPONENTS
-  FAIL_MESSAGE
-    "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR"
-)
-
-mark_as_advanced(OPENSSL_INCLUDE_DIR OPENSSL_LIBRARIES)
-
-if(OPENSSL_FOUND)
-  if(NOT TARGET OpenSSL::Crypto AND
-      (EXISTS "${OPENSSL_CRYPTO_LIBRARY}" OR
-        EXISTS "${LIB_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${LIB_EAY_LIBRARY_RELEASE}")
-      )
-    add_library(OpenSSL::Crypto UNKNOWN IMPORTED)
-    set_target_properties(OpenSSL::Crypto PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}")
-    if(EXISTS "${OPENSSL_CRYPTO_LIBRARY}")
-      set_target_properties(OpenSSL::Crypto PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-        IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}")
-    endif()
-    if(EXISTS "${LIB_EAY_LIBRARY_RELEASE}")
-      set_property(TARGET OpenSSL::Crypto APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS RELEASE)
-      set_target_properties(OpenSSL::Crypto PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C"
-        IMPORTED_LOCATION_RELEASE "${LIB_EAY_LIBRARY_RELEASE}")
-    endif()
-    if(EXISTS "${LIB_EAY_LIBRARY_DEBUG}")
-      set_property(TARGET OpenSSL::Crypto APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS DEBUG)
-      set_target_properties(OpenSSL::Crypto PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C"
-        IMPORTED_LOCATION_DEBUG "${LIB_EAY_LIBRARY_DEBUG}")
-    endif()
-    _OpenSSL_target_add_dependencies(OpenSSL::Crypto)
-  endif()
-
-  if(NOT TARGET OpenSSL::SSL AND
-      (EXISTS "${OPENSSL_SSL_LIBRARY}" OR
-        EXISTS "${SSL_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${SSL_EAY_LIBRARY_RELEASE}")
-      )
-    add_library(OpenSSL::SSL UNKNOWN IMPORTED)
-    set_target_properties(OpenSSL::SSL PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}")
-    if(EXISTS "${OPENSSL_SSL_LIBRARY}")
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-        IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}")
-    endif()
-    if(EXISTS "${SSL_EAY_LIBRARY_RELEASE}")
-      set_property(TARGET OpenSSL::SSL APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS RELEASE)
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C"
-        IMPORTED_LOCATION_RELEASE "${SSL_EAY_LIBRARY_RELEASE}")
-    endif()
-    if(EXISTS "${SSL_EAY_LIBRARY_DEBUG}")
-      set_property(TARGET OpenSSL::SSL APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS DEBUG)
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C"
-        IMPORTED_LOCATION_DEBUG "${SSL_EAY_LIBRARY_DEBUG}")
-    endif()
-    if(TARGET OpenSSL::Crypto)
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        INTERFACE_LINK_LIBRARIES OpenSSL::Crypto)
-    endif()
-    _OpenSSL_target_add_dependencies(OpenSSL::SSL)
-  endif()
-endif()
-
-# Restore the original find library ordering
-if(OPENSSL_USE_STATIC_LIBS)
-  set(CMAKE_FIND_LIBRARY_SUFFIXES ${_openssl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
-endif()
--- a/CMakeModules/FindPackageHandleStandardArgs.cmake
+++ b/CMakeModules/FindPackageHandleStandardArgs.cmake
@@ -1,268 +0,0 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
-include(${CMAKE_CURRENT_LIST_DIR}/FindPackageMessage.cmake)
-
-# internal helper macro
-macro(_FPHSA_FAILURE_MESSAGE _msg)
-  set (__msg "${_msg}")
-  if (FPHSA_REASON_FAILURE_MESSAGE)
-    string(APPEND __msg "\n    Reason given by package: ${FPHSA_REASON_FAILURE_MESSAGE}\n")
-  endif()
-  if (${_NAME}_FIND_REQUIRED)
-    message(FATAL_ERROR "${__msg}")
-  else ()
-    if (NOT ${_NAME}_FIND_QUIETLY)
-      message(STATUS "${__msg}")
-    endif ()
-  endif ()
-endmacro()
-
-
-# internal helper macro to generate the failure message when used in CONFIG_MODE:
-macro(_FPHSA_HANDLE_FAILURE_CONFIG_MODE)
-  # <PackageName>_CONFIG is set, but FOUND is false, this means that some other of the REQUIRED_VARS was not found:
-  if(${_NAME}_CONFIG)
-    _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE}: missing:${MISSING_VARS} (found ${${_NAME}_CONFIG} ${VERSION_MSG})")
-  else()
-    # If _CONSIDERED_CONFIGS is set, the config-file has been found, but no suitable version.
-    # List them all in the error message:
-    if(${_NAME}_CONSIDERED_CONFIGS)
-      set(configsText "")
-      list(LENGTH ${_NAME}_CONSIDERED_CONFIGS configsCount)
-      math(EXPR configsCount "${configsCount} - 1")
-      foreach(currentConfigIndex RANGE ${configsCount})
-        list(GET ${_NAME}_CONSIDERED_CONFIGS ${currentConfigIndex} filename)
-        list(GET ${_NAME}_CONSIDERED_VERSIONS ${currentConfigIndex} version)
-        string(APPEND configsText "\n    ${filename} (version ${version})")
-      endforeach()
-      if (${_NAME}_NOT_FOUND_MESSAGE)
-        if (FPHSA_REASON_FAILURE_MESSAGE)
-          string(PREPEND FPHSA_REASON_FAILURE_MESSAGE "${${_NAME}_NOT_FOUND_MESSAGE}\n    ")
-        else()
-          set(FPHSA_REASON_FAILURE_MESSAGE "${${_NAME}_NOT_FOUND_MESSAGE}")
-        endif()
-      else()
-        string(APPEND configsText "\n")
-      endif()
-      _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE} ${VERSION_MSG}, checked the following files:${configsText}")
-
-    else()
-      # Simple case: No Config-file was found at all:
-      _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE}: found neither ${_NAME}Config.cmake nor ${_NAME_LOWER}-config.cmake ${VERSION_MSG}")
-    endif()
-  endif()
-endmacro()
-
-
-function(FIND_PACKAGE_HANDLE_STANDARD_ARGS _NAME _FIRST_ARG)
-
-# Set up the arguments for `cmake_parse_arguments`.
-  set(options  CONFIG_MODE  HANDLE_COMPONENTS)
-  set(oneValueArgs  FAIL_MESSAGE  REASON_FAILURE_MESSAGE VERSION_VAR  FOUND_VAR)
-  set(multiValueArgs REQUIRED_VARS)
-
-# Check whether we are in 'simple' or 'extended' mode:
-  set(_KEYWORDS_FOR_EXTENDED_MODE  ${options} ${oneValueArgs} ${multiValueArgs} )
-  list(FIND _KEYWORDS_FOR_EXTENDED_MODE "${_FIRST_ARG}" INDEX)
-
-  if(${INDEX} EQUAL -1)
-    set(FPHSA_FAIL_MESSAGE ${_FIRST_ARG})
-    set(FPHSA_REQUIRED_VARS ${ARGN})
-    set(FPHSA_VERSION_VAR)
-  else()
-    cmake_parse_arguments(FPHSA "${options}" "${oneValueArgs}" "${multiValueArgs}"  ${_FIRST_ARG} ${ARGN})
-
-    if(FPHSA_UNPARSED_ARGUMENTS)
-      message(FATAL_ERROR "Unknown keywords given to FIND_PACKAGE_HANDLE_STANDARD_ARGS(): \"${FPHSA_UNPARSED_ARGUMENTS}\"")
-    endif()
-
-    if(NOT FPHSA_FAIL_MESSAGE)
-      set(FPHSA_FAIL_MESSAGE  "DEFAULT_MSG")
-    endif()
-
-    # In config-mode, we rely on the variable <PackageName>_CONFIG, which is set by find_package()
-    # when it successfully found the config-file, including version checking:
-    if(FPHSA_CONFIG_MODE)
-      list(INSERT FPHSA_REQUIRED_VARS 0 ${_NAME}_CONFIG)
-      list(REMOVE_DUPLICATES FPHSA_REQUIRED_VARS)
-      set(FPHSA_VERSION_VAR ${_NAME}_VERSION)
-    endif()
-
-    if(NOT FPHSA_REQUIRED_VARS)
-      message(FATAL_ERROR "No REQUIRED_VARS specified for FIND_PACKAGE_HANDLE_STANDARD_ARGS()")
-    endif()
-  endif()
-
-# now that we collected all arguments, process them
-
-  if("x${FPHSA_FAIL_MESSAGE}" STREQUAL "xDEFAULT_MSG")
-    set(FPHSA_FAIL_MESSAGE "Could NOT find ${_NAME}")
-  endif()
-
-  list(GET FPHSA_REQUIRED_VARS 0 _FIRST_REQUIRED_VAR)
-
-  string(TOUPPER ${_NAME} _NAME_UPPER)
-  string(TOLOWER ${_NAME} _NAME_LOWER)
-
-  if(FPHSA_FOUND_VAR)
-    if(FPHSA_FOUND_VAR MATCHES "^${_NAME}_FOUND$"  OR  FPHSA_FOUND_VAR MATCHES "^${_NAME_UPPER}_FOUND$")
-      set(_FOUND_VAR ${FPHSA_FOUND_VAR})
-    else()
-      message(FATAL_ERROR "The argument for FOUND_VAR is \"${FPHSA_FOUND_VAR}\", but only \"${_NAME}_FOUND\" and \"${_NAME_UPPER}_FOUND\" are valid names.")
-    endif()
-  else()
-    set(_FOUND_VAR ${_NAME_UPPER}_FOUND)
-  endif()
-
-  # collect all variables which were not found, so they can be printed, so the
-  # user knows better what went wrong (#6375)
-  set(MISSING_VARS "")
-  set(DETAILS "")
-  # check if all passed variables are valid
-  set(FPHSA_FOUND_${_NAME} TRUE)
-  foreach(_CURRENT_VAR ${FPHSA_REQUIRED_VARS})
-    if(NOT ${_CURRENT_VAR})
-      set(FPHSA_FOUND_${_NAME} FALSE)
-      string(APPEND MISSING_VARS " ${_CURRENT_VAR}")
-    else()
-      string(APPEND DETAILS "[${${_CURRENT_VAR}}]")
-    endif()
-  endforeach()
-  if(FPHSA_FOUND_${_NAME})
-    set(${_NAME}_FOUND TRUE)
-    set(${_NAME_UPPER}_FOUND TRUE)
-  else()
-    set(${_NAME}_FOUND FALSE)
-    set(${_NAME_UPPER}_FOUND FALSE)
-  endif()
-
-  # component handling
-  unset(FOUND_COMPONENTS_MSG)
-  unset(MISSING_COMPONENTS_MSG)
-
-  if(FPHSA_HANDLE_COMPONENTS)
-    foreach(comp ${${_NAME}_FIND_COMPONENTS})
-      if(${_NAME}_${comp}_FOUND)
-
-        if(NOT DEFINED FOUND_COMPONENTS_MSG)
-          set(FOUND_COMPONENTS_MSG "found components:")
-        endif()
-        string(APPEND FOUND_COMPONENTS_MSG " ${comp}")
-
-      else()
-
-        if(NOT DEFINED MISSING_COMPONENTS_MSG)
-          set(MISSING_COMPONENTS_MSG "missing components:")
-        endif()
-        string(APPEND MISSING_COMPONENTS_MSG " ${comp}")
-
-        if(${_NAME}_FIND_REQUIRED_${comp})
-          set(${_NAME}_FOUND FALSE)
-          string(APPEND MISSING_VARS " ${comp}")
-        endif()
-
-      endif()
-    endforeach()
-    set(COMPONENT_MSG "${FOUND_COMPONENTS_MSG} ${MISSING_COMPONENTS_MSG}")
-    string(APPEND DETAILS "[c${COMPONENT_MSG}]")
-  endif()
-
-  # version handling:
-  set(VERSION_MSG "")
-  set(VERSION_OK TRUE)
-
-  # check with DEFINED here as the requested or found version may be "0"
-  if (DEFINED ${_NAME}_FIND_VERSION)
-    if(DEFINED ${FPHSA_VERSION_VAR})
-      set(_FOUND_VERSION ${${FPHSA_VERSION_VAR}})
-
-      if(${_NAME}_FIND_VERSION_EXACT)       # exact version required
-        # count the dots in the version string
-        string(REGEX REPLACE "[^.]" "" _VERSION_DOTS "${_FOUND_VERSION}")
-        # add one dot because there is one dot more than there are components
-        string(LENGTH "${_VERSION_DOTS}." _VERSION_DOTS)
-        if (_VERSION_DOTS GREATER ${_NAME}_FIND_VERSION_COUNT)
-          # Because of the C++ implementation of find_package() ${_NAME}_FIND_VERSION_COUNT
-          # is at most 4 here. Therefore a simple lookup table is used.
-          if (${_NAME}_FIND_VERSION_COUNT EQUAL 1)
-            set(_VERSION_REGEX "[^.]*")
-          elseif (${_NAME}_FIND_VERSION_COUNT EQUAL 2)
-            set(_VERSION_REGEX "[^.]*\\.[^.]*")
-          elseif (${_NAME}_FIND_VERSION_COUNT EQUAL 3)
-            set(_VERSION_REGEX "[^.]*\\.[^.]*\\.[^.]*")
-          else ()
-            set(_VERSION_REGEX "[^.]*\\.[^.]*\\.[^.]*\\.[^.]*")
-          endif ()
-          string(REGEX REPLACE "^(${_VERSION_REGEX})\\..*" "\\1" _VERSION_HEAD "${_FOUND_VERSION}")
-          unset(_VERSION_REGEX)
-          if (NOT ${_NAME}_FIND_VERSION VERSION_EQUAL _VERSION_HEAD)
-            set(VERSION_MSG "Found unsuitable version \"${_FOUND_VERSION}\", but required is exact version \"${${_NAME}_FIND_VERSION}\"")
-            set(VERSION_OK FALSE)
-          else ()
-            set(VERSION_MSG "(found suitable exact version \"${_FOUND_VERSION}\")")
-          endif ()
-          unset(_VERSION_HEAD)
-        else ()
-          if (NOT ${_NAME}_FIND_VERSION VERSION_EQUAL _FOUND_VERSION)
-            set(VERSION_MSG "Found unsuitable version \"${_FOUND_VERSION}\", but required is exact version \"${${_NAME}_FIND_VERSION}\"")
-            set(VERSION_OK FALSE)
-          else ()
-            set(VERSION_MSG "(found suitable exact version \"${_FOUND_VERSION}\")")
-          endif ()
-        endif ()
-        unset(_VERSION_DOTS)
-
-      else()     # minimum version specified:
-        if (${_NAME}_FIND_VERSION VERSION_GREATER _FOUND_VERSION)
-          set(VERSION_MSG "Found unsuitable version \"${_FOUND_VERSION}\", but required is at least \"${${_NAME}_FIND_VERSION}\"")
-          set(VERSION_OK FALSE)
-        else ()
-          set(VERSION_MSG "(found suitable version \"${_FOUND_VERSION}\", minimum required is \"${${_NAME}_FIND_VERSION}\")")
-        endif ()
-      endif()
-
-    else()
-
-      # if the package was not found, but a version was given, add that to the output:
-      if(${_NAME}_FIND_VERSION_EXACT)
-         set(VERSION_MSG "(Required is exact version \"${${_NAME}_FIND_VERSION}\")")
-      else()
-         set(VERSION_MSG "(Required is at least version \"${${_NAME}_FIND_VERSION}\")")
-      endif()
-
-    endif()
-  else ()
-    # Check with DEFINED as the found version may be 0.
-    if(DEFINED ${FPHSA_VERSION_VAR})
-      set(VERSION_MSG "(found version \"${${FPHSA_VERSION_VAR}}\")")
-    endif()
-  endif ()
-
-  if(VERSION_OK)
-    string(APPEND DETAILS "[v${${FPHSA_VERSION_VAR}}(${${_NAME}_FIND_VERSION})]")
-  else()
-    set(${_NAME}_FOUND FALSE)
-  endif()
-
-
-  # print the result:
-  if (${_NAME}_FOUND)
-    FIND_PACKAGE_MESSAGE(${_NAME} "Found ${_NAME}: ${${_FIRST_REQUIRED_VAR}} ${VERSION_MSG} ${COMPONENT_MSG}" "${DETAILS}")
-  else ()
-
-    if(FPHSA_CONFIG_MODE)
-      _FPHSA_HANDLE_FAILURE_CONFIG_MODE()
-    else()
-      if(NOT VERSION_OK)
-        _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE}: ${VERSION_MSG} (found ${${_FIRST_REQUIRED_VAR}})")
-      else()
-        _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE} (missing:${MISSING_VARS}) ${VERSION_MSG}")
-      endif()
-    endif()
-
-  endif ()
-
-  set(${_NAME}_FOUND ${${_NAME}_FOUND} PARENT_SCOPE)
-  set(${_NAME_UPPER}_FOUND ${${_NAME}_FOUND} PARENT_SCOPE)
-endfunction()
--- a/CMakeModules/FindPackageMessage.cmake
+++ b/CMakeModules/FindPackageMessage.cmake
@@ -1,48 +0,0 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
-#[=======================================================================[.rst:
-FindPackageMessage
------------------
-
-.. code-block:: cmake
-
-  find_package_message(<name> "message for user" "find result details")
-
-This function is intended to be used in FindXXX.cmake modules files.
-It will print a message once for each unique find result.  This is
-useful for telling the user where a package was found.  The first
-argument specifies the name (XXX) of the package.  The second argument
-specifies the message to display.  The third argument lists details
-about the find result so that if they change the message will be
-displayed again.  The macro also obeys the QUIET argument to the
-find_package command.
-
-Example:
-
-.. code-block:: cmake
-
-  if(X11_FOUND)
-    find_package_message(X11 "Found X11: ${X11_X11_LIB}"
-      "[${X11_X11_LIB}][${X11_INCLUDE_DIR}]")
-  else()
-   ...
-  endif()
-#]=======================================================================]
-
-function(find_package_message pkg msg details)
-  # Avoid printing a message repeatedly for the same find result.
-  if(NOT ${pkg}_FIND_QUIETLY)
-    string(REPLACE "\n" "" details "${details}")
-    set(DETAILS_VAR FIND_PACKAGE_MESSAGE_DETAILS_${pkg})
-    if(NOT "${details}" STREQUAL "${${DETAILS_VAR}}")
-      # The message has not yet been printed.
-      message(STATUS "${msg}")
-
-      # Save the find details in the cache to avoid printing the same
-      # message again.
-      set("${DETAILS_VAR}" "${details}"
-        CACHE INTERNAL "Details about finding ${pkg}")
-    endif()
-  endif()
-endfunction()
--- a/Docker/build.sh
+++ b/Docker/build.sh
@@ -1,15 +1,14 @@
-rm ./sist2
-cp ../sist2 .
+rm ./sist2 sist2_debug
+cp ../sist2.gz .
+gzip -d sist2.gz
 strip sist2

 version=$(./sist2 --version)

 echo "Version ${version}"
-docker build . -t simon987/sist2:${version} -t simon987/sist2:latest \
-  -t docker.pkg.github.com/simon987/sist2/sist2:latest -t docker.pkg.github.com/simon987/sist2/sist2:${version}
+docker build . -t simon987/sist2:${version} -t simon987/sist2:latest
+
 docker push simon987/sist2:${version}
 docker push simon987/sist2:latest
-docker push docker.pkg.github.com/simon987/sist2/sist2:latest
-docker push docker.pkg.github.com/simon987/sist2/sist2:${version}

-docker run --rm -it simon987/sist2 -v
+docker run --rm simon987/sist2 -v
--- a/README.md
+++ b/README.md
@@ -8,25 +8,29 @@ sist2 (Simple incremental search tool)

 *Warning: sist2 is in early development*

-![sist2.png](sist2.png)
+![sist2.png](docs/sist2.png)

 ## Features

 * Fast, low memory usage, multi-threaded
 * Mobile-friendly Web interface
 * Portable (all its features are packaged in a single executable)
-* Extracts text from common file types \*
+* Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
-* Automatic tagging from file attributes via [user scripts](scripting/README.md)
+* Manual tagging from the UI and automatic tagging based on file attributes via [user scripts](docs/scripting.md)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
+* Stats page & disk utilisation visualization


 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)    

+![stats](docs/stats.png)
+
+
 ## Getting Started

 1. Have an Elasticsearch (>= 6.X.X) instance running
@@ -49,7 +53,7 @@ sist2 (Simple incremental search tool)
    1. *(or)* Download a [development snapshot](https://files.simon987.net/artifacts/Sist2/Build/) *(Not recommended!)*
    1. *(or)* `docker pull simon987/sist2:latest`

-1. See [Usage guide](USAGE.md)
+1. See [Usage guide](docs/USAGE.md)
   

 \* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)    
@@ -57,7 +61,7 @@ sist2 (Simple incremental search tool)

 ## Example usage

-See [Usage guide](USAGE.md) for more details
+See [Usage guide](docs/USAGE.md) for more details

 1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
 1. Push index to Elasticsearch: `sist2 index ./docs_idx`
@@ -68,14 +72,18 @@ See [Usage guide](USAGE.md) for more details

 File type | Library | Content | Thumbnail | Metadata
 :---|:---|:---|:---|:---
-pdf,xps,cbz,cbr,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
-`audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags |
-`video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist |
-`image/*` | ffmpeg | - | yes, `jpeg` | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+pdf,xps,fb2,epub | MuPDF | text+ocr | yes | title |
+cbz,cbr | *(none)* | - | yes | - |
+`audio/*` | ffmpeg | - | yes | ID3 tags |
+`video/*` | ffmpeg | - | yes | title, comment, artist |
+`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
 `text/plain` | *(none)* | yes | no | - |
+html, xml | *(none)* | yes | no | - |
 tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
-docx, xlsx, pptx | *(none)* | yes | no | no |
+docx, xlsx, pptx | *(none)* | yes | if embedded | creator, modified_by, title |
+mobi, azw, azw3 | libmobi | yes | no | author, title |

 \* *See [Archive files](#archive-files)*
 
@@ -85,18 +93,16 @@ they were directly in the file system. Recursive (archives inside archives)
 scan is also supported.

 **Limitations**:
-* Parsing media files with formats that require
-*seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) is not supported.
+* Support for parsing media files with formats that require *seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) 
+  is limitted (see `--mem-buffer` option)
 * Archive files are scanned sequentially, by a single thread. On systems where
 **sist2** is not I/O bound, scans might be faster when larger archives are split
 into smaller parts.
-
-To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
 
 
 ### OCR

-You can enable OCR support for pdf,xps,cbz,cbr,fb2,epub file types with the
+You can enable OCR support for pdf,xps,fb2,epub file types with the
 `--ocr <lang>` option. Download the language data files with your
 package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).

@@ -113,23 +119,17 @@ sist2 scan --ocr eng ~/Books/Textbooks/
 ## Build from source

 You can compile **sist2** by yourself if you don't want to use the pre-compiled
-binaries.
+binaries (GCC 7+ required).

 1. Install compile-time dependencies

-    *(Debian)*
-    ```bash
-    apt install git cmake pkg-config libglib2.0-dev \
-        libssl-dev uuid-dev python3 libmagic-dev libfreetype6-dev \
-        libcurl4-openssl-dev libbz2-dev yasm libharfbuzz-dev ragel \
-        libarchive-dev libtiff5 libpng16-16 libpango1.0-dev \
-        libxml2-dev libopenjp2-7-dev libleptonica-dev
+   ```bash
+   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic libraw
   ```

 2. Build
    ```bash
-    git clone --recurse-submodules https://github.com/simon987/sist2
-    ./scripts/get_static_libs.sh
-    cmake .
+    git clone --recursive https://github.com/simon987/sist2/
+    cmake -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
    make
    ```
--- a/1
+++ b/1
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -1,12 +1,17 @@
 #!/usr/bin/env bash

-./scripts/get_static_libs.sh
+VCPKG_ROOT="/vcpkg"

-rm -rf CMakeFiles CmakeCache.txt
-cmake -DSIST_DEBUG=off .
-make
+rm *.gz
+
+rm -rf CMakeFiles CMakeCache.txt
+cmake -DSIST_DEBUG=off -DVCPKG_BUILD_TYPE=release -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j 12
 strip sist2
+gzip -9 sist2

-rm -rf CMakeFiles CmakeCache.txt
-cmake -DSIST_DEBUG=on .
-make
+rm -rf CMakeFiles CMakeCache.txt
+cmake -DSIST_DEBUG=on -DVCPKG_BUILD_TYPE=debug -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j 12
+cp /usr/lib/x86_64-linux-gnu/libasan.so.2.0.0 libasan.so.2
+tar -czf sist2_debug.tar.gz sist2_debug libasan.so.2
--- a/docs/USAGE.md
+++ b/docs/USAGE.md
@@ -14,47 +14,57 @@
    * [examples](#web-examples)
    * [rewrite_url](#rewrite_url)
    * [link to specific indices](#link-to-specific-indices)
+* [exec-script](#exec-script)
+* [tagging](#tagging)

 ```
 Usage: sist2 scan [OPTION]... PATH
   or: sist2 index [OPTION]... INDEX
   or: sist2 web [OPTION]... INDEX...
+   or: sist2 exec-script [OPTION]... INDEX
 Lightning-fast file system indexer and search tool.

-    -h, --help            show this help message and exit
-    -v, --version         Show version and exit
-    --verbose             Turn on logging
-    --very-verbose        Turn on debug messages
+    -h, --help                    show this help message and exit
+    -v, --version                 Show version and exit
+    --verbose                     Turn on logging
+    --very-verbose                Turn on debug messages

 Scan options
-    -t, --threads=<int>   Number of threads. DEFAULT=1
-    -q, --quality=<flt>   Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5
-    --size=<int>          Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
-    --content-size=<int>  Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
-    --incremental=<str>   Reuse an existing index and only scan modified files.
-    -o, --output=<str>    Output directory. DEFAULT=index.sist2/
-    --rewrite-url=<str>   Serve files from this url instead of from disk.
-    --name=<str>          Index display name. DEFAULT: (name of the directory)
-    --depth=<int>         Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>       Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
-    --ocr=<str>           Tesseract language (use tesseract --list-langs to see which are installed on your machine)
-    -e, --exclude=<str>   Files that match this regex will not be scanned
-    --fast                Only index file names & mime type
+    -t, --threads=<int>           Number of threads. DEFAULT=1
+    -q, --quality=<flt>           Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5
+    --size=<int>                  Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
+    --content-size=<int>          Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
+    --incremental=<str>           Reuse an existing index and only scan modified files.
+    -o, --output=<str>            Output directory. DEFAULT=index.sist2/
+    --rewrite-url=<str>           Serve files from this url instead of from disk.
+    --name=<str>                  Index display name. DEFAULT: (name of the directory)
+    --depth=<int>                 Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
+    --archive=<str>               Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --ocr=<str>                   Tesseract language (use tesseract --list-langs to see which are installed on your machine)
+    -e, --exclude=<str>           Files that match this regex will not be scanned
+    --fast                        Only index file names & mime type
+    --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000

 Index options
-    --es-url=<str>        Elasticsearch url with port. DEFAULT=http://localhost:9200
-    -p, --print           Just print JSON documents to stdout.
-    --script-file=<str>   Path to user script.
-    --batch-size=<int>    Index batch size. DEFAULT: 100
-    -f, --force-reset     Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
+    -t, --threads=<int>           Number of threads. DEFAULT=1
+    --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
+    -p, --print                   Just print JSON documents to stdout.
+    --script-file=<str>           Path to user script.
+    --async-script                Execute user script asynchronously.
+    --batch-size=<int>            Index batch size. DEFAULT: 100
+    -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)

 Web options
-    --es-url=<str>        Elasticsearch url. DEFAULT=http://localhost:9200
-    --bind=<str>          Listen on this address. DEFAULT=localhost
-    --port=<str>          Listen on this port. DEFAULT=4090
-    --auth=<str>          Basic auth in user:password format
-Made by simon987 <me@simon987.net>. Released under GPL-3.0
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
+    --auth=<str>                  Basic auth in user:password format
+    --tag-auth=<str>              Basic auth in user:password format for tagging

+Exec-script options
+    --script-file=<str>           Path to user script.
+    --async-script                Execute user script asynchronously.
+Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```

 ## Scan
@@ -62,7 +72,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ### Scan options

 * `-t, --threads` 
-      Number of threads for file parsing. **Do not set a number higher than `$(nproc)`!**.
+      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-WmiObject Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
 * `-q, --quality` 
    Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. *Does not affect PDF thumbnails quality*
 * `--size` 
@@ -82,7 +92,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
    * list: Only get file names as text
    * shallow: Don't parse archives inside archives.
    * recurse: Scan archives recursively (default)
-* `--ocr` See [OCR](README.md#OCR)
+* `--ocr` See [OCR](../README.md#OCR)
 * `-e, --exclude` Regex pattern to exclude files. A file is excluded if the pattern matches any 
    part of the full absolute path.
    
@@ -94,6 +104,18 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
    * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
     `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
 * `--fast` Only index file names and mime type
+* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
+    will not be considered for the disk utilisation visualization; their size will be added to
+    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
+    and so on.
+    
+    In effect, smaller `treemap-threshold` values will yield a more detailed 
+    (but also a more cluttered and harder to read) visualization. 
+    
+* `--mem-buffer` Maximum memory buffer size in MB (per thread) for files inside archives. Media files 
+    larger than this number will be read sequentially and no *seek* operations will be supported.
+
+    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`

 ### Scan examples

@@ -122,7 +144,15 @@ documents.idx/
 ├── _index_139965425223424
 ├── _index_139965433616128
 ├── _index_139965442008832
-└── thumbs
+├── _index_139965442008832
+├── treemap.csv
+├── agg_mime.csv
+├── agg_date.csv
+├── add_size.csv
+├── thumbs
+|   ├── data.mdb
+|   └── lock.mdb
+└── tags
    ├── data.mdb
    └── lock.mdb
 ```
@@ -137,6 +167,8 @@ database containing the thumbnails.
 The `descriptor.json` file contains general information about the index. The 
 following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.

+The `.csv` are pre-computed aggregations necessary for the stats page.
+

 *Advanced usage*

@@ -189,7 +221,7 @@ my_index/
 }
 ```

-You can find the full list of supported fields [here](src/io/serialize.c#L90)
+You can find the full list of supported fields [here](../src/io/serialize.c#L90)

 The `_keyword.*` items will be indexed and searchable as **keyword** fields (only full matches allowed).
 The `_text.*` items will be indexed and searchable as **text** fields (fuzzy searching allowed)
@@ -212,7 +244,10 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
 * `-p, --print` 
    Print index in JSON format to stdout.
 * `--script-file` 
-    Path to user script. See [Scripting](scripting/README.md).
+    Path to user script. See [Scripting](scripting.md).
+ * `--async-script` 
+    Use `wait_for_completion=false` elasticsearch option while executing user script.
+     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
 * `--batch-size=<int>` 
    Index batch size. Indexing is generally faster with larger batches, but payloads that
    are too large will fail and additional overhead for retrying with smaller sizes may slow
@@ -244,14 +279,15 @@ sist2 index --print ./my_index/ | jq | less
 ### Web options
 * `--es-url=<str>` Elasticsearch url.
 * `--bind=<str>` Listen on this address.
- * `--port=<str>` Listen on this port.
 * `--auth=<str>` Basic auth in user:password format
+ * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
+    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
 
 ### Web examples

 **Single index**
 ```bash
-sist2 web --auth admin:hunter2 --bind 0.0.0.0 --port 8888 my_index
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index
 ```

 **Multiple indices**
@@ -273,3 +309,35 @@ Both the `root` and `rewrite_url` fields are safe to manually modify from the
 To link to specific indices, you can add a list of comma-separated index name to 
 the URL: `?i=<name>,<name>`. By default, indices with `"(nsfw)"` in their name are
 not displayed.
+
+## exec-script
+
+The `exec-script` command is used to execute a user script for an index that has already been imported to Elasticsearch with the `index` command. Note that the documents will not be reset to their default state before each execution as the `index` command does: if you make undesired changes to the documents by accident, you will need to run `index` again to revert to the original state.
+
+
+# Tagging
+
+### Manual tagging
+
+You can modify tags of individual documents directly from the 
+ `web` interface. Note that you can setup authentication for this feature
+ with the `--tag-auth` option (See [web options](#web-options))
+
+![manual_tag](manual_tag.png)
+
+Tags that are manually added are saved both in the 
+ index folder (in `/tags/`) and in Elasticsearch*. When re-`index`ing, 
+ they are read from the index and automatically applied.
+ 
+You can safely copy the `/tags/` database to another index.
+
+See [Automatic tagging](#automatic-tagging) for information about tag 
+ hierarchies and tag colors.
+
+\* *It can take a few seconds to take effect in new search queries, and the page needs 
+    to be reloaded for the tag tab to update*
+
+
+### Automatic tagging
+
+See [scripting](docs/scripting.md) documentation.
--- a/scripting/genre_example.png
+++ b/scripting/genre_example.png
--- a/docs/manual_tag.png
+++ b/docs/manual_tag.png
--- a/scripting/README.md
+++ b/scripting/README.md
@@ -39,7 +39,7 @@ it adds the `genre.<genre>` tag.
 ArrayList tags = ctx._source.tag = new ArrayList();

 if (ctx._source?.genre != null) {
-    tags.add("genre." + ctx._source.genre.toLowerCase())
+    tags.add("genre." + ctx._source.genre.toLowerCase());
 }
 ```

@@ -67,7 +67,7 @@ ArrayList tags = ctx._source.tag = new ArrayList();

 Matcher m = /[\(\.+](20[0-9]{2})[\)\.+]/.matcher(ctx._source.name);
 if (m.find()) {
-    tags.add("year." + m.group(1))
+    tags.add("year." + m.group(1));
 }
 ```

@@ -111,16 +111,6 @@ if (ctx._source.path != "") {
 }
 ```

-Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
-```Java
-ArrayList tags = ctx._source.tag = new ArrayList();
-
-if (ctx._source.path != "") {
-    String[] names = ctx._source.path.splitOnToken('/');
-    tags.add("studio." + names[names.length-1]);
-}
-```
-
 Parse `EXIF:F Number` tag
 ```Java
 if (ctx._source?.exif_fnumber != null) {
--- a/docs/sist2.png
+++ b/docs/sist2.png
--- a/docs/stats.png
+++ b/docs/stats.png
--- a/lib/bzip2-1.0.6
+++ b/lib/bzip2-1.0.6
--- a/lib/ffmpeg
+++ b/lib/ffmpeg
--- a/lib/harfbuzz
+++ b/lib/harfbuzz
--- a/lib/leptonica
+++ b/lib/leptonica
--- a/lib/libmagic
+++ b/lib/libmagic
--- a/lib/libpng
+++ b/lib/libpng
--- a/lib/libtiff
+++ b/lib/libtiff
--- a/lib/mupdf
+++ b/lib/mupdf
--- a/lib/onion
+++ b/lib/onion
--- a/lib/openjpeg
+++ b/lib/openjpeg
--- a/lib/tesseract
+++ b/lib/tesseract
--- a/1
+++ b/1
--- a/schema/mappings.json
+++ b/schema/mappings.json
@@ -10,12 +10,30 @@
    "path": {
      "type": "text",
      "analyzer": "path_analyzer",
+      "copy_to": "suggest-path",
      "fielddata": true,
-      "index_prefixes": {}
+      "fields": {
+        "nGram": {
+          "type": "text",
+          "analyzer": "my_nGram"
+        },
+        "text": {
+          "type": "text",
+          "analyzer": "content_analyzer"
+        }
+      }
+    },
+    "suggest-path": {
+      "type": "completion",
+      "analyzer": "case_insensitive_kw_analyzer"
    },
    "mime": {
      "type": "keyword"
    },
+    "thumbnail": {
+      "type": "keyword",
+      "index": false
+    },
    "videoc": {
      "type": "keyword",
      "index": false
@@ -108,7 +126,12 @@
      }
    },
    "tag": {
-      "type": "keyword"
+      "type": "keyword",
+      "copy_to": "suggest-tag"
+    },
+    "suggest-tag": {
+      "type": "completion",
+      "analyzer": "case_insensitive_kw_analyzer"
    },
    "exif_make": {
      "type": "text"
@@ -133,6 +156,12 @@
    },
    "exif_user_comment": {
      "type": "text"
+    },
+    "author": {
+      "type": "text"
+    },
+    "modified_by": {
+      "type": "text"
    }
  }
 }
--- a/schema/settings.json
+++ b/schema/settings.json
@@ -1,7 +1,8 @@
 {
  "index": {
    "refresh_interval": "30s",
-    "codec": "best_compression"
+    "codec": "best_compression",
+    "number_of_replicas": 0
  },
  "analysis": {
    "tokenizer": {
--- a/scripts/before_build.sh
+++ b/scripts/before_build.sh
@@ -2,15 +2,15 @@

 rm -rf index.sist2/

-rm web/js/bundle.js 2> /dev/null
-cat `ls web/js/*.min.js` > web/js/bundle.js
-cat web/js/{util,dom,search}.js >> web/js/bundle.js
+rm src/static/js/bundle.js 2> /dev/null
+cat `ls src/static/js/*.min.js` > src/static/js/bundle.js
+cat src/static/js/{util,dom}.js >> src/static/js/bundle.js

-rm web/css/bundle*.css 2> /dev/null
-cat web/css/*.min.css > web/css/bundle.css
-cat web/css/light.css >> web/css/bundle.css
-cat web/css/*.min.css > web/css/bundle_dark.css
-cat web/css/dark.css >> web/css/bundle_dark.css
+rm src/static/css/bundle*.css 2> /dev/null
+cat src/static/css/*.min.css > src/static/css/bundle.css
+cat src/static/css/light.css >> src/static/css/bundle.css
+cat src/static/css/*.min.css > src/static/css/bundle_dark.css
+cat src/static/css/dark.css >> src/static/css/bundle_dark.css

 python3 scripts/mime.py > src/parsing/mime_generated.c
 python3 scripts/serve_static.py > src/web/static_generated.c
--- a/scripts/get_static_libs.sh
+++ b/scripts/get_static_libs.sh
@@ -1,128 +0,0 @@
-#!/usr/bin/env bash
-
-THREADS=$(nproc)
-
-cd lib
-
-cd mupdf
-CFLAGS=-fPIC make USE_SYSTEM_HARFBUZZ=yes USE_SYSTEM_OPENJPEG=yes HAVE_X11=no HAVE_GLUT=no -j $THREADS
-cd ..
-
-mv mupdf/build/release/libmupdf.a .
-mv mupdf/build/release/libmupdf-third.a .
-
-# openjp2
-cd openjpeg
-cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-O3 -DNDEBUG -fPIC"
-make -j $THREADS
-cd ..
-mv openjpeg/bin/libopenjp2.a .
-
-# harfbuzz
-cd harfbuzz
-./autogen.sh
-CFLAGS=-fPIC ./configure --disable-shared --enable-static
-make -j $THREADS
-cd ..
-mv harfbuzz/src/.libs/libharfbuzz.a .
-
-# ffmpeg
-cd ffmpeg
-./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay \
- --disable-ffprobe --disable-doc\
- --disable-manpages --disable-postproc --disable-avfilter \
- --disable-alsa --disable-lzma --disable-xlib --disable-debug\
- --disable-vdpau --disable-vaapi --disable-sdl2 --disable-network\
- --extra-cflags=-fPIC
-make -j $THREADS
-cd ..
-
-mv ffmpeg/libavcodec/libavcodec.a .
-mv ffmpeg/libavformat/libavformat.a .
-mv ffmpeg/libavutil/libavutil.a .
-mv ffmpeg/libswresample/libswresample.a .
-mv ffmpeg/libswscale/libswscale.a .
-
-# onion
-cd onion
-mkdir build 2> /dev/null
-cd build
-cmake -DONION_USE_SSL=false -DONION_USE_PAM=false -DONION_USE_PNG=false -DONION_USE_JPEG=false \
-DONION_USE_JPEG=false -DONION_USE_XML2=false -DONION_USE_SYSTEMD=false -DONION_USE_SQLITE3=false \
-DONION_USE_REDIS=false -DONION_USE_GC=false -DONION_USE_TESTS=false -DONION_EXAMPLES=false \
-DONION_USE_BINDINGS_CPP=false ..
-make -j $THREADS
-cd ../..
-
-mv onion/build/src/onion/libonion_static.a .
-
-#bzip2
-cd bzip2-1.0.6
-make -j $THREADS
-cd ..
-mv bzip2-1.0.6/libbz2.a .
-
-# magic
-cd libmagic
-./autogen.sh
-./configure --enable-static --disable-shared
-make -j $THREADS
-cd ..
-mv libmagic/src/.libs/libmagic.a .
-
-# tesseract
-cd tesseract
-mkdir build
-cd build
-cmake -DSTATIC=on -DBUILD_TRAINING_TOOLS=off -DBUILD_TESTS=off -DCMAKE_BUILD_TYPE=Release \
-  -DCMAKE_CXX_FLAGS="-fPIC" -DAUTO_OPTIMIZE=off ..
-make -j $THREADS
-cd ../..
-mv tesseract/build/libtesseract.a .
-
-# leptonica
-cd leptonica
-./autogen.sh
-CFLAGS="-fPIC" ./configure --without-zlib --without-jpeg --without-giflib \
-  --without-giflib --without-libwebp --without-libwebpmux --without-libopenjpeg \
-  --enable-static --disable-shared
-make -j $THREADS
-cd ..
-mv leptonica/src/.libs/liblept.a .
-
-# tiff
-cd libtiff
-./autogen.sh
-CFLAGS="-fPIC" CXXFLAGS="-fPIC" CXX_FLAGS="-fPIC" ./configure --enable-static --disable-shared --disable-lzw --disable-jpeg --disable-webp \
-  --disable-lzma --disable-zstd --disable-jbig
-make -j $THREADS
-cd ..
-mv libtiff/libtiff/.libs/libtiff.a .
-
-# png
-cd libpng
-CFLAGS="-fPIC" ./configure --enable-static --disable-shared
-make -j $THREADS
-cd ..
-mv libpng/.libs/libpng16.a .
-
-# openssl...
-git clone --depth 1 -b OpenSSL_1_1_0-stable https://github.com/openssl/openssl
-cd openssl
-./config --prefix=$(pwd)/../ssl
-make depend
-make -j $THREADS
-make install
-cd ..
-mv ./openssl/libcrypto.a ./openssl/libssl.a .
-
-# curl
-wget -nc https://curl.haxx.se/download/curl-7.68.0.tar.gz
-tar -xzf curl-7.68.0.tar.gz
-cd curl-7.68.0
-./configure --disable-ldap --disable-ldaps --without-librtmp --disable-rtsp --disable-crypto-auth \
-  --disable-smtp --without-libidn2 --without-nghttp2 --without-brotli --enable-static --disable-shared \
-  --without-libpsl --with-ssl=$(pwd)/../ssl
-make -j $THREADS
-cd ..
-mv curl-7.68.0/lib/.libs/libcurl.a .
--- a/scripts/index_static.py
+++ b/scripts/index_static.py
@@ -14,4 +14,5 @@ def clean(filepath):
 for file in files:
    with open(file, "r") as f:
        data = json.dumps(json.load(f), separators=(",", ":")).encode()
+    data += b'\0'
    print("char %s[%d] = {%s};" % (clean(file), len(data), ",".join(str(int(b)) for b in data)))
--- a/scripts/mime.csv
+++ b/scripts/mime.csv
@@ -13,7 +13,7 @@ application/epub+zip, epub
 application/freeloader, frl
 application/futuresplash, spl
 application/groupwise, vew
-application/gzip, gz
+application/gzip, gz|tgz
 application/hta, hta
 application/i-deas, unv
 application/iges, iges|igs
@@ -111,7 +111,7 @@ application/x-dbf, dbf
 application/x-dbt,
 application/x-debian-package, deb
 application/x-deepv, deepv
-application/x-director, dcr|dir|dxr
+application/x-director, dir|dxr
 application/x-dmp, dmp
 application/x-dosdriver,
 application/x-dosexec, dll
@@ -175,6 +175,7 @@ application/x-mif, mif
 application/x-mix-transfer, nix
 application/xml, opf
 application/x-mobipocket-ebook, mobi
+application/vnd.amazon.mobi8-ebook, azw|azw3
 application/x-msaccess, accdb
 application/x-ms-compress-szdd, fon
 application/x-ms-pdb, pdb
@@ -346,7 +347,8 @@ text/javascript, js
 text/mcf, mcf
 text/pascal, pas
 text/PGP,
-text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml
+text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
 text/scriplet, wsc
@@ -427,3 +429,23 @@ video/x-msvideo, divx
 video/x-qtc, qtc
 video/x-sgi-movie, movie|mv
 x-epoc/x-sisx-app,
+application/x-zstd-dictionary,
+application/vnd.ms-outlook, msg
+image/x-olympus-orf, orf
+image/x-nikon-nef, nef
+image/x-fuji-raf, raf
+image/x-panasonic-raw, rw2|raw
+image/x-adobe-dng, dng
+image/x-canon-cr2, cr2
+image/x-canon-crw, crw
+image/x-dcraw,
+image/x-kodak-dcr, dcr
+image/x-kodak-k25, k25
+image/x-kodak-kdc, kdc
+image/x-minolta-mrw, mrw
+image/x-pentax-pef, pef
+image/x-sigma-x3f, xf3
+image/x-sony-arw, arw
+image/x-sony-sr2, sr2
+image/x-sony-srf, srf
+image/x-epson-erf, erf
--- a/scripts/mime.py
+++ b/scripts/mime.py
@@ -18,7 +18,6 @@ major_mime = {

 pdf = (
    "application/pdf",
-    "application/x-cbz",
    "application/epub+zip",
    "application/vnd.ms-xpsdocument",
 )
@@ -62,6 +61,40 @@ doc = (
    "application/vnd.openxmlformats-officedocument.presentationml.presentation"
 )

+mobi = (
+    "application/x-mobipocket-ebook",
+    "application/vnd.amazon.mobi8-ebook"
+)
+
+markup = (
+    "text/xml",
+    "text/html",
+    "text/x-sgml"
+)
+
+raw = (
+    "image/x-olympus-orf",
+    "image/x-nikon-nef",
+    "image/x-fuji-raf",
+    "image/x-panasonic-raw",
+    "image/x-adobe-dng",
+    "image/x-canon-cr2",
+    "image/x-canon-crw",
+    "image/x-dcraw",
+    "image/x-kodak-dcr",
+    "image/x-kodak-k25",
+    "image/x-kodak-kdc",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-sigma-x3f",
+    "image/x-sony-arw",
+    "image/x-sony-sr2",
+    "image/x-sony-srf",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-epson-erf",
+)
+
 cnt = 1


@@ -82,6 +115,12 @@ def mime_id(mime):
        mime_id += " | 0x08000000"
    elif mime in doc:
        mime_id += " | 0x04000000"
+    elif mime in mobi:
+        mime_id += " | 0x02000000"
+    elif mime in markup:
+        mime_id += " | 0x01000000"
+    elif mime in raw:
+        mime_id += " | 0x00800000"
    elif mime == "application/x-empty":
        return "1"
    return mime_id
@@ -91,7 +130,7 @@ def clean(t):
    return t.replace("/", "_").replace(".", "_").replace("+", "_").replace("-", "_")


-with open("mime.csv") as f:
+with open("scripts/mime.csv") as f:
    for l in f:
        mime, ext_list = l.split(",")
        if l.startswith("!"):
@@ -103,7 +142,7 @@ with open("mime.csv") as f:
    print("// **Generated by mime.py**")
    print("#ifndef MIME_GENERATED_C")
    print("#define MIME_GENERATED_C")
-    print("#include <glib-2.0/glib.h>\n")
+    print("#include <glib.h>\n")
    print("#include <stdlib.h>\n")
    # Enum
    print("enum mime {")
--- a/scripts/serve_static.py
+++ b/scripts/serve_static.py
@@ -1,10 +1,12 @@
 files = [
-    "web/css/bundle.css",
-    "web/css/bundle_dark.css",
-    "web/js/bundle.js",
-    "web/img/sprite-skin-flat.png",
-    "web/img/sprite-skin-flat-dark.png",
-    "web/search.html",
+    "src/static/css/bundle.css",
+    "src/static/css/bundle_dark.css",
+    "src/static/js/bundle.js",
+    "src/static/js/search.js",
+    "src/static/img/sprite-skin-flat.png",
+    "src/static/img/sprite-skin-flat-dark.png",
+    "src/static/search.html",
+    "src/static/stats.html",
 ]


--- a/src/cli.c
+++ b/src/cli.c
@@ -1,5 +1,6 @@
 #include "cli.h"
 #include "ctx.h"
+#include <tesseract/capi.h>

 #define DEFAULT_OUTPUT "index.sist2/"
 #define DEFAULT_CONTENT_SIZE 32768
@@ -10,10 +11,12 @@
 #define DEFAULT_ES_URL "http://localhost:9200"
 #define DEFAULT_BATCH_SIZE 100

-#define DEFAULT_BIND_ADDR "localhost"
-#define DEFAULT_PORT "4090"
+#define DEFAULT_LISTEN_ADDRESS "localhost:4090"
+#define DEFAULT_TREEMAP_THRESHOLD 0.0005

-const char* TESS_DATAPATHS[] = {
+#define DEFAULT_MAX_MEM_BUFFER 2000
+
+const char *TESS_DATAPATHS[] = {
        "/usr/share/tessdata/",
        "/usr/share/tesseract-ocr/tessdata/",
        "./",
@@ -29,10 +32,18 @@ scan_args_t *scan_args_create() {
    return args;
 }

+exec_args_t *exec_args_create() {
+    exec_args_t *args = calloc(sizeof(exec_args_t), 1);
+    return args;
+}
+
 void scan_args_destroy(scan_args_t *args) {
    if (args->name != NULL) {
        free(args->name);
    }
+    if (args->incremental != NULL) {
+        free(args->incremental);
+    }
    if (args->path != NULL) {
        free(args->path);
    }
@@ -52,6 +63,10 @@ void web_args_destroy(web_args_t *args) {
    free(args);
 }

+void exec_args_destroy(exec_args_t *args) {
+    free(args);
+}
+
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
    if (argc < 2) {
        fprintf(stderr, "Required positional argument: PATH.\n");
@@ -67,7 +82,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
    }

    if (args->incremental != NULL) {
-        abs_path = abspath(args->incremental);
+        args->incremental = abspath(args->incremental);
        if (abs_path == NULL) {
            sist_log("main.c", SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
            args->incremental = NULL;
@@ -112,7 +127,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
        return 1;
    }

-    if (args->depth < 0) {
+    if (args->depth <= 0) {
        args->depth = G_MAXINT32;
    } else {
        args->depth += 1;
@@ -120,6 +135,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {

    if (args->name == NULL) {
        args->name = g_path_get_basename(args->output);
+    } else {
+        char* tmp = malloc(strlen(args->name) + 1);
+        strcpy(tmp, args->name);
+        args->name = tmp;
    }

    if (args->rewrite_url == NULL) {
@@ -144,7 +163,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {

        char filename[128];
        sprintf(filename, "%s.traineddata", args->tesseract_lang);
-        const char * path = find_file_in_paths(TESS_DATAPATHS, filename);
+        const char *path = find_file_in_paths(TESS_DATAPATHS, filename);
        if (path == NULL) {
            LOG_FATAL("cli.c", "Could not find tesseract language file!");
        }
@@ -180,6 +199,16 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
        ScanCtx.exclude = NULL;
    }

+    if (args->treemap_threshold_str == 0) {
+        args->treemap_threshold = DEFAULT_TREEMAP_THRESHOLD;
+    } else {
+        args->treemap_threshold = atof(args->treemap_threshold_str);
+    }
+
+    if (args->max_memory_buffer == 0) {
+        args->max_memory_buffer = DEFAULT_MAX_MEM_BUFFER;
+    }
+
    LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
    LOG_DEBUGF("cli.c", "arg size=%d", args->size)
    LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
@@ -195,6 +224,36 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
    LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
    LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)
    LOG_DEBUGF("cli.c", "arg fast=%d", args->fast)
+    LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold)
+    LOG_DEBUGF("cli.c", "arg max_memory_buffer=%d", args->max_memory_buffer)
+
+    return 0;
+}
+
+int load_script(const char *script_path, char **dst) {
+    struct stat info;
+    int res = stat(script_path, &info);
+
+    if (res == -1) {
+        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        return 1;
+    }
+
+    int fd = open(script_path, O_RDONLY);
+    if (fd == -1) {
+        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        return 1;
+    }
+
+    *dst = malloc(info.st_size + 1);
+    res = read(fd, *dst, info.st_size);
+    if (res < 0) {
+        fprintf(stderr, "Error reading script file '%s': %s\n", script_path, strerror(errno));
+        return 1;
+    }
+
+    *(*dst + info.st_size) = '\0';
+    close(fd);

    return 0;
 }
@@ -208,6 +267,13 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
        return 1;
    }

+    if (args->threads == 0) {
+        args->threads = 1;
+    } else if (args->threads < 0) {
+        fprintf(stderr, "Invalid threads: %d\n", args->threads);
+        return 1;
+    }
+
    char *index_path = abspath(argv[1]);
    if (index_path == NULL) {
        fprintf(stderr, "File not found: %s\n", argv[1]);
@@ -222,29 +288,9 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
    }

    if (args->script_path != NULL) {
-        struct stat info;
-        int res = stat(args->script_path, &info);
-
-        if (res == -1) {
-            fprintf(stderr, "Error opening script file '%s': %s\n", args->script_path, strerror(errno));
+        if (load_script(args->script_path, &args->script) != 0) {
            return 1;
        }
-
-        int fd = open(args->script_path, O_RDONLY);
-        if (fd == -1) {
-            fprintf(stderr, "Error opening script file '%s': %s\n", args->script_path, strerror(errno));
-            return 1;
-        }
-
-        args->script = malloc(info.st_size + 1);
-        res = read(fd, args->script, info.st_size);
-        if (res < 0) {
-            fprintf(stderr, "Error reading script file '%s': %s\n", args->script_path, strerror(errno));
-            return 1;
-        }
-
-        *(args->script + info.st_size) = '\0';
-        close(fd);
    }

    if (args->batch_size == 0) {
@@ -254,6 +300,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg async_script=%s", args->async_script)
    LOG_DEBUGF("cli.c", "arg script=%s", args->script)
    LOG_DEBUGF("cli.c", "arg print=%d", args->print)
    LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
@@ -275,18 +322,53 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
        args->es_url = DEFAULT_ES_URL;
    }

-    if (args->bind == NULL) {
-        args->bind = DEFAULT_BIND_ADDR;
-    }
-
-    if (args->port == NULL) {
-        args->port = DEFAULT_PORT;
+    if (args->listen_address == NULL) {
+        args->listen_address = DEFAULT_LISTEN_ADDRESS;
    }

    if (args->credentials != NULL) {
-        args->b64credentials = onion_base64_encode(args->credentials, (int) strlen(args->credentials));
-        //Remove trailing newline
-        *(args->b64credentials + strlen(args->b64credentials) - 1) = '\0';
+        char *ptr = strstr(args->credentials, ":");
+        if (ptr == NULL) {
+            fprintf(stderr, "Invalid --auth format, see usage\n");
+            return 1;
+        }
+
+        strncpy(args->auth_user, args->credentials, (ptr - args->credentials));
+        strcpy(args->auth_pass, ptr + 1);
+
+        if (strlen(args->auth_user) == 0) {
+            fprintf(stderr, "--auth username must be at least one character long");
+            return 1;
+        }
+
+        args->auth_enabled = TRUE;
+    } else {
+        args->auth_enabled = FALSE;
+    }
+
+    if (args->tag_credentials != NULL && args->credentials != NULL) {
+        fprintf(stderr, "--auth and --tag-auth are mutually exclusive");
+        return 1;
+    }
+
+    if (args->tag_credentials != NULL) {
+        char *ptr = strstr(args->tag_credentials, ":");
+        if (ptr == NULL) {
+            fprintf(stderr, "Invalid --tag-auth format, see usage\n");
+            return 1;
+        }
+
+        strncpy(args->auth_user, args->tag_credentials, (ptr - args->tag_credentials));
+        strcpy(args->auth_pass, ptr + 1);
+
+        if (strlen(args->auth_user) == 0) {
+            fprintf(stderr, "--tag-auth username must be at least one character long");
+            return 1;
+        }
+
+        args->tag_auth_enabled = TRUE;
+    } else {
+        args->tag_auth_enabled = FALSE;
    }

    args->index_count = argc - 1;
@@ -301,10 +383,11 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
    }

    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
-    LOG_DEBUGF("cli.c", "arg bind=%s", args->bind)
-    LOG_DEBUGF("cli.c", "arg port=%s", args->port)
+    LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)
    LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
-    LOG_DEBUGF("cli.c", "arg b64credentials=%s", args->b64credentials)
+    LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials)
+    LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user)
+    LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass)
    LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count)
    for (int i = 0; i < args->index_count; i++) {
        LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i])
@@ -323,3 +406,35 @@ web_args_t *web_args_create() {
    return args;
 }

+int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
+
+    if (argc < 2) {
+        fprintf(stderr, "Required positional argument: PATH.\n");
+        return 1;
+    }
+
+    char *index_path = abspath(argv[1]);
+    if (index_path == NULL) {
+        fprintf(stderr, "File not found: %s\n", argv[1]);
+        return 1;
+    } else {
+        args->index_path = argv[1];
+        free(index_path);
+    }
+
+    if (args->es_url == NULL) {
+        args->es_url = DEFAULT_ES_URL;
+    }
+
+    if (args->script_path == NULL) {
+        LOG_FATAL("cli.c", "--script-file argument is required");
+    }
+
+    if (load_script(args->script_path, &args->script) != 0) {
+        return 1;
+    }
+
+    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg script=%s", args->script)
+    return 0;
+}
--- a/src/cli.h
+++ b/src/cli.h
@@ -3,6 +3,8 @@

 #include "sist.h"

+#include "libscan/arc/arc.h"
+
 typedef struct scan_args {
    float quality;
    int size;
@@ -20,6 +22,9 @@ typedef struct scan_args {
    const char *tesseract_path;
    char *exclude_regex;
    int fast;
+    const char* treemap_threshold_str;
+    double treemap_threshold;
+    int max_memory_buffer;
 } scan_args_t;

 scan_args_t *scan_args_create();
@@ -35,19 +40,32 @@ typedef struct index_args {
    char *script;
    int print;
    int batch_size;
+    int async_script;
    int force_reset;
+    int threads;
 } index_args_t;

 typedef struct web_args {
    char *es_url;
-    char *bind;
-    char *port;
+    char *listen_address;
    char *credentials;
-    char *b64credentials;
+    char *tag_credentials;
+    char auth_user[256];
+    char auth_pass[256];
+    int auth_enabled;
+    int tag_auth_enabled;
    int index_count;
    const char **indices;
 } web_args_t;

+typedef struct exec_args {
+    char *es_url;
+    const char *index_path;
+    const char *script_path;
+    int async_script;
+    char *script;
+} exec_args_t;
+
 index_args_t *index_args_create();

 void index_args_destroy(index_args_t *args);
@@ -60,4 +78,10 @@ int index_args_validate(index_args_t *args, int argc, const char **argv);

 int web_args_validate(web_args_t *args, int argc, const char **argv);

+exec_args_t *exec_args_create();
+
+void exec_args_destroy(exec_args_t *args);
+
+int exec_args_validate(exec_args_t *args, int argc, const char **argv);
+
 #endif
--- a/src/ctx.c
+++ b/src/ctx.c
@@ -0,0 +1,6 @@
+#include "ctx.h"
+
+ScanCtx_t ScanCtx;
+WebCtx_t WebCtx;
+IndexCtx_t IndexCtx;
+LogCtx_t LogCtx;
--- a/src/ctx.h
+++ b/src/ctx.h
@@ -2,8 +2,23 @@
 #define SIST2_CTX_H

 #include "sist.h"
+#include "tpool.h"
+#include "libscan/scan.h"
+#include "libscan/arc/arc.h"
+#include "libscan/comic/comic.h"
+#include "libscan/ebook/ebook.h"
+#include "libscan/font/font.h"
+#include "libscan/media/media.h"
+#include "libscan/ooxml/ooxml.h"
+#include "libscan/text/text.h"
+#include "libscan/mobi/scan_mobi.h"
+#include "libscan/raw/raw.h"
+#include "src/io/store.h"

-struct {
+#include <glib.h>
+#include <pcre.h>
+
+typedef struct {
    struct index_t index;

    GHashTable *mime_table;
@@ -11,14 +26,8 @@ struct {

    tpool_t *pool;

-    int tn_size;
    int threads;
-    int content_size;
-    float tn_qscale;
    int depth;
-    archive_mode_t archive_mode;
-    int verbose;
-    int very_verbose;

    size_t stat_tn_size;
    size_t stat_index_size;
@@ -26,31 +35,49 @@ struct {
    GHashTable *original_table;
    GHashTable *copy_table;

-    pthread_mutex_t mupdf_mu;
-    char * tesseract_lang;
-    const char * tesseract_path;
    pcre *exclude;
    pcre_extra *exclude_extra;
    int fast;
-} ScanCtx;

-struct {
+    scan_arc_ctx_t arc_ctx;
+    scan_comic_ctx_t comic_ctx;
+    scan_ebook_ctx_t ebook_ctx;
+    scan_font_ctx_t font_ctx;
+    scan_media_ctx_t media_ctx;
+    scan_ooxml_ctx_t ooxml_ctx;
+    scan_text_ctx_t text_ctx;
+    scan_mobi_ctx_t mobi_ctx;
+    scan_raw_ctx_t raw_ctx;
+} ScanCtx_t;
+
+typedef struct {
    int verbose;
    int very_verbose;
    int no_color;
-} LogCtx;
+} LogCtx_t;

-struct {
+typedef struct {
    char *es_url;
    int batch_size;
-} IndexCtx;
+    tpool_t *pool;
+    store_t *tag_store;
+    GHashTable *tags;
+} IndexCtx_t;

-struct {
+typedef struct {
    char *es_url;
    int index_count;
-    char *b64credentials;
-    struct index_t indices[16];
-} WebCtx;
+    char *auth_user;
+    char *auth_pass;
+    int auth_enabled;
+    int tag_auth_enabled;
+    struct index_t indices[64];
+} WebCtx_t;
+
+extern ScanCtx_t ScanCtx;
+extern WebCtx_t WebCtx;
+extern IndexCtx_t IndexCtx;
+extern LogCtx_t LogCtx;


 #endif
--- a/src/index/elastic.c
+++ b/src/index/elastic.c
@@ -1,6 +1,8 @@
 #include "elastic.h"
 #include "src/ctx.h"

+#include "web.h"
+
 #include "static_generated.c"


@@ -12,9 +14,18 @@ typedef struct es_indexer {
 } es_indexer_t;


-static es_indexer_t *Indexer;
+static __thread es_indexer_t *Indexer;

 void delete_queue(int max);
+void elastic_flush();
+
+void elastic_cleanup() {
+    elastic_flush();
+    if (Indexer != NULL) {
+        free(Indexer->es_url);
+        free(Indexer);
+    }
+}

 void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {

@@ -33,8 +44,12 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
    cJSON_Delete(line);
 }

-void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
+void index_json_func(void *arg) {
+    es_bulk_line_t *line = arg;
+    elastic_index_line(line);
+}

+void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
    char *json = cJSON_PrintUnformatted(document);

    size_t json_len = strlen(json);
@@ -46,10 +61,14 @@ void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
    bulk_line->next = NULL;

    cJSON_free(json);
-    elastic_index_line(bulk_line);
+    tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }

-void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]) {
+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]) {
+
+    if (Indexer == NULL) {
+        Indexer = create_indexer(IndexCtx.es_url);
+    }

    cJSON *body = cJSON_CreateObject();
    cJSON *script_obj = cJSON_AddObjectToObject(body, "script");
@@ -63,9 +82,15 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
    char *str = cJSON_Print(body);

    char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?pretty", Indexer->es_url);
-    response_t *r = web_post(bulk_url, str, "Content-Type: application/json");
-    LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    if (async) {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/sist2/_update_by_query?wait_for_completion=false", Indexer->es_url);
+    } else {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/sist2/_update_by_query", Indexer->es_url);
+    }
+    response_t *r = web_post(bulk_url, str);
+    if (!async) {
+        LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    }
    cJSON *resp = cJSON_Parse(r->body);

    cJSON_free(str);
@@ -80,44 +105,64 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
        cJSON_free(error_str);
    }

+    if (async) {
+        cJSON *task = cJSON_GetObjectItem(resp, "task");
+        LOG_INFOF("elastic.c", "User script queued: %s/_tasks/%s", Indexer->es_url, task->valuestring);
+    }
+
    cJSON_Delete(resp);
 }

+#define ACTION_STR_LEN 91
+
 void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
    es_bulk_line_t *line = Indexer->line_head;
    *count = 0;

    size_t buf_size = 0;
    size_t buf_cur = 0;
-    char *buf = malloc(1);
+    char *buf = malloc(8192);
+    size_t buf_capacity = 8192;

    while (line != NULL && *count < max) {
-        char action_str[512];
-        snprintf(action_str, 512,
+        char action_str[256];
+        snprintf(action_str, 256,
                 "{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str);
-        size_t action_str_len = strlen(action_str);

        size_t line_len = strlen(line->line);
-        buf = realloc(buf, buf_size + line_len + action_str_len);
-        buf_size += line_len + action_str_len;

-        memcpy(buf + buf_cur, action_str, action_str_len);
-        buf_cur += action_str_len;
+        while (buf_size + line_len + ACTION_STR_LEN > buf_capacity) {
+            buf_capacity *= 2;
+            buf = realloc(buf, buf_capacity);
+        }
+
+        buf_size += line_len + ACTION_STR_LEN;
+
+        memcpy(buf + buf_cur, action_str, ACTION_STR_LEN);
+        buf_cur += ACTION_STR_LEN;
        memcpy(buf + buf_cur, line->line, line_len);
        buf_cur += line_len;

        line = line->next;
        (*count)++;
    }
-    buf = realloc(buf, buf_size + 1);
+
+    if (buf_size + 1 > buf_capacity) {
+        buf = realloc(buf, buf_capacity + 1);
+    }
+
    *(buf + buf_cur) = '\0';

    *buf_len = buf_cur;
    return buf;
 }

-void *print_errors(response_t *r) {
-    cJSON *ret_json = cJSON_Parse(r->body);
+void print_errors(response_t *r) {
+    char *tmp = malloc(r->size + 1);
+    memcpy(tmp, r->body, r->size);
+    *(tmp + r->size) = '\0';
+
+    cJSON *ret_json = cJSON_Parse(tmp);
    if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
        cJSON *err;
        cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
@@ -129,16 +174,23 @@ void *print_errors(response_t *r) {
        }
    }
    cJSON_Delete(ret_json);
+    free(tmp);
 }

 void _elastic_flush(int max) {
+
+    if (max == 0) {
+        LOG_WARNING("elastic.c", "calling _elastic_flush with 0 in queue")
+        return;
+    }
+
    size_t buf_len;
    int count;
    void *buf = create_bulk_buffer(max, &count, &buf_len);

    char bulk_url[4096];
    snprintf(bulk_url, 4096, "%s/sist2/_bulk?pipeline=tie", Indexer->es_url);
-    response_t *r = web_post(bulk_url, buf, "Content-Type: application/x-ndjson");
+    response_t *r = web_post(bulk_url, buf);

    if (r->status_code == 0) {
        LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url)
@@ -164,6 +216,15 @@ void _elastic_flush(int max) {
        _elastic_flush(max / 2);
        return;

+    } else if (r->status_code == 429) {
+
+        free_response(r);
+        free(buf);
+        LOG_WARNING("elastic.c", "Got 429 status, will retry after delay")
+        usleep(1000000 * 20);
+        _elastic_flush(max);
+        return;
+
    } else if (r->status_code != 200) {
        print_errors(r);
        delete_queue(Indexer->queued);
@@ -189,9 +250,8 @@ void delete_queue(int max) {
        Indexer->line_head = tmp->next;
        if (Indexer->line_head == NULL) {
            Indexer->line_tail = NULL;
-        } else {
-            free(tmp);
        }
+        free(tmp);
        Indexer->queued -= 1;
    }
 }
@@ -241,33 +301,34 @@ es_indexer_t *create_indexer(const char *url) {
    return indexer;
 }

-void destroy_indexer(char *script, char index_id[UUID_STR_LEN]) {
+void finish_indexer(char *script, int async_script, char *index_id) {

    char url[4096];

    snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
-    response_t *r = web_post(url, "", NULL);
+    response_t *r = web_post(url, "");
    LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
    free_response(r);

    if (script != NULL) {
-        execute_update_script(script, index_id);
+        execute_update_script(script, async_script, index_id);
+        free(script);
+
+        snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
+        r = web_post(url, "");
+        LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
+        free_response(r);
    }

-    snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
-    r = web_post(url, "", NULL);
-    LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
-    free_response(r);
-
    snprintf(url, sizeof(url), "%s/sist2/_forcemerge", IndexCtx.es_url);
-    r = web_post(url, "", NULL);
+    r = web_post(url, "");
    LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code);
    free_response(r);

-    if (Indexer != NULL) {
-        free(Indexer->es_url);
-        free(Indexer);
-    }
+    snprintf(url, sizeof(url), "%s/sist2/_settings", IndexCtx.es_url);
+    r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}");
+    LOG_INFOF("elastic.c", "Set refresh interval <%d>", r->status_code);
+    free_response(r);
 }

 void elastic_init(int force_reset) {
@@ -275,7 +336,7 @@ void elastic_init(int force_reset) {
    // Check if index exists
    char url[4096];
    snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
    int index_exists = r->status_code == 200;
    free_response(r);

@@ -285,32 +346,32 @@ void elastic_init(int force_reset) {
        free_response(r);

        snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
-        r = web_put(url, "", NULL);
+        r = web_put(url, "");
        LOG_INFOF("elastic.c", "Create index <%d>", r->status_code);
        free_response(r);

        snprintf(url, 4096, "%s/sist2/_close", IndexCtx.es_url);
-        r = web_post(url, "", NULL);
+        r = web_post(url, "");
        LOG_INFOF("elastic.c", "Close index <%d>", r->status_code);
        free_response(r);

        snprintf(url, 4096, "%s/_ingest/pipeline/tie", IndexCtx.es_url);
-        r = web_put(url, pipeline_json, "Content-Type: application/json");
+        r = web_put(url, pipeline_json);
        LOG_INFOF("elastic.c", "Create pipeline <%d>", r->status_code);
        free_response(r);

        snprintf(url, 4096, "%s/sist2/_settings", IndexCtx.es_url);
-        r = web_put(url, settings_json, "Content-Type: application/json");
+        r = web_put(url, settings_json);
        LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code);
        free_response(r);

        snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url);
-        r = web_put(url, mappings_json, "Content-Type: application/json");
+        r = web_put(url, mappings_json);
        LOG_INFOF("elastic.c", "Update mappings <%d>", r->status_code);
        free_response(r);

        snprintf(url, 4096, "%s/sist2/_open", IndexCtx.es_url);
-        r = web_post(url, "", NULL);
+        r = web_post(url, "");
        LOG_INFOF("elastic.c", "Open index <%d>", r->status_code);
        free_response(r);
    }
@@ -320,7 +381,7 @@ cJSON *elastic_get_document(const char *uuid_str) {
    char url[4096];
    snprintf(url, 4096, "%s/sist2/_doc/%s", WebCtx.es_url, uuid_str);

-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 3);
    cJSON *json = NULL;
    if (r->status_code == 200) {
        json = cJSON_Parse(r->body);
@@ -334,7 +395,7 @@ char *elastic_get_status() {
    snprintf(url, 4096,
             "%s/_cluster/state/metadata/sist2?filter_path=metadata.indices.*.state", WebCtx.es_url);

-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
    cJSON *json = NULL;
    char *status = malloc(128 * sizeof(char));
    status[0] = '\0';
--- a/src/index/elastic.h
+++ b/src/index/elastic.h
@@ -16,15 +16,14 @@ typedef struct es_indexer es_indexer_t;

 void elastic_index_line(es_bulk_line_t *line);

-void elastic_flush();
-
 void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);

 void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);

 es_indexer_t *create_indexer(const char* es_url);

-void destroy_indexer(char *script, char index_id[UUID_STR_LEN]);
+void elastic_cleanup();
+void finish_indexer(char *script, int async_script, char *index_id);

 void elastic_init(int force_reset);

@@ -32,4 +31,6 @@ cJSON *elastic_get_document(const char *uuid_str);

 char *elastic_get_status();

+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]);
+
 #endif
--- a/src/index/static_generated.c
+++ b/src/index/static_generated.c
--- a/src/index/web.c
+++ b/src/index/web.c
@@ -1,4 +1,10 @@
 #include "web.h"
+#include "src/sist.h"
+
+#include <mongoose.h>
+#include <pthread.h>
+#include <curl/curl.h>
+

 size_t write_cb(char *ptr, size_t size, size_t nmemb, void *user_data) {

@@ -9,11 +15,104 @@ size_t write_cb(char *ptr, size_t size, size_t nmemb, void *user_data) {
 }

 void free_response(response_t *resp) {
-    free(resp->body);
+    if (resp->body != NULL) {
+        free(resp->body);
+    }
    free(resp);
 }

-response_t *web_get(const char *url) {
+#define SIST2_HEADERS "User-Agent: sist2\r\nContent-Type: application/json\r\n"
+
+
+void http_req_ev(struct mg_connection *nc, int ev, void *ptr) {
+
+    http_ev_data_t *ev_data = (http_ev_data_t *) nc->user_data;
+
+    switch (ev) {
+        case MG_EV_CONNECT: {
+            int connect_status = *(int *) ptr;
+            if (connect_status != 0) {
+                ev_data->done = TRUE;
+                ev_data->resp->status_code = 0;
+            }
+            break;
+        }
+        case MG_EV_HTTP_REPLY: {
+            struct http_message *hm = (struct http_message *) ptr;
+
+            //TODO: Check errors?
+
+            ev_data->resp->size = hm->body.len;
+            ev_data->resp->status_code = hm->resp_code;
+            ev_data->resp->body = malloc(hm->body.len + 1);
+            memcpy(ev_data->resp->body, hm->body.p, hm->body.len);
+            *(ev_data->resp->body + hm->body.len) = '\0';
+
+            ev_data->done = TRUE;
+            break;
+        }
+        case MG_EV_CLOSE: {
+            ev_data->done = TRUE;
+            break;
+        }
+        default:
+            break;
+    }
+}
+
+subreq_ctx_t *http_req(const char *url, const char *extra_headers, const char *post_data, const char *method) {
+
+    struct mg_str scheme;
+    struct mg_str user_info;
+    struct mg_str host;
+    unsigned int port;
+    struct mg_str path;
+    struct mg_str query;
+    struct mg_str fragment;
+
+    if (post_data == NULL) post_data = "";
+    if (extra_headers == NULL) extra_headers = "";
+    if (path.len == 0) path = mg_mk_str("/");
+    if (host.len == 0) host = mg_mk_str("");
+
+    // [scheme://[user_info@]]host[:port][/path][?query][#fragment]
+    mg_parse_uri(mg_mk_str(url), &scheme, &user_info, &host, &port, &path, &query, &fragment);
+
+    if (query.len > 0) path.len += query.len + 1;
+
+    subreq_ctx_t *ctx = malloc(sizeof(subreq_ctx_t));
+    mg_mgr_init(&ctx->mgr, NULL);
+
+    char address[8192];
+    snprintf(address, sizeof(address), "tcp://%.*s:%u", (int) host.len, host.p, port);
+    struct mg_connection *nc = mg_connect(&ctx->mgr, address, http_req_ev);
+    nc->user_data = &ctx->ev_data;
+    mg_set_protocol_http_websocket(nc);
+
+    ctx->ev_data.resp = calloc(1, sizeof(response_t));
+    ctx->ev_data.done = FALSE;
+
+    mg_printf(
+            nc, "%s %.*s HTTP/1.1\r\n"
+                  "Host: %.*s\r\n"
+                  "Content-Length: %zu\r\n"
+                  "%s\r\n"
+                  "%s",
+            method, (int) path.len, path.p,
+            (int) (path.p - host.p), host.p,
+            strlen(post_data),
+            extra_headers,
+            post_data
+    );
+
+    return ctx;
+}
+
+subreq_ctx_t *web_post_async(const char *url, const char *data) {
+    return http_req(url, SIST2_HEADERS, data, "POST");
+}
+
+response_t *web_get(const char *url, int timeout) {
    response_t *resp = malloc(sizeof(response_t));

    CURL *curl;
@@ -24,18 +123,23 @@ response_t *web_get(const char *url) {
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
+
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_perform(curl);
    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);

    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);

    resp->body = buffer.buf;
    resp->size = buffer.cur;
    return resp;
 }

-response_t *web_post(const char *url, const char *data, const char *header) {
+response_t *web_post(const char *url, const char *data) {

    response_t *resp = malloc(sizeof(response_t));

@@ -49,11 +153,8 @@ response_t *web_post(const char *url, const char *data, const char *header) {
    curl_easy_setopt(curl, CURLOPT_POST, 1);
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");

-    struct curl_slist *headers = NULL;
-    if (header != NULL) {
-        headers = curl_slist_append(headers, header);
-        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
-    }
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);

@@ -70,7 +171,7 @@ response_t *web_post(const char *url, const char *data, const char *header) {
 }


-response_t *web_put(const char *url, const char *data, const char *header) {
+response_t *web_put(const char *url, const char *data) {

    response_t *resp = malloc(sizeof(response_t));

@@ -86,11 +187,8 @@ response_t *web_put(const char *url, const char *data, const char *header) {
    curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
    curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4 );

-    if (header != NULL) {
-        struct curl_slist *headers = NULL;
-        headers = curl_slist_append(headers, header);
-        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
-    }
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);

@@ -98,6 +196,7 @@ response_t *web_put(const char *url, const char *data, const char *header) {
    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);

    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);

    resp->body = buffer.buf;
    resp->size = buffer.cur;
@@ -119,13 +218,16 @@ response_t *web_delete(const char *url) {
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");

    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_perform(curl);
    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);

    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);

    resp->body = buffer.buf;
    resp->size = buffer.cur;
    return resp;
-}
+}
--- a/src/index/web.h
+++ b/src/index/web.h
@@ -2,6 +2,7 @@
 #define SIST2_WEB_H

 #include "src/sist.h"
+#include <mongoose.h>

 typedef struct response {
    char *body;
@@ -9,9 +10,20 @@ typedef struct response {
    int status_code;
 } response_t;

-response_t *web_get(const char *url);
-response_t *web_post(const char * url, const char * data, const char* header);
-response_t *web_put(const char *url, const char *data, const char *header);
+typedef struct {
+    response_t *resp;
+    int done;
+} http_ev_data_t;
+
+typedef struct {
+    http_ev_data_t ev_data;
+    struct mg_mgr mgr;
+} subreq_ctx_t;
+
+response_t *web_get(const char *url, int timeout);
+response_t *web_post(const char * url, const char * data);
+subreq_ctx_t *web_post_async(const char *url, const char *data);
+response_t *web_put(const char *url, const char *data);
 response_t *web_delete(const char *url);

 void free_response(response_t *resp);
--- a/src/io/serialize.c
+++ b/src/io/serialize.c
@@ -1,5 +1,7 @@
 #include "src/ctx.h"
 #include "serialize.h"
+#include "src/parsing/parse.h"
+#include "src/parsing/mime.h"

 static __thread int index_fd = -1;

@@ -40,10 +42,13 @@ void write_index_descriptor(char *path, index_descriptor_t *desc) {

    int fd = open(path, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
    if (fd < 0) {
-        LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno));
+        LOG_FATALF("serialize.c", "Could not open index descriptor: %s", strerror(errno));
    }
    char *str = cJSON_Print(json);
-    write(fd, str, strlen(str));
+    int ret = write(fd, str, strlen(str));
+    if (ret == -1) {
+        LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno));
+    }
    free(str);
    close(fd);

@@ -57,11 +62,14 @@ index_descriptor_t read_index_descriptor(char *path) {
    int fd = open(path, O_RDONLY);

    if (fd == -1) {
-        LOG_FATALF("serialize.c", "Invalid/corrupt index (Could not find descriptor): %s: %s\n", path ,strerror(errno))
+        LOG_FATALF("serialize.c", "Invalid/corrupt index (Could not find descriptor): %s: %s\n", path, strerror(errno))
    }

    char *buf = malloc(info.st_size + 1);
-    read(fd, buf, info.st_size);
+    int ret = read(fd, buf, info.st_size);
+    if (ret == -1) {
+        LOG_FATALF("serialize.c", "Could not read index descriptor: %s", strerror(errno));
+    }
    *(buf + info.st_size) = '\0';
    close(fd);

@@ -136,6 +144,12 @@ char *get_meta_key_text(enum metakey meta_key) {
            return "exif_model";
        case MetaExifDateTime:
            return "exif_datetime";
+        case MetaAuthor:
+            return "author";
+        case MetaModifiedBy:
+            return "modified_by";
+        case MetaThumbnail:
+            return "thumbnail";
        default:
            return NULL;
    }
@@ -158,8 +172,8 @@ void write_document(document_t *doc) {
    dyn_buffer_t buf = dyn_buffer_create();

    // Ignore root directory in the file path
-    doc->ext = doc->ext - ScanCtx.index.desc.root_len;
-    doc->base = doc->base - ScanCtx.index.desc.root_len;
+    doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
+    doc->base = (short) (doc->base - ScanCtx.index.desc.root_len);
    doc->filepath += ScanCtx.index.desc.root_len;

    dyn_buffer_write(&buf, doc, sizeof(line_t));
@@ -170,11 +184,11 @@ void write_document(document_t *doc) {
        dyn_buffer_write_char(&buf, meta->key);

        if (IS_META_INT(meta->key)) {
-            dyn_buffer_write_int(&buf, meta->intval);
+            dyn_buffer_write_int(&buf, meta->int_val);
        } else if (IS_META_LONG(meta->key)) {
-            dyn_buffer_write_long(&buf, meta->longval);
+            dyn_buffer_write_long(&buf, meta->long_val);
        } else {
-            dyn_buffer_write_str(&buf, meta->strval);
+            dyn_buffer_write_str(&buf, meta->str_val);
        }

        meta_line_t *tmp = meta;
@@ -205,7 +219,7 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
    FILE *file = fopen(path, "rb");
    while (1) {
        buf.cur = 0;
-        fread((void *) &line, 1, sizeof(line_t), file);
+        size_t _ = fread((void *) &line, 1, sizeof(line_t), file);
        if (feof(file)) {
            break;
        }
@@ -216,7 +230,7 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
        char uuid_str[UUID_STR_LEN];
        uuid_unparse(line.uuid, uuid_str);

-        const char* mime_text = mime_get_mime_text(line.mime);
+        const char *mime_text = mime_get_mime_text(line.mime);
        if (mime_text == NULL) {
            cJSON_AddNullToObject(document, "mime");
        } else {
@@ -225,54 +239,61 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
        cJSON_AddNumberToObject(document, "size", (double) line.size);
        cJSON_AddNumberToObject(document, "mtime", line.mtime);

-        int c;
+        int c = 0;
        while ((c = getc(file)) != 0) {
            dyn_buffer_write_char(&buf, (char) c);
        }
        dyn_buffer_write_char(&buf, '\0');

+        if (IndexCtx.tags != NULL) {
+            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, buf.buf);
+            if (tags_string != NULL) {
+                cJSON *tags_arr = cJSON_Parse(tags_string);
+                cJSON_AddItemToObject(document, "tag", tags_arr);
+            }
+        }
+
        cJSON_AddStringToObject(document, "extension", buf.buf + line.ext);
        if (*(buf.buf + line.ext - 1) == '.') {
            *(buf.buf + line.ext - 1) = '\0';
        } else {
            *(buf.buf + line.ext) = '\0';
        }
-        cJSON_AddStringToObject(document, "name", buf.buf + line.base);
+
+        char tmp[PATH_MAX * 3];
+
+        str_escape(tmp, buf.buf + line.base);
+        cJSON_AddStringToObject(document, "name", tmp);
+
        if (line.base > 0) {
            *(buf.buf + line.base - 1) = '\0';
-            cJSON_AddStringToObject(document, "path", buf.buf);
+
+            str_escape(tmp, buf.buf);
+            cJSON_AddStringToObject(document, "path", tmp);
        } else {
            cJSON_AddStringToObject(document, "path", "");
        }

        enum metakey key = getc(file);
+        size_t ret = 0;
        while (key != '\n') {
            switch (key) {
                case MetaWidth:
                case MetaHeight: {
                    int value;
-                    fread(&value, sizeof(int), 1, file);
+                    ret = fread(&value, sizeof(int), 1, file);
                    cJSON_AddNumberToObject(document, get_meta_key_text(key), value);
                    break;
                }
                case MetaMediaDuration:
                case MetaMediaBitrate: {
                    long value;
-                    fread(&value, sizeof(long), 1, file);
+                    ret = fread(&value, sizeof(long), 1, file);
                    cJSON_AddNumberToObject(document, get_meta_key_text(key), (double) value);
                    break;
                }
                case MetaMediaAudioCodec:
-                case MetaMediaVideoCodec: {
-                    int value;
-                    fread(&value, sizeof(int), 1, file);
-                    const AVCodecDescriptor *desc = avcodec_descriptor_get(value);
-                    if (desc != NULL) {
-                        cJSON_AddStringToObject(document, get_meta_key_text(key), desc->name);
-                    }
-                    break;
-                }
-
+                case MetaMediaVideoCodec:
                case MetaContent:
                case MetaArtist:
                case MetaAlbum:
@@ -289,6 +310,9 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
                case MetaExifIsoSpeedRatings:
                case MetaExifDateTime:
                case MetaExifModel:
+                case MetaAuthor:
+                case MetaModifiedBy:
+                case MetaThumbnail:
                case MetaTitle: {
                    buf.cur = 0;
                    while ((c = getc(file)) != 0) {
@@ -402,8 +426,8 @@ void incremental_read(GHashTable *table, const char *filepath) {
    line_t line;

    while (1) {
-        fread((void *) &line, 1, sizeof(line_t), file);
-        if (feof(file)) {
+        size_t ret = fread((void *) &line, 1, sizeof(line_t), file);
+        if (ret != 1 || feof(file)) {
            break;
        }

@@ -426,8 +450,8 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
    line_t line;

    while (1) {
-        fread((void *) &line, 1, sizeof(line_t), file);
-        if (feof(file)) {
+        size_t ret = fread((void *) &line, 1, sizeof(line_t), file);
+        if (ret != 1 || feof(file)) {
            break;
        }

@@ -455,11 +479,11 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,

                if (IS_META_INT(key)) {
                    int val;
-                    fread(&val, sizeof(val), 1, file);
+                    ret = fread(&val, sizeof(val), 1, file);
                    fwrite(&val, sizeof(val), 1, dst_file);
                } else if (IS_META_LONG(key)) {
                    long val;
-                    fread(&val, sizeof(val), 1, file);
+                    ret = fread(&val, sizeof(val), 1, file);
                    fwrite(&val, sizeof(val), 1, dst_file);
                } else {
                    while ((c = (char) getc(file))) {
@@ -467,6 +491,10 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
                    }
                    fwrite("\0", sizeof(c), 1, dst_file);
                }
+
+                if (ret != 1) {
+                    break;
+                }
            }
        } else {
            skip_meta(file);
--- a/src/io/serialize.h
+++ b/src/io/serialize.h
@@ -2,7 +2,10 @@
 #define SIST2_SERIALIZE_H

 #include "src/sist.h"
+#include "store.h"
+
 #include <sys/syscall.h>
+#include <glib.h>

 typedef void(*index_func)(cJSON *, const char[UUID_STR_LEN]);

--- a/src/io/store.c
+++ b/src/io/store.c
@@ -1,9 +1,10 @@
 #include "store.h"
 #include "src/ctx.h"

-store_t *store_create(char *path) {
+store_t *store_create(char *path, size_t chunk_size) {

    store_t *store = malloc(sizeof(struct store_t));
+    store->chunk_size = chunk_size;
    pthread_rwlock_init(&store->lock, NULL);

    mdb_env_create(&store->env);
@@ -18,7 +19,7 @@ store_t *store_create(char *path) {
        LOG_FATALF("store.c", "Error while opening store: %s (%s)\n", mdb_strerror(open_ret), path)
    }

-    store->size = (size_t) 1024 * 1024 * 5;
+    store->size = (size_t) store->chunk_size;
    ScanCtx.stat_tn_size = 0;
    mdb_env_set_mapsize(store->env, store->size);

@@ -69,7 +70,7 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
        // Cannot resize when there is a opened transaction.
        //  Resize take effect on the next commit.
        pthread_rwlock_wrlock(&store->lock);
-        store->size += 1024 * 1024 * 50;
+        store->size += store->chunk_size;
        mdb_env_set_mapsize(store->env, store->size);
        mdb_txn_begin(store->env, NULL, 0, &txn);
        put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
@@ -110,3 +111,40 @@ char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen)
    return buf;
 }

+GHashTable *store_read_all(store_t *store) {
+
+    int count = 0;
+
+    GHashTable *table = g_hash_table_new_full(g_str_hash, g_str_equal, free, free);
+
+    MDB_txn *txn = NULL;
+    mdb_txn_begin(store->env, NULL, MDB_RDONLY, &txn);
+
+    MDB_cursor *cur = NULL;
+    mdb_cursor_open(txn, store->dbi, &cur);
+
+    MDB_val key;
+    MDB_val value;
+
+    while (mdb_cursor_get(cur, &key, &value, MDB_NEXT) == 0) {
+        char *key_str = malloc(key.mv_size);
+        memcpy(key_str, key.mv_data, key.mv_size);
+        char *val_str = malloc(value.mv_size);
+        memcpy(val_str, value.mv_data, value.mv_size);
+
+        g_hash_table_insert(table, key_str, val_str);
+        count += 1;
+    }
+
+    LOG_DEBUGF("store.c", "Read tags for %d documents", count);
+
+    mdb_cursor_close(cur);
+    mdb_txn_abort(txn);
+    return table;
+}
+
+
+void store_copy(store_t *store, const char *destination) {
+    mkdir(destination, S_IWUSR | S_IRUSR | S_IXUSR);
+    mdb_env_copy(store->env, destination);
+}
--- a/src/io/store.h
+++ b/src/io/store.h
@@ -4,16 +4,20 @@
 #include <pthread.h>
 #include <lmdb.h>

+#include <glib.h>
+
+#define STORE_SIZE_TN 1024 * 1024 * 5
+#define STORE_SIZE_TAG 1024 * 16
+
 typedef struct store_t {
    MDB_dbi dbi;
    MDB_env *env;
    size_t size;
+    size_t chunk_size;
    pthread_rwlock_t lock;
 } store_t;

-#include "src/sist.h"
-
-store_t *store_create(char *path);
+store_t *store_create(char *path, size_t chunk_size);

 void store_destroy(store_t *store);

@@ -21,4 +25,8 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu

 char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen);

+GHashTable *store_read_all(store_t *store);
+
+void store_copy(store_t *store, const char *destination);
+
 #endif
--- a/src/io/walk.c
+++ b/src/io/walk.c
@@ -1,5 +1,8 @@
 #include "walk.h"
 #include "src/ctx.h"
+#include "src/parsing/parse.h"
+
+#include <ftw.h>

 __always_inline
 parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info, int base) {
@@ -15,12 +18,13 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
        job->ext = len;
    }

-    job->info = *info;
+    job->vfile.info = *info;

    memset(job->parent, 0, 16);

    job->vfile.filepath = job->filepath;
    job->vfile.read = fs_read;
+    job->vfile.reset = fs_reset;
    job->vfile.close = fs_close;
    job->vfile.fd = -1;
    job->vfile.is_fs_file = TRUE;
--- a/src/io/walk.h
+++ b/src/io/walk.h
@@ -3,8 +3,6 @@

 #define _XOPEN_SOURCE 500

-#include "src/sist.h"
-
 int walk_directory_tree(const char *);

 #endif
--- a/src/log.c
+++ b/src/log.c
@@ -1,5 +1,8 @@
 #include "log.h"

+#include <pthread.h>
+#include <stdarg.h>
+
 const char *log_colors[] = {
        "\033[34m", "\033[01;34m", "\033[0m",
        "\033[01;33m", "\033[31m", "\033[01;31m"
@@ -9,7 +12,7 @@ const char *log_levels[] = {
        "DEBUG", "INFO", "WARNING", "ERROR", "FATAL"
 };

-void sist_logf(char *filepath, int level, char *format, ...) {
+void vsist_logf(const char *filepath, int level, char *format, va_list ap) {

    static int is_tty = -1;
    if (is_tty == -1) {
@@ -31,23 +34,20 @@ void sist_logf(char *filepath, int level, char *format, ...) {
    if (is_tty) {
        log_len = snprintf(
                log_str, sizeof(log_str),
-                "\033[%dm[%04X]%s [%s] [%s %s] ",
+                "\033[%dm[%04llX]%s [%s] [%s %s] ",
                31 + ((unsigned int) (pid)) % 7, pid, log_colors[level],
                datetime, log_levels[level], filepath
        );
    } else {
        log_len = snprintf(
                log_str, sizeof(log_str),
-                "[%04X] [%s] [%s %s] ",
+                "[%04llX] [%s] [%s %s] ",
                pid, datetime, log_levels[level], filepath
        );
    }

-    va_list ap;
-    va_start(ap, format);
    size_t maxsize = sizeof(log_str) - log_len;
    log_len += vsnprintf(log_str + log_len, maxsize, format, ap);
-    va_end(ap);

    if (is_tty) {
        log_len += sprintf(log_str + log_len, "\033[0m\n");
@@ -56,10 +56,20 @@ void sist_logf(char *filepath, int level, char *format, ...) {
        log_len += 1;
    }

-    write(STDERR_FILENO, log_str, log_len);
+    int ret = write(STDERR_FILENO, log_str, log_len);
+    if (ret == -1) {
+        LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno))
+    }
 }

-void sist_log(char *filepath, int level, char *str) {
+void sist_logf(const char *filepath, int level, char *format, ...) {
+    va_list ap;
+    va_start(ap, format);
+    vsist_logf(filepath, level, format, ap);
+    va_end(ap);
+}
+
+void sist_log(const char *filepath, int level, char *str) {

    static int is_tty = -1;
    if (is_tty == -1) {
@@ -81,7 +91,7 @@ void sist_log(char *filepath, int level, char *str) {
    if (is_tty) {
        log_len = snprintf(
                log_str, sizeof(log_str),
-                "\033[%dm[%04X]%s [%s] [%s %s] %s \033[0m\n",
+                "\033[%dm[%04llX]%s [%s] [%s %s] %s \033[0m\n",
                31 + ((unsigned int) (pid)) % 7, pid, log_colors[level],
                datetime, log_levels[level], filepath,
                str
@@ -89,11 +99,14 @@ void sist_log(char *filepath, int level, char *str) {
    } else {
        log_len = snprintf(
                log_str, sizeof(log_str),
-                "[%04X] [%s] [%s %s] %s \n",
+                "[%04llX] [%s] [%s %s] %s \n",
                pid, datetime, log_levels[level], filepath,
                str
        );
    }

-    write(STDERR_FILENO, log_str, log_len);
+    int ret = write(STDERR_FILENO, log_str, log_len);
+    if (ret == -1) {
+        LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno));
+    }
 }
--- a/src/log.h
+++ b/src/log.h
@@ -1,6 +1,7 @@
 #ifndef SIST2_LOG_H
 #define SIST2_LOG_H

+
 #define LOG_MAX_LENGTH 8192

 #define SIST_DEBUG 0
@@ -36,10 +37,11 @@
    sist_log(filepath, SIST_FATAL, str);\
    exit(-1);

-#include "src/sist.h"
+#include "sist.h"

-void sist_logf(char *filepath, int level, char *format, ...);
+void sist_logf(const char *filepath, int level, char *format, ...);
+void vsist_logf(const char *filepath, int level, char *format, va_list ap);

-void sist_log(char *filepath, int level, char *str);
+void sist_log(const char *filepath, int level, char *str);

 #endif
--- a/src/main.c
+++ b/src/main.c
@@ -1,24 +1,35 @@
 #include "sist.h"
 #include "ctx.h"

+#include <third-party/argparse/argparse.h>
+#include <locale.h>
+
+#include "cli.h"
+#include "io/serialize.h"
+#include "io/store.h"
+#include "tpool.h"
+#include "io/walk.h"
+#include "index/elastic.h"
+#include "web/serve.h"
+#include "parsing/mime.h"
+#include "parsing/parse.h"
+
+#include "stats.h"
+
 #define DESCRIPTION "Lightning-fast file system indexer and search tool."

 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"


-static const char *const Version = "1.3.2";
+static const char *const Version = "2.8.0";
 static const char *const usage[] = {
        "sist2 scan [OPTION]... PATH",
        "sist2 index [OPTION]... INDEX",
        "sist2 web [OPTION]... INDEX...",
+        "sist2 exec-script [OPTION]... INDEX",
        NULL,
 };

-void global_init() {
-    curl_global_init(CURL_GLOBAL_NOTHING);
-    av_log_set_level(AV_LOG_QUIET);
-}
-
 void init_dir(const char *dirpath) {
    char path[PATH_MAX];
    snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
@@ -37,34 +48,139 @@ void scan_print_header() {
    LOG_INFOF("main.c", "sist2 v%s", Version)
 }

-void sist2_scan(scan_args_t *args) {
+void _store(char *key, size_t key_len, char *buf, size_t buf_len) {
+    store_write(ScanCtx.index.store, key, key_len, buf, buf_len);
+}
+
+void _log(const char *filepath, int level, char *str) {
+    if (level == LEVEL_FATAL) {
+        sist_log(filepath, level, str);
+        exit(-1);
+    }
+
+    if (LogCtx.verbose) {
+        if (level == LEVEL_DEBUG) {
+            if (LogCtx.very_verbose) {
+                sist_log(filepath, level, str);
+            }
+        } else {
+            sist_log(filepath, level, str);
+        }
+    }
+}
+
+void _logf(const char *filepath, int level, char *format, ...) {
+
+    va_list args;
+
+    va_start(args, format);
+    if (level == LEVEL_FATAL) {
+        vsist_logf(filepath, level, format, args);
+        exit(-1);
+    }
+
+    if (LogCtx.verbose) {
+        if (level == LEVEL_DEBUG) {
+            if (LogCtx.very_verbose) {
+                vsist_logf(filepath, level, format, args);
+            }
+        } else {
+            vsist_logf(filepath, level, format, args);
+        }
+    }
+    va_end(args);
+}
+
+void initialize_scan_context(scan_args_t *args) {
+
+    // Arc
+    ScanCtx.arc_ctx.mode = args->archive_mode;
+    ScanCtx.arc_ctx.log = _log;
+    ScanCtx.arc_ctx.logf = _logf;
+    ScanCtx.arc_ctx.parse = (parse_callback_t) parse;
+
+    // Comic
+    ScanCtx.comic_ctx.log = _log;
+    ScanCtx.comic_ctx.logf = _logf;
+    ScanCtx.comic_ctx.store = _store;
+    ScanCtx.comic_ctx.tn_size = args->size;
+    ScanCtx.comic_ctx.tn_qscale = args->quality;
+    ScanCtx.comic_ctx.cbr_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbr");
+    ScanCtx.comic_ctx.cbz_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbz");
+
+    // Ebook
+    pthread_mutex_init(&ScanCtx.ebook_ctx.mupdf_mutex, NULL);
+    ScanCtx.ebook_ctx.content_size = args->content_size;
+    ScanCtx.ebook_ctx.tn_size = args->size;
+    ScanCtx.ebook_ctx.tesseract_lang = args->tesseract_lang;
+    ScanCtx.ebook_ctx.tesseract_path = args->tesseract_path;
+    ScanCtx.ebook_ctx.log = _log;
+    ScanCtx.ebook_ctx.logf = _logf;
+    ScanCtx.ebook_ctx.store = _store;
+
+    // Font
+    ScanCtx.font_ctx.enable_tn = args->size > 0;
+    ScanCtx.font_ctx.log = _log;
+    ScanCtx.font_ctx.logf = _logf;
+    ScanCtx.font_ctx.store = _store;
+
+    // Media
+    ScanCtx.media_ctx.tn_qscale = args->quality;
+    ScanCtx.media_ctx.tn_size = args->size;
+    ScanCtx.media_ctx.log = _log;
+    ScanCtx.media_ctx.logf = _logf;
+    ScanCtx.media_ctx.store = _store;
+    ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer * 1024 * 1024;
+    init_media();
+
+    // OOXML
+    ScanCtx.ooxml_ctx.content_size = args->content_size;
+    ScanCtx.ooxml_ctx.log = _log;
+    ScanCtx.ooxml_ctx.logf = _logf;
+    ScanCtx.ooxml_ctx.store = _store;
+
+    // MOBI
+    ScanCtx.mobi_ctx.content_size = args->content_size;
+    ScanCtx.mobi_ctx.log = _log;
+    ScanCtx.mobi_ctx.logf = _logf;
+
+    // TEXT
+    ScanCtx.text_ctx.content_size = args->content_size;
+    ScanCtx.text_ctx.log = _log;
+    ScanCtx.text_ctx.logf = _logf;

-    ScanCtx.tn_qscale = args->quality;
-    ScanCtx.tn_size = args->size;
-    ScanCtx.content_size = args->content_size;
    ScanCtx.threads = args->threads;
    ScanCtx.depth = args->depth;
-    ScanCtx.archive_mode = args->archive_mode;
+
    strncpy(ScanCtx.index.path, args->output, sizeof(ScanCtx.index.path));
    strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name));
    strncpy(ScanCtx.index.desc.root, args->path, sizeof(ScanCtx.index.desc.root));
    strncpy(ScanCtx.index.desc.rewrite_url, args->rewrite_url, sizeof(ScanCtx.index.desc.rewrite_url));
    ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root);
-    ScanCtx.tesseract_lang = args->tesseract_lang;
-    ScanCtx.tesseract_path = args->tesseract_path;
    ScanCtx.fast = args->fast;

-    init_dir(ScanCtx.index.path);
+    // Raw
+    ScanCtx.raw_ctx.tn_qscale = args->quality;
+    ScanCtx.raw_ctx.tn_size = args->size;
+    ScanCtx.raw_ctx.log = _log;
+    ScanCtx.raw_ctx.logf = _logf;
+    ScanCtx.raw_ctx.store = _store;
+}
+
+
+void sist2_scan(scan_args_t *args) {

    ScanCtx.mime_table = mime_get_mime_table();
    ScanCtx.ext_table = mime_get_ext_table();

-    cbr_init();
+    initialize_scan_context(args);
+
+    init_dir(ScanCtx.index.path);

    char store_path[PATH_MAX];
    snprintf(store_path, PATH_MAX, "%sthumbs", ScanCtx.index.path);
    mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
-    ScanCtx.index.store = store_create(store_path);
+    ScanCtx.index.store = store_create(store_path, STORE_SIZE_TN);

    scan_print_header();

@@ -99,17 +215,19 @@ void sist2_scan(scan_args_t *args) {
        LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table))
    }

-    ScanCtx.pool = tpool_create(args->threads, thread_cleanup);
+    ScanCtx.pool = tpool_create(args->threads, thread_cleanup, TRUE);
    tpool_start(ScanCtx.pool);
    walk_directory_tree(ScanCtx.index.desc.root);
    tpool_wait(ScanCtx.pool);
    tpool_destroy(ScanCtx.pool);

+    generate_stats(&ScanCtx.index, args->treemap_threshold, ScanCtx.index.path);
+
    if (args->incremental != NULL) {
        char dst_path[PATH_MAX];
        snprintf(store_path, PATH_MAX, "%sthumbs", args->incremental);
        snprintf(dst_path, PATH_MAX, "%s_index_original", ScanCtx.index.path);
-        store_t *source = store_create(store_path);
+        store_t *source = store_create(store_path, STORE_SIZE_TN);

        DIR *dir = opendir(args->incremental);
        if (dir == NULL) {
@@ -126,6 +244,13 @@ void sist2_scan(scan_args_t *args) {
        }
        closedir(dir);
        store_destroy(source);
+
+        snprintf(store_path, PATH_MAX, "%stags", args->incremental);
+        snprintf(dst_path, PATH_MAX, "%stags", ScanCtx.index.path);
+        mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
+        store_t *source_tags = store_create(store_path, STORE_SIZE_TAG);
+        store_copy(source_tags, dst_path);
+        store_destroy(source_tags);
    }

    store_destroy(ScanCtx.index.store);
@@ -157,6 +282,12 @@ void sist2_index(index_args_t *args) {
        LOG_FATALF("main.c", "Could not open index %s: %s", args->index_path, strerror(errno))
    }

+    char path_tmp[PATH_MAX];
+    snprintf(path_tmp, sizeof(path_tmp), "%s/tags", args->index_path);
+    mkdir(path_tmp, S_IWUSR | S_IRUSR | S_IXUSR);
+    IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG);
+    IndexCtx.tags = store_read_all(IndexCtx.tag_store);
+
    index_func f;
    if (args->print) {
        f = print_json;
@@ -164,6 +295,16 @@ void sist2_index(index_args_t *args) {
        f = index_json;
    }

+    void (*cleanup)();
+    if (args->print) {
+        cleanup = NULL;
+    } else {
+        cleanup = elastic_cleanup;
+    }
+
+    IndexCtx.pool = tpool_create(args->threads, cleanup, FALSE);
+    tpool_start(IndexCtx.pool);
+
    struct dirent *de;
    while ((de = readdir(dir)) != NULL) {
        if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
@@ -174,17 +315,43 @@ void sist2_index(index_args_t *args) {
    }
    closedir(dir);

+    tpool_wait(IndexCtx.pool);
+
+    tpool_destroy(IndexCtx.pool);
+
    if (!args->print) {
-        elastic_flush();
-        destroy_indexer(args->script, desc.uuid);
+        finish_indexer(args->script, args->async_script, desc.uuid);
    }
+
+    store_destroy(IndexCtx.tag_store);
+    g_hash_table_remove_all(IndexCtx.tags);
+    g_hash_table_destroy(IndexCtx.tags);
+}
+
+void sist2_exec_script(exec_args_t *args) {
+
+    LogCtx.verbose = TRUE;
+
+    char descriptor_path[PATH_MAX];
+    snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path);
+    index_descriptor_t desc = read_index_descriptor(descriptor_path);
+
+    IndexCtx.es_url = args->es_url;
+
+    LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)
+
+    execute_update_script(args->script, args->async_script, desc.uuid);
+    free(args->script);
 }

 void sist2_web(web_args_t *args) {

    WebCtx.es_url = args->es_url;
    WebCtx.index_count = args->index_count;
-    WebCtx.b64credentials = args->b64credentials;
+    WebCtx.auth_user = args->auth_user;
+    WebCtx.auth_pass = args->auth_pass;
+    WebCtx.auth_enabled = args->auth_enabled;
+    WebCtx.tag_auth_enabled = args->tag_auth_enabled;

    for (int i = 0; i < args->index_count; i++) {
        char *abs_path = abspath(args->indices[i]);
@@ -194,7 +361,11 @@ void sist2_web(web_args_t *args) {
        char path_tmp[PATH_MAX];

        snprintf(path_tmp, PATH_MAX, "%sthumbs", abs_path);
-        WebCtx.indices[i].store = store_create(path_tmp);
+        WebCtx.indices[i].store = store_create(path_tmp, STORE_SIZE_TN);
+
+        snprintf(path_tmp, PATH_MAX, "%stags", abs_path);
+        mkdir(path_tmp, S_IWUSR | S_IRUSR | S_IXUSR);
+        WebCtx.indices[i].tag_store = store_create(path_tmp, STORE_SIZE_TAG);

        snprintf(path_tmp, PATH_MAX, "%sdescriptor.json", abs_path);
        WebCtx.indices[i].desc = read_index_descriptor(path_tmp);
@@ -204,21 +375,24 @@ void sist2_web(web_args_t *args) {
        free(abs_path);
    }

-    serve(args->bind, args->port);
+    serve(args->listen_address);
 }


 int main(int argc, const char *argv[]) {
-
-    global_init();
+    setlocale(LC_ALL, "");

    scan_args_t *scan_args = scan_args_create();
    index_args_t *index_args = index_args_create();
    web_args_t *web_args = web_args_create();
+    exec_args_t *exec_args = exec_args_create();

    int arg_version = 0;

    char *common_es_url = NULL;
+    char *common_script_path = NULL;
+    int common_async_script = 0;
+    int common_threads = 0;

    struct argparse_option options[] = {
            OPT_HELP(),
@@ -228,7 +402,7 @@ int main(int argc, const char *argv[]) {
            OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages"),

            OPT_GROUP("Scan options"),
-            OPT_INTEGER('t', "threads", &scan_args->threads, "Number of threads. DEFAULT=1"),
+            OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
            OPT_FLOAT('q', "quality", &scan_args->quality,
                      "Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5"),
            OPT_INTEGER(0, "size", &scan_args->size,
@@ -249,20 +423,31 @@ int main(int argc, const char *argv[]) {
                                                             "which are installed on your machine)"),
            OPT_STRING('e', "exclude", &scan_args->exclude_regex, "Files that match this regex will not be scanned"),
            OPT_BOOLEAN(0, "fast", &scan_args->fast, "Only index file names & mime type"),
+            OPT_STRING(0, "treemap-threshold", &scan_args->treemap_threshold_str, "Relative size threshold for treemap "
+                                                                                  "(see USAGE.md). DEFAULT: 0.0005"),
+            OPT_INTEGER(0, "mem-buffer", &scan_args->max_memory_buffer,
+                        "Maximum memory buffer size per thread in MB for files inside archives "
+                        "(see USAGE.md). DEFAULT: 2000"),

            OPT_GROUP("Index options"),
+            OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
            OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
            OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
-            OPT_STRING(0, "script-file", &index_args->script_path, "Path to user script."),
+            OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
            OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 100"),
            OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "
                                                                      "(You must use this option the first time you use the index command)"),

            OPT_GROUP("Web options"),
            OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
-            OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
-            OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
+            OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"),
            OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
+            OPT_STRING(0, "tag-auth", &web_args->tag_credentials, "Basic auth in user:password format for tagging"),
+
+            OPT_GROUP("Exec-script options"),
+            OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),

            OPT_END(),
    };
@@ -283,6 +468,13 @@ int main(int argc, const char *argv[]) {

    web_args->es_url = common_es_url;
    index_args->es_url = common_es_url;
+    exec_args->es_url = common_es_url;
+    index_args->script_path = common_script_path;
+    exec_args->script_path = common_script_path;
+    index_args->threads = common_threads;
+    scan_args->threads = common_threads;
+    exec_args->async_script = common_async_script;
+    index_args->async_script = common_async_script;

    if (argc == 0) {
        argparse_usage(&argparse);
@@ -311,6 +503,14 @@ int main(int argc, const char *argv[]) {
        }
        sist2_web(web_args);

+    }  else if (strcmp(argv[0], "exec-script") == 0) {
+
+        int err = exec_args_validate(exec_args, argc, argv);
+        if (err != 0) {
+            goto end;
+        }
+        sist2_exec_script(exec_args);
+
    } else {
        fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
        argparse_usage(&argparse);
@@ -322,6 +522,7 @@ int main(int argc, const char *argv[]) {
    scan_args_destroy(scan_args);
    index_args_destroy(index_args);
    web_args_destroy(web_args);
+    exec_args_destroy(exec_args);

    return 0;
 }
--- a/src/parsing/arc.c
+++ b/src/parsing/arc.c
@@ -1,155 +0,0 @@
-#include "arc.h"
-#include "src/ctx.h"
-
-int should_parse_filtered_file(const char *filepath, int ext) {
-    char tmp[PATH_MAX * 2];
-
-    if (ext == 0) {
-        return FALSE;
-    }
-
-    memcpy(tmp, filepath, ext - 1);
-    *(tmp + ext - 1) = '\0';
-
-    char *idx = strrchr(tmp, '.');
-
-    if (idx == NULL) {
-        return FALSE;
-    }
-
-    if (strcmp(idx, ".tar") == 0) {
-        return TRUE;
-    }
-
-    return FALSE;
-}
-
-int arc_read(struct vfile *f, void *buf, size_t size) {
-    return archive_read_data(f->arc, buf, size);
-}
-
-typedef struct arc_data {
-    vfile_t *f;
-    char buf[ARC_BUF_SIZE];
-} arc_data_f;
-
-int vfile_open_callback(struct archive *a, void *user_data) {
-    arc_data_f *data = user_data;
-
-    if (data->f->is_fs_file && data->f->fd == -1) {
-        data->f->fd = open(data->f->filepath, O_RDONLY);
-    }
-
-    return ARCHIVE_OK;
-}
-
-long vfile_read_callback(struct archive *a, void *user_data, const void **buf) {
-    arc_data_f *data = user_data;
-
-    *buf = data->buf;
-    return data->f->read(data->f, data->buf, ARC_BUF_SIZE);
-}
-
-int vfile_close_callback(struct archive *a, void *user_data) {
-    arc_data_f *data = user_data;
-
-    if (data->f->close != NULL) {
-        data->f->close(data->f);
-    }
-
-    return ARCHIVE_OK;
-}
-
-void parse_archive(vfile_t *f, document_t *doc) {
-
-    struct archive *a;
-    struct archive_entry *entry;
-
-
-    arc_data_f data;
-    data.f = f;
-
-    int ret = 0;
-    if (data.f->is_fs_file) {
-
-        a = archive_read_new();
-        archive_read_support_filter_all(a);
-        archive_read_support_format_all(a);
-
-        ret = archive_read_open_filename(a, doc->filepath, ARC_BUF_SIZE);
-    } else if (ScanCtx.archive_mode == ARC_MODE_RECURSE) {
-
-        a = archive_read_new();
-        archive_read_support_filter_all(a);
-        archive_read_support_format_all(a);
-
-        ret = archive_read_open(
-                a, &data,
-                vfile_open_callback,
-                vfile_read_callback,
-                vfile_close_callback
-        );
-    } else {
-        return;
-    }
-
-    if (ret != ARCHIVE_OK) {
-        LOG_ERRORF(doc->filepath, "(arc.c) [%d] %s", ret, archive_error_string(a))
-        archive_read_free(a);
-        return;
-    }
-
-    if (ScanCtx.archive_mode == ARC_MODE_LIST) {
-
-        dyn_buffer_t buf = dyn_buffer_create();
-
-        while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
-            if (S_ISREG(archive_entry_stat(entry)->st_mode)) {
-
-                char *path = (char *) archive_entry_pathname(entry);
-
-                dyn_buffer_append_string(&buf, path);
-                dyn_buffer_write_char(&buf, '\n');
-            }
-        }
-        dyn_buffer_write_char(&buf, '\0');
-
-        meta_line_t *meta_list = malloc(sizeof(meta_line_t) + buf.cur);
-        meta_list->key = MetaContent;
-        strcpy(meta_list->strval, buf.buf);
-        APPEND_META(doc, meta_list);
-        dyn_buffer_destroy(&buf);
-
-    } else {
-
-        parse_job_t *sub_job = malloc(sizeof(parse_job_t) + PATH_MAX * 2);
-
-        sub_job->vfile.close = NULL;
-        sub_job->vfile.read = arc_read;
-        sub_job->vfile.arc = a;
-        sub_job->vfile.filepath = sub_job->filepath;
-        sub_job->vfile.is_fs_file = FALSE;
-        memcpy(sub_job->parent, doc->uuid, sizeof(uuid_t));
-
-        while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
-            sub_job->info = *archive_entry_stat(entry);
-            if (S_ISREG(sub_job->info.st_mode)) {
-                sprintf(sub_job->filepath, "%s#/%s", f->filepath, archive_entry_pathname(entry));
-                sub_job->base = (int) (strrchr(sub_job->filepath, '/') - sub_job->filepath) + 1;
-
-                char *p = strrchr(sub_job->filepath, '.');
-                if (p != NULL) {
-                    sub_job->ext = (int) (p - sub_job->filepath + 1);
-                } else {
-                    sub_job->ext = (int) strlen(sub_job->filepath);
-                }
-
-                parse(sub_job);
-            }
-        }
-
-        free(sub_job);
-    }
-
-    archive_read_free(a);
-}
--- a/src/parsing/arc.h
+++ b/src/parsing/arc.h
@@ -1,13 +0,0 @@
-#ifndef SIST2_ARC_H
-#define SIST2_ARC_H
-
-#include "src/sist.h"
-#define ARC_BUF_SIZE 8192
-
-int should_parse_filtered_file(const char *filepath, int ext);
-
-void parse_archive(vfile_t *f, document_t *doc);
-
-int arc_read(struct vfile * f, void *buf, size_t size);
-
-#endif
--- a/src/parsing/cbr.c
+++ b/src/parsing/cbr.c
@@ -1,52 +0,0 @@
-#include "cbr.h"
-#include "src/ctx.h"
-
-unsigned int cbr_mime;
-unsigned int cbz_mime;
-
-void cbr_init() {
-    cbr_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbr");
-    cbz_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbz");
-}
-
-int is_cbr(unsigned int mime) {
-    return mime == cbr_mime;
-}
-
-void parse_cbr(void *buf, size_t buf_len, document_t *doc) {
-    char *out_buf = malloc(buf_len * 2);
-    size_t out_buf_used = 0;
-
-    struct archive *rar_in = archive_read_new();
-    archive_read_support_filter_none(rar_in);
-    archive_read_support_format_rar(rar_in);
-
-    archive_read_open_memory(rar_in, buf, buf_len);
-
-    struct archive *zip_out = archive_write_new();
-    archive_write_set_format_zip(zip_out);
-    archive_write_open_memory(zip_out, out_buf, buf_len * 2, &out_buf_used);
-
-    struct archive_entry *entry;
-    while (archive_read_next_header(rar_in, &entry) == ARCHIVE_OK) {
-        archive_write_header(zip_out, entry);
-
-        char arc_buf[ARC_BUF_SIZE];
-        int len = archive_read_data(rar_in, arc_buf, ARC_BUF_SIZE);
-        while (len > 0) {
-            archive_write_data(zip_out, arc_buf, len);
-            len = archive_read_data(rar_in, arc_buf, ARC_BUF_SIZE);
-        }
-    }
-
-    archive_write_close(zip_out);
-    archive_write_free(zip_out);
-
-    archive_read_close(rar_in);
-    archive_read_free(rar_in);
-
-    doc->mime = cbz_mime;
-    parse_pdf(out_buf, out_buf_used, doc);
-    doc->mime = cbr_mime;
-    free(out_buf);
-}
--- a/src/parsing/cbr.h
+++ b/src/parsing/cbr.h
@@ -1,12 +0,0 @@
-#ifndef SIST2_CBR_H
-#define SIST2_CBR_H
-
-#include "src/sist.h"
-
-void cbr_init();
-
-int is_cbr(unsigned int mime);
-
-void parse_cbr(void *buf, size_t buf_len, document_t *doc);
-
-#endif
--- a/src/parsing/doc.c
+++ b/src/parsing/doc.c
@@ -1,140 +0,0 @@
-#include "doc.h"
-#include "src/ctx.h"
-
-
-#define STR_STARTS_WITH(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
-
-__always_inline
-static int should_read_part(const char *part) {
-
-    LOG_DEBUGF("doc.c", "Got part : %s", part)
-
-    if (part == NULL) {
-        return FALSE;
-    }
-
-    if (    // Word
-            STR_STARTS_WITH(part, "word/document.xml")
-            || STR_STARTS_WITH(part, "word/footnotes.xml")
-            || STR_STARTS_WITH(part, "word/endnotes.xml")
-            || STR_STARTS_WITH(part, "word/footer")
-            || STR_STARTS_WITH(part, "word/header")
-            // PowerPoint
-            || STR_STARTS_WITH(part, "ppt/slides/slide")
-            || STR_STARTS_WITH(part, "ppt/notesSlides/slide")
-            // Excel
-            || STR_STARTS_WITH(part, "xl/worksheets/sheet")
-            || STR_STARTS_WITH(part, "xl/sharedStrings.xml")
-            || STR_STARTS_WITH(part, "xl/workbook.xml")
-            ) {
-        return TRUE;
-    }
-
-    return FALSE;
-}
-
-int extract_text(xmlDoc *xml, xmlNode *node, text_buffer_t *buf) {
-    //TODO: Check which nodes are likely to have a 't' child, and ignore nodes that aren't
-    xmlErrorPtr err = xmlGetLastError();
-    if (err != NULL) {
-        if (err->level == XML_ERR_FATAL) {
-            LOG_ERRORF("doc.c", "Got fatal XML error while parsing document: %s", err->message)
-            return -1;
-        } else {
-            LOG_ERRORF("doc.c", "Got recoverable XML error while parsing document: %s", err->message)
-        }
-    }
-
-    for (xmlNode *child = node; child; child = child->next) {
-        if (*child->name == 't' && *(child->name + 1) == '\0') {
-            xmlChar *text = xmlNodeListGetString(xml, child->xmlChildrenNode, 1);
-
-            if (text) {
-                text_buffer_append_string0(buf, (char *) text);
-                text_buffer_append_char(buf, ' ');
-                xmlFree(text);
-            }
-        }
-
-        extract_text(xml, child->children, buf);
-    }
-}
-
-int xml_io_read(void *context, char *buffer, int len) {
-    struct archive *a = context;
-    return archive_read_data(a, buffer, len);
-}
-
-int xml_io_close(UNUSED(void *context)) {
-    //noop
-    return 0;
-}
-
-__always_inline
-static int read_part(struct archive *a, text_buffer_t *buf, document_t *doc) {
-
-    xmlDoc *xml = xmlReadIO(xml_io_read, xml_io_close, a, "/", NULL, XML_PARSE_RECOVER | XML_PARSE_NOWARNING | XML_PARSE_NOERROR | XML_PARSE_NONET);
-
-    if (xml == NULL) {
-        LOG_ERROR(doc->filepath, "Could not parse XML")
-        return -1;
-    }
-
-    xmlNode *root = xmlDocGetRootElement(xml);
-    if (root == NULL) {
-        LOG_ERROR(doc->filepath, "Empty document")
-        xmlFreeDoc(xml);
-        return -1;
-    }
-
-    extract_text(xml, root, buf);
-    xmlFreeDoc(xml);
-
-    return 0;
-}
-
-void parse_doc(void *mem, size_t mem_len, document_t *doc) {
-
-    if (mem == NULL) {
-        return;
-    }
-
-    struct archive *a = archive_read_new();
-    archive_read_support_format_zip(a);
-
-    int ret = archive_read_open_memory(a, mem, mem_len);
-    if (ret != ARCHIVE_OK) {
-        LOG_ERRORF(doc->filepath, "Could not read archive: %s", archive_error_string(a))
-        archive_read_free(a);
-        return;
-    }
-
-    text_buffer_t buf = text_buffer_create(ScanCtx.content_size);
-
-    struct archive_entry *entry;
-    while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
-        if (S_ISREG(archive_entry_stat(entry)->st_mode)) {
-            const char *path = archive_entry_pathname(entry);
-
-            if (should_read_part(path)) {
-                ret = read_part(a, &buf, doc);
-                if (ret != 0) {
-                    break;
-                }
-            }
-        }
-    }
-
-    if (buf.dyn_buffer.cur > 0) {
-        text_buffer_terminate_string(&buf);
-
-        meta_line_t *meta = malloc(sizeof(meta_line_t) + buf.dyn_buffer.cur);
-        meta->key = MetaContent;
-        strcpy(meta->strval, buf.dyn_buffer.buf);
-        APPEND_META(doc, meta)
-    }
-
-    archive_read_close(a);
-    archive_read_free(a);
-    text_buffer_destroy(&buf);
-}
--- a/src/parsing/doc.h
+++ b/src/parsing/doc.h
@@ -1,8 +0,0 @@
-#ifndef SIST2_DOC_H
-#define SIST2_DOC_H
-
-#include "src/sist.h"
-
-void parse_doc(void *buf, size_t buf_len, document_t *doc);
-
-#endif
--- a/src/parsing/font.c
+++ b/src/parsing/font.c
@@ -1,233 +0,0 @@
-#include "font.h"
-
-
-#include "src/ctx.h"
-
-__thread FT_Library ft_lib = NULL;
-
-
-typedef struct text_dimensions {
-    unsigned int width;
-    unsigned int height;
-    unsigned int baseline;
-} text_dimensions_t;
-
-typedef struct glyph {
-    int top;
-    int height;
-    int width;
-    int descent;
-    int ascent;
-    int advance_width;
-    unsigned char *pixmap;
-} glyph_t;
-
-
-__always_inline
-int kerning_offset(char c, char pc, FT_Face face) {
-    FT_Vector kerning;
-    FT_Get_Kerning(face, c, pc, FT_KERNING_DEFAULT, &kerning);
-
-    return (int) (kerning.x / 64);
-}
-
-__always_inline
-glyph_t ft_glyph_to_glyph(FT_GlyphSlot slot) {
-    glyph_t glyph;
-
-    glyph.pixmap = slot->bitmap.buffer;
-
-    glyph.width = (int) slot->bitmap.width;
-    glyph.height = (int) slot->bitmap.rows;
-    glyph.top = slot->bitmap_top;
-    glyph.advance_width = (int) slot->advance.x / 64;
-
-    glyph.descent = MAX(0, glyph.height - glyph.top);
-    glyph.ascent = MAX(0, MAX(glyph.top, glyph.height) - glyph.descent);
-
-    return glyph;
-}
-
-text_dimensions_t text_dimension(char *text, FT_Face face) {
-    text_dimensions_t dimensions;
-
-    dimensions.width = 0;
-
-    int num_chars = (int) strlen(text);
-
-    unsigned int max_ascent = 0;
-    int max_descent = 0;
-
-    char pc = 0;
-    for (int i = 0; i < num_chars; i++) {
-        char c = text[i];
-
-        FT_Load_Char(face, c, 0);
-        glyph_t glyph = ft_glyph_to_glyph(face->glyph);
-
-        max_descent = MAX(max_descent, glyph.descent);
-        max_ascent = MAX(max_ascent, MAX(glyph.height, glyph.ascent));
-
-        int kerning_x = kerning_offset(c, pc, face);
-        dimensions.width += MAX(glyph.advance_width, glyph.width) + kerning_x;
-
-        pc = c;
-    }
-
-    dimensions.height = max_ascent + max_descent;
-    dimensions.baseline = max_descent;
-
-    return dimensions;
-}
-
-void draw_glyph(glyph_t *glyph, int x, int y, struct text_dimensions text_info, unsigned char *bitmap) {
-    unsigned int src = 0;
-    unsigned int dst = y * text_info.width + x;
-    unsigned int row_offset = text_info.width - glyph->width;
-    unsigned int buf_len = text_info.width * text_info.height;
-
-    for (unsigned int sy = 0; sy < glyph->height; sy++) {
-        for (unsigned int sx = 0; sx < glyph->width; sx++) {
-            if (dst < buf_len) {
-                bitmap[dst] |= glyph->pixmap[src];
-            }
-            src++;
-            dst++;
-        }
-        dst += row_offset;
-    }
-}
-
-void bmp_format(dyn_buffer_t *buf, text_dimensions_t dimensions, const unsigned char *bitmap) {
-
-    dyn_buffer_write_short(buf, 0x4D42); // Magic
-    dyn_buffer_write_int(buf, 0); // Size placeholder
-    dyn_buffer_write_int(buf, 0x5157); //Reserved
-    dyn_buffer_write_int(buf, 14 + 40 + 256 * 4); // pixels offset
-
-    dyn_buffer_write_int(buf, 40); // DIB size
-    dyn_buffer_write_int(buf, (int) dimensions.width);
-    dyn_buffer_write_int(buf, (int) dimensions.height);
-    dyn_buffer_write_short(buf, 1); // Color planes
-    dyn_buffer_write_short(buf, 8); // bits per pixel
-    dyn_buffer_write_int(buf, 0); // compression
-    dyn_buffer_write_int(buf, 0); // Ignored
-    dyn_buffer_write_int(buf, 3800); // hres
-    dyn_buffer_write_int(buf, 3800); // vres
-    dyn_buffer_write_int(buf, 256); // Color count
-    dyn_buffer_write_int(buf, 0); // Ignored
-
-    // RGBA32 Color table (Grayscale)
-    for (int i = 255; i >= 0; i--) {
-        dyn_buffer_write_int(buf, i + (i << 8) + (i << 16));
-    }
-
-    // Pixel array: write from bottom to top, with rows padded to multiples of 4-bytes
-    for (int y = (int) dimensions.height - 1; y >= 0; y--) {
-        for (unsigned int x = 0; x < dimensions.width; x++) {
-            dyn_buffer_write_char(buf, (char) bitmap[y * dimensions.width + x]);
-        }
-        while (buf->cur % 4 != 0) {
-            dyn_buffer_write_char(buf, 0);
-        }
-    }
-
-    // Size
-    *(int *) ((char *) buf->buf + 2) = buf->cur;
-}
-
-void parse_font(const char *buf, size_t buf_len, document_t *doc) {
-    if (ft_lib == NULL) {
-        FT_Init_FreeType(&ft_lib);
-    }
-
-    if (buf == NULL) {
-        return;
-    }
-
-    FT_Face face;
-    FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face);
-    if (err != 0) {
-        LOG_ERRORF(doc->filepath, "(font.c) FT_New_Memory_Face() returned error code [%d] %s", err, ft_error_string(err));
-        return;
-    }
-
-    char font_name[1024];
-
-    if (face->style_name == NULL || *(face->style_name) == '?') {
-        if (face->family_name == NULL) {
-            strcpy(font_name, "(null)");
-        } else {
-            strcpy(font_name, face->family_name);
-        }
-    } else {
-        snprintf(font_name, sizeof(font_name), "%s %s", face->family_name, face->style_name);
-    }
-
-    meta_line_t *meta_name = malloc(sizeof(meta_line_t) + strlen(font_name));
-    meta_name->key = MetaFontName;
-    strcpy(meta_name->strval, font_name);
-    APPEND_META(doc, meta_name)
-
-    if (ScanCtx.tn_size <= 0) {
-        FT_Done_Face(face);
-        return;
-    }
-
-    int pixel = 64;
-    int num_chars = (int) strlen(font_name);
-
-    err = FT_Set_Pixel_Sizes(face, 0, pixel);
-    if (err != 0) {
-        LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err, ft_error_string(err))
-        FT_Done_Face(face);
-        return;
-    }
-
-    text_dimensions_t dimensions = text_dimension(font_name, face);
-    unsigned char *bitmap = calloc(dimensions.width * dimensions.height, 1);
-
-    FT_Vector pen;
-    pen.x = 0;
-
-    char pc = 0;
-    for (int i = 0; i < num_chars; i++) {
-        char c = font_name[i];
-
-        err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER);
-        if (err != 0) {
-            c = c >= 'a' && c <= 'z' ? c - 32 : c + 32;
-            err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER);
-            if (err != 0) {
-                LOG_WARNINGF(doc->filepath, "(font.c) FT_Load_Char() returned error code [%d] %s", err, ft_error_string(err));
-                continue;
-            }
-        }
-        glyph_t glyph = ft_glyph_to_glyph(face->glyph);
-
-        pen.x += kerning_offset(c, pc, face);
-        if (pen.x <= 0) {
-            pen.x = ABS(glyph.advance_width - glyph.width);
-        }
-        pen.y = dimensions.height - glyph.ascent - dimensions.baseline;
-
-        draw_glyph(&glyph, pen.x, pen.y, dimensions, bitmap);
-
-        pen.x += glyph.advance_width;
-        pc = c;
-    }
-
-    dyn_buffer_t bmp_data = dyn_buffer_create();
-    bmp_format(&bmp_data, dimensions, bitmap);
-
-    store_write(ScanCtx.index.store, (char *) doc->uuid, sizeof(doc->uuid), (char *) bmp_data.buf, bmp_data.cur);
-
-    dyn_buffer_destroy(&bmp_data);
-    free(bitmap);
-
-    FT_Done_Face(face);
-}
-
-void cleanup_font() {
-    FT_Done_FreeType(ft_lib);
-}
--- a/src/parsing/font.h
+++ b/src/parsing/font.h
@@ -1,10 +0,0 @@
-#ifndef SIST2_FONT_H
-#define SIST2_FONT_H
-
-#include "src/sist.h"
-
-
-void parse_font(const char * buf, size_t buf_len, document_t *doc);
-void cleanup_font();
-
-#endif
--- a/src/parsing/media.c
+++ b/src/parsing/media.c
@@ -1,402 +0,0 @@
-#include "src/sist.h"
-#include "src/ctx.h"
-
-#define MIN_SIZE 32
-#define AVIO_BUF_SIZE 8192
-
-__always_inline
-static AVCodecContext *alloc_jpeg_encoder(int dstW, int dstH, float qscale) {
-
-    AVCodec *jpeg_codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
-    AVCodecContext *jpeg = avcodec_alloc_context3(jpeg_codec);
-    jpeg->width = dstW;
-    jpeg->height = dstH;
-    jpeg->time_base.den = 1000000;
-    jpeg->time_base.num = 1;
-    jpeg->i_quant_factor = qscale;
-
-    jpeg->pix_fmt = AV_PIX_FMT_YUVJ420P;
-    int ret = avcodec_open2(jpeg, jpeg_codec, NULL);
-
-    if (ret != 0) {
-        printf("Could not open jpeg encoder: %s!\n", av_err2str(ret));
-        return NULL;
-    }
-
-    return jpeg;
-}
-
-__always_inline
-AVFrame *scale_frame(const AVCodecContext *decoder, const AVFrame *frame, int size) {
-
-    int dstW;
-    int dstH;
-    if (frame->width <= size && frame->height <= size) {
-        dstW = frame->width;
-        dstH = frame->height;
-    } else {
-        double ratio = (double) frame->width / frame->height;
-        if (frame->width > frame->height) {
-            dstW = size;
-            dstH = (int) (size / ratio);
-        } else {
-            dstW = (int) (size * ratio);
-            dstH = size;
-        }
-    }
-
-    if (dstW <= MIN_SIZE || dstH <= MIN_SIZE) {
-        return NULL;
-    }
-
-    AVFrame *scaled_frame = av_frame_alloc();
-
-    struct SwsContext *ctx = sws_getContext(
-            decoder->width, decoder->height, decoder->pix_fmt,
-            dstW, dstH, AV_PIX_FMT_YUVJ420P,
-            SWS_FAST_BILINEAR, 0, 0, 0
-    );
-
-    int dst_buf_len = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, dstW, dstH, 1);
-    uint8_t *dst_buf = (uint8_t *) av_malloc(dst_buf_len);
-
-    av_image_fill_arrays(scaled_frame->data, scaled_frame->linesize, dst_buf, AV_PIX_FMT_YUV420P, dstW, dstH, 1);
-
-    sws_scale(ctx,
-              (const uint8_t *const *) frame->data, frame->linesize,
-              0, decoder->height,
-              scaled_frame->data, scaled_frame->linesize
-    );
-
-    scaled_frame->width = dstW;
-    scaled_frame->height = dstH;
-    scaled_frame->format = AV_PIX_FMT_YUV420P;
-
-    sws_freeContext(ctx);
-
-    return scaled_frame;
-}
-
-__always_inline
-static AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx, document_t *doc) {
-    AVFrame *frame = av_frame_alloc();
-
-    AVPacket avPacket;
-    av_init_packet(&avPacket);
-
-    int receive_ret = -EAGAIN;
-    while (receive_ret == -EAGAIN) {
-        // Get video frame
-        while (1) {
-            int read_frame_ret = av_read_frame(pFormatCtx, &avPacket);
-
-            if (read_frame_ret != 0) {
-                if (read_frame_ret != AVERROR_EOF) {
-                    LOG_WARNINGF(doc->filepath,
-                                 "(media.c) avcodec_read_frame() returned error code [%d] %s",
-                                 read_frame_ret, av_err2str(read_frame_ret)
-                    )
-                }
-                av_frame_free(&frame);
-                av_packet_unref(&avPacket);
-                return NULL;
-            }
-
-            //Ignore audio/other frames
-            if (avPacket.stream_index != stream_idx) {
-                av_packet_unref(&avPacket);
-                continue;
-            }
-            break;
-        }
-
-        // Feed it to decoder
-        int decode_ret = avcodec_send_packet(decoder, &avPacket);
-        if (decode_ret != 0) {
-            LOG_ERRORF(doc->filepath,
-                         "(media.c) avcodec_send_packet() returned error code [%d] %s",
-                         decode_ret, av_err2str(decode_ret)
-            )
-            av_frame_free(&frame);
-            av_packet_unref(&avPacket);
-            return NULL;
-        }
-        av_packet_unref(&avPacket);
-        receive_ret = avcodec_receive_frame(decoder, frame);
-    }
-    return frame;
-}
-
-#define APPEND_TAG_META(doc, tag_, keyname) \
-    text_buffer_t tex = text_buffer_create(-1); \
-    text_buffer_append_string0(&tex, tag_->value); \
-    text_buffer_terminate_string(&tex); \
-    meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \
-    meta_tag->key = keyname; \
-    strcpy(meta_tag->strval, tex.dyn_buffer.buf); \
-    APPEND_META(doc, meta_tag) \
-    text_buffer_destroy(&tex);
-
-__always_inline
-static void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
-
-    AVDictionaryEntry *tag = NULL;
-    while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
-        char key[256];
-        strncpy(key, tag->key, sizeof(key));
-
-        char *ptr = key;
-        for (; *ptr; ++ptr) *ptr = (char) tolower(*ptr);
-
-        if (strcmp(key, "artist") == 0) {
-            APPEND_TAG_META(doc, tag, MetaArtist)
-        } else if (strcmp(key, "genre") == 0) {
-            APPEND_TAG_META(doc, tag, MetaGenre)
-        } else if (strcmp(key, "title") == 0) {
-            APPEND_TAG_META(doc, tag, MetaTitle)
-        } else if (strcmp(key, "album_artist") == 0) {
-            APPEND_TAG_META(doc, tag, MetaAlbumArtist)
-        } else if (strcmp(key, "album") == 0) {
-            APPEND_TAG_META(doc, tag, MetaAlbum)
-        }
-    }
-}
-
-__always_inline
-static void
-append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
-
-    if (is_video) {
-        meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
-        meta_duration->key = MetaMediaDuration;
-        meta_duration->longval = pFormatCtx->duration / AV_TIME_BASE;
-        APPEND_META(doc, meta_duration)
-
-        meta_line_t *meta_bitrate = malloc(sizeof(meta_line_t));
-        meta_bitrate->key = MetaMediaBitrate;
-        meta_bitrate->longval = pFormatCtx->bit_rate;
-        APPEND_META(doc, meta_bitrate)
-    }
-
-    AVDictionaryEntry *tag = NULL;
-    if (is_video) {
-        while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
-            if (include_audio_tags && strcmp(tag->key, "title") == 0) {
-                APPEND_TAG_META(doc, tag, MetaTitle)
-            } else if (strcmp(tag->key, "comment") == 0) {
-                APPEND_TAG_META(doc, tag, MetaContent)
-            } else if (include_audio_tags && strcmp(tag->key, "artist") == 0) {
-                APPEND_TAG_META(doc, tag, MetaArtist)
-            }
-        }
-    } else {
-        // EXIF metadata
-        while ((tag = av_dict_get(frame->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
-            if (include_audio_tags && strcmp(tag->key, "Artist") == 0) {
-                APPEND_TAG_META(doc, tag, MetaArtist)
-            } else if (strcmp(tag->key, "ImageDescription") == 0) {
-                APPEND_TAG_META(doc, tag, MetaContent)
-            } else if (strcmp(tag->key, "Make") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifMake)
-            } else if (strcmp(tag->key, "Model") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifModel)
-            } else if (strcmp(tag->key, "Software") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifSoftware)
-            } else if (strcmp(tag->key, "FNumber") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifFNumber)
-            } else if (strcmp(tag->key, "FocalLength") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifFocalLength)
-            } else if (strcmp(tag->key, "UserComment") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifUserComment)
-            } else if (strcmp(tag->key, "ISOSpeedRatings") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifIsoSpeedRatings)
-            } else if (strcmp(tag->key, "ExposureTime") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifExposureTime)
-            } else if (strcmp(tag->key, "DateTime") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifDateTime)
-            }
-        }
-    }
-}
-
-void parse_media(AVFormatContext *pFormatCtx, document_t *doc) {
-
-    int video_stream = -1;
-    int audio_stream = -1;
-
-    avformat_find_stream_info(pFormatCtx, NULL);
-
-    for (int i = (int) pFormatCtx->nb_streams - 1; i >= 0; i--) {
-        AVStream *stream = pFormatCtx->streams[i];
-
-        if (stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
-            if (audio_stream == -1) {
-                meta_line_t *meta_audio = malloc(sizeof(meta_line_t));
-                meta_audio->key = MetaMediaAudioCodec;
-                meta_audio->intval = stream->codecpar->codec_id;
-                APPEND_META(doc, meta_audio)
-
-                append_audio_meta(pFormatCtx, doc);
-                audio_stream = i;
-            }
-        } else if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
-
-            if (video_stream == -1) {
-                meta_line_t *meta_vid = malloc(sizeof(meta_line_t));
-                meta_vid->key = MetaMediaVideoCodec;
-                meta_vid->intval = stream->codecpar->codec_id;
-                APPEND_META(doc, meta_vid)
-
-                meta_line_t *meta_w = malloc(sizeof(meta_line_t));
-                meta_w->key = MetaWidth;
-                meta_w->intval = stream->codecpar->width;
-                APPEND_META(doc, meta_w)
-
-                meta_line_t *meta_h = malloc(sizeof(meta_line_t));
-                meta_h->key = MetaHeight;
-                meta_h->intval = stream->codecpar->height;
-                APPEND_META(doc, meta_h)
-
-                video_stream = i;
-            }
-        }
-    }
-
-    if (video_stream != -1 && ScanCtx.tn_size > 0) {
-        AVStream *stream = pFormatCtx->streams[video_stream];
-
-        if (stream->codecpar->width <= MIN_SIZE || stream->codecpar->height <= MIN_SIZE) {
-            avformat_close_input(&pFormatCtx);
-            avformat_free_context(pFormatCtx);
-            return;
-        }
-
-        // Decoder
-        AVCodec *video_codec = avcodec_find_decoder(stream->codecpar->codec_id);
-        AVCodecContext *decoder = avcodec_alloc_context3(video_codec);
-        avcodec_parameters_to_context(decoder, stream->codecpar);
-        avcodec_open2(decoder, video_codec, NULL);
-
-        //Seek
-        if (stream->nb_frames > 1 && stream->codecpar->codec_id != AV_CODEC_ID_GIF) {
-            int seek_ret = 0;
-            for (int i = 20; i >= 0; i--) {
-                seek_ret = av_seek_frame(pFormatCtx, video_stream,
-                                         stream->duration * 0.10, 0);
-                if (seek_ret == 0) {
-                    break;
-                }
-            }
-        }
-
-        AVFrame *frame = read_frame(pFormatCtx, decoder, video_stream, doc);
-        if (frame == NULL) {
-            avcodec_free_context(&decoder);
-            avformat_close_input(&pFormatCtx);
-            avformat_free_context(pFormatCtx);
-            return;
-        }
-
-        append_video_meta(pFormatCtx, frame, doc, audio_stream == -1, stream->nb_frames > 1);
-
-        // Scale frame
-        AVFrame *scaled_frame = scale_frame(decoder, frame, ScanCtx.tn_size);
-
-        if (scaled_frame == NULL) {
-            av_frame_free(&frame);
-            avcodec_free_context(&decoder);
-            avformat_close_input(&pFormatCtx);
-            avformat_free_context(pFormatCtx);
-            return;
-        }
-
-        // Encode frame to jpeg
-        AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height, ScanCtx.tn_qscale);
-        avcodec_send_frame(jpeg_encoder, scaled_frame);
-
-        AVPacket jpeg_packet;
-        av_init_packet(&jpeg_packet);
-        avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
-
-        // Save thumbnail
-        store_write(ScanCtx.index.store, (char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data,
-                    jpeg_packet.size);
-
-        av_packet_unref(&jpeg_packet);
-        av_frame_free(&frame);
-        av_free(*scaled_frame->data);
-        av_frame_free(&scaled_frame);
-        avcodec_free_context(&jpeg_encoder);
-        avcodec_free_context(&decoder);
-    }
-
-    avformat_close_input(&pFormatCtx);
-    avformat_free_context(pFormatCtx);
-}
-
-void parse_media_filename(const char *filepath, document_t *doc) {
-
-    AVFormatContext *pFormatCtx = avformat_alloc_context();
-    if (pFormatCtx == NULL) {
-        LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
-        return;
-    }
-    int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
-    if (res < 0) {
-        LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
-        avformat_close_input(&pFormatCtx);
-        avformat_free_context(pFormatCtx);
-        return;
-    }
-
-    parse_media(pFormatCtx, doc);
-}
-
-
-int vfile_read(void *ptr, uint8_t *buf, int buf_size) {
-    struct vfile *f = ptr;
-
-    int ret = f->read(f, buf, buf_size);
-
-    if (ret == 0) {
-        return AVERROR_EOF;
-    }
-    return ret;
-}
-
-void parse_media_vfile(struct vfile *f, document_t *doc) {
-
-    AVFormatContext *pFormatCtx = avformat_alloc_context();
-    if (pFormatCtx == NULL) {
-        LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
-        return;
-    }
-
-    unsigned char *buffer = (unsigned char *) av_malloc(AVIO_BUF_SIZE);
-    AVIOContext *io_ctx = avio_alloc_context(buffer, AVIO_BUF_SIZE, 0, f, vfile_read, NULL, NULL);
-
-    pFormatCtx->pb = io_ctx;
-    pFormatCtx->flags |= AVFMT_FLAG_CUSTOM_IO;
-
-    int res = avformat_open_input(&pFormatCtx, "", NULL, NULL);
-    if (res == -5) {
-        // Tried to parse media that requires seek
-        av_free(io_ctx->buffer);
-        avio_context_free(&io_ctx);
-        avformat_close_input(&pFormatCtx);
-        avformat_free_context(pFormatCtx);
-        return;
-    } else if (res < 0) {
-        LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
-        av_free(io_ctx->buffer);
-        avio_context_free(&io_ctx);
-        avformat_close_input(&pFormatCtx);
-        avformat_free_context(pFormatCtx);
-        return;
-    }
-
-    parse_media(pFormatCtx, doc);
-    av_free(io_ctx->buffer);
-    avio_context_free(&io_ctx);
-}
-
--- a/src/parsing/media.h
+++ b/src/parsing/media.h
@@ -1,14 +0,0 @@
-#ifndef SIST2_MEDIA_H
-#define SIST2_MEDIA_H
-
-
-#include "src/sist.h"
-
-#define MIN_VIDEO_SIZE 1024 * 64
-#define MIN_IMAGE_SIZE 1024 * 2
-
-void parse_media_filename(const char * filepath, document_t *doc);
-
-void parse_media_vfile(struct vfile *f, document_t *doc);
-
-#endif
--- a/src/parsing/mime.h
+++ b/src/parsing/mime.h
@@ -1,9 +1,9 @@
 #ifndef SIST2_MIME_H
 #define SIST2_MIME_H

-#include "src/sist.h"
+#include "../sist.h"

-#define MAJOR_MIME(mime_id) (mime_id & 0x0FFF0000) >> 16
+#define MAJOR_MIME(mime_id) (mime_id & 0x000F0000) >> 16

 #define MIME_EMPTY 1

@@ -25,6 +25,15 @@
 #define DOC_MASK 0x04000000
 #define IS_DOC(mime_id) (mime_id & DOC_MASK) == DOC_MASK

+#define MOBI_MASK 0x02000000
+#define IS_MOBI(mime_id) (mime_id & MOBI_MASK) == MOBI_MASK
+
+#define MARKUP_MASK 0x01000000
+#define IS_MARKUP(mime_id) (mime_id & MARKUP_MASK) == MARKUP_MASK
+
+#define RAW_MASK 0x00800000
+#define IS_RAW(mime_id) (mime_id & RAW_MASK) == RAW_MASK
+
 enum major_mime {
    MimeInvalid = 0,
    MimeModel = 1,
--- a/src/parsing/mime_generated.c
+++ b/src/parsing/mime_generated.c
@@ -1,7 +1,7 @@
 // **Generated by mime.py**
 #ifndef MIME_GENERATED_C
 #define MIME_GENERATED_C
-#include <glib-2.0/glib.h>
+#include <glib.h>

 #include <stdlib.h>

@@ -53,386 +53,408 @@ enum mime {
    application_step=655405,
    application_streamingmedia=655406,
    application_vda=655407,
-    application_vnd_fdf=655408,
-    application_vnd_font_fontforge_sfd=655409,
-    application_vnd_hp_hpgl=655410,
-    application_vnd_iccprofile=655411,
-    application_vnd_lotus_1_2_3=655412,
-    application_vnd_ms_cab_compressed=655413,
-    application_vnd_ms_excel=655414,
-    application_vnd_ms_fontobject=655415,
-    application_vnd_ms_opentype=655416 | 0x20000000,
-    application_vnd_ms_pki_certstore=655417,
-    application_vnd_ms_pki_pko=655418,
-    application_vnd_ms_pki_seccat=655419,
-    application_vnd_ms_powerpoint=655420,
-    application_vnd_ms_project=655421,
-    application_vnd_oasis_opendocument_base=655422,
-    application_vnd_oasis_opendocument_formula=655423,
-    application_vnd_oasis_opendocument_graphics=655424,
-    application_vnd_oasis_opendocument_presentation=655425,
-    application_vnd_oasis_opendocument_spreadsheet=655426,
-    application_vnd_oasis_opendocument_text=655427,
-    application_vnd_openxmlformats_officedocument_presentationml_presentation=655428 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655429 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655430 | 0x04000000,
-    application_vnd_symbian_install=655431,
-    application_vnd_tcpdump_pcap=655432,
-    application_vnd_wap_wmlc=655433,
-    application_vnd_wap_wmlscriptc=655434,
-    application_vnd_xara=655435,
-    application_vocaltec_media_desc=655436,
-    application_vocaltec_media_file=655437,
-    application_warc=655438,
-    application_winhelp=655439,
-    application_wordperfect=655440,
-    application_wordperfect6_0=655441,
-    application_wordperfect6_1=655442,
-    application_x_123=655443,
-    application_x_7z_compressed=655444 | 0x10000000,
-    application_x_aim=655445,
-    application_x_apple_diskimage=655446,
-    application_x_arc=655447 | 0x10000000,
-    application_x_archive=655448,
-    application_x_atari_7800_rom=655449,
-    application_x_authorware_bin=655450,
-    application_x_authorware_map=655451,
-    application_x_authorware_seg=655452,
-    application_x_avira_qua=655453,
-    application_x_bcpio=655454,
-    application_x_bittorrent=655455,
-    application_x_bsh=655456,
-    application_x_bytecode_python=655457,
-    application_x_bzip=655458,
-    application_x_bzip2=655459 | 0x08000000,
-    application_x_cbr=655460,
-    application_x_cbz=655461 | 0x40000000,
-    application_x_cdlink=655462,
-    application_x_chat=655463,
-    application_x_chrome_extension=655464,
-    application_x_cocoa=655465,
-    application_x_conference=655466,
-    application_x_coredump=655467,
-    application_x_cpio=655468,
-    application_x_dbf=655469,
-    application_x_dbt=655470,
-    application_x_debian_package=655471,
-    application_x_deepv=655472,
-    application_x_director=655473,
-    application_x_dmp=655474,
-    application_x_dosdriver=655475,
-    application_x_dosexec=655476,
-    application_x_dvi=655477,
-    application_x_elc=655478,
+    application_vnd_amazon_mobi8_ebook=655408 | 0x02000000,
+    application_vnd_coffeescript=655409,
+    application_vnd_fdf=655410,
+    application_vnd_font_fontforge_sfd=655411,
+    application_vnd_hp_hpgl=655412,
+    application_vnd_iccprofile=655413,
+    application_vnd_lotus_1_2_3=655414,
+    application_vnd_ms_cab_compressed=655415,
+    application_vnd_ms_excel=655416,
+    application_vnd_ms_fontobject=655417,
+    application_vnd_ms_opentype=655418 | 0x20000000,
+    application_vnd_ms_outlook=655419,
+    application_vnd_ms_pki_certstore=655420,
+    application_vnd_ms_pki_pko=655421,
+    application_vnd_ms_pki_seccat=655422,
+    application_vnd_ms_powerpoint=655423,
+    application_vnd_ms_project=655424,
+    application_vnd_oasis_opendocument_base=655425,
+    application_vnd_oasis_opendocument_formula=655426,
+    application_vnd_oasis_opendocument_graphics=655427,
+    application_vnd_oasis_opendocument_presentation=655428,
+    application_vnd_oasis_opendocument_spreadsheet=655429,
+    application_vnd_oasis_opendocument_text=655430,
+    application_vnd_openxmlformats_officedocument_presentationml_presentation=655431 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655432 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655433 | 0x04000000,
+    application_vnd_symbian_install=655434,
+    application_vnd_tcpdump_pcap=655435,
+    application_vnd_wap_wmlc=655436,
+    application_vnd_wap_wmlscriptc=655437,
+    application_vnd_xara=655438,
+    application_vocaltec_media_desc=655439,
+    application_vocaltec_media_file=655440,
+    application_warc=655441,
+    application_winhelp=655442,
+    application_wordperfect=655443,
+    application_wordperfect6_0=655444,
+    application_wordperfect6_1=655445,
+    application_x_123=655446,
+    application_x_7z_compressed=655447 | 0x10000000,
+    application_x_aim=655448,
+    application_x_apple_diskimage=655449,
+    application_x_arc=655450 | 0x10000000,
+    application_x_archive=655451,
+    application_x_atari_7800_rom=655452,
+    application_x_authorware_bin=655453,
+    application_x_authorware_map=655454,
+    application_x_authorware_seg=655455,
+    application_x_avira_qua=655456,
+    application_x_bcpio=655457,
+    application_x_bittorrent=655458,
+    application_x_bsh=655459,
+    application_x_bytecode_python=655460,
+    application_x_bzip=655461,
+    application_x_bzip2=655462 | 0x08000000,
+    application_x_cbr=655463,
+    application_x_cbz=655464,
+    application_x_cdlink=655465,
+    application_x_chat=655466,
+    application_x_chrome_extension=655467,
+    application_x_cocoa=655468,
+    application_x_conference=655469,
+    application_x_coredump=655470,
+    application_x_cpio=655471,
+    application_x_dbf=655472,
+    application_x_dbt=655473,
+    application_x_debian_package=655474,
+    application_x_deepv=655475,
+    application_x_director=655476,
+    application_x_dmp=655477,
+    application_x_dosdriver=655478,
+    application_x_dosexec=655479,
+    application_x_dvi=655480,
+    application_x_elc=655481,
    application_x_empty=1,
-    application_x_envoy=655480,
-    application_x_esrehber=655481,
-    application_x_excel=655482,
-    application_x_executable=655483,
-    application_x_font_gdos=655484,
-    application_x_font_pf2=655485,
-    application_x_font_pfm=655486,
-    application_x_font_sfn=655487,
-    application_x_font_ttf=655488 | 0x20000000,
-    application_x_fptapplication_x_dbt=655489,
-    application_x_freelance=655490,
-    application_x_gamecube_rom=655491,
-    application_x_gdbm=655492,
-    application_x_gettext_translation=655493,
-    application_x_git=655494,
-    application_x_gsp=655495,
-    application_x_gss=655496,
-    application_x_gtar=655497,
-    application_x_gzip=655498,
-    application_x_hdf=655499,
-    application_x_helpfile=655500,
-    application_x_httpd_imap=655501,
-    application_x_ima=655502,
-    application_x_innosetup=655503,
-    application_x_internett_signup=655504,
-    application_x_inventor=655505,
-    application_x_ip2=655506,
-    application_x_java_applet=655507,
-    application_x_java_commerce=655508,
-    application_x_java_image=655509,
-    application_x_java_jmod=655510,
-    application_x_java_keystore=655511,
-    application_x_kdelnk=655512,
-    application_x_koan=655513,
-    application_x_latex=655514,
-    application_x_livescreen=655515,
-    application_x_lotus=655516,
-    application_x_lz4=655517 | 0x08000000,
-    application_x_lz4_json=655518,
-    application_x_lzh=655519,
-    application_x_lzh_compressed=655520,
-    application_x_lzip=655521 | 0x08000000,
-    application_x_lzma=655522 | 0x08000000,
-    application_x_lzop=655523 | 0x08000000,
-    application_x_lzx=655524,
-    application_x_mach_binary=655525,
-    application_x_mach_executable=655526,
-    application_x_magic_cap_package_1_0=655527,
-    application_x_mathcad=655528,
-    application_x_maxis_dbpf=655529,
-    application_x_meme=655530,
-    application_x_midi=655531,
-    application_x_mif=655532,
-    application_x_mix_transfer=655533,
-    application_x_mobipocket_ebook=655534,
-    application_x_ms_compress_szdd=655535,
-    application_x_ms_pdb=655536,
-    application_x_ms_reader=655537,
-    application_x_msaccess=655538,
-    application_x_n64_rom=655539,
-    application_x_navi_animation=655540,
-    application_x_navidoc=655541,
-    application_x_navimap=655542,
-    application_x_navistyle=655543,
-    application_x_nes_rom=655544,
-    application_x_netcdf=655545,
-    application_x_newton_compatible_pkg=655546,
-    application_x_nintendo_ds_rom=655547,
-    application_x_object=655548,
-    application_x_omc=655549,
-    application_x_omcdatamaker=655550,
-    application_x_omcregerator=655551,
-    application_x_pagemaker=655552,
-    application_x_pcl=655553,
-    application_x_pgp_keyring=655554,
-    application_x_pixclscript=655555,
-    application_x_pkcs7_certreqresp=655556,
-    application_x_pkcs7_signature=655557,
-    application_x_project=655558,
-    application_x_qpro=655559,
-    application_x_rar=655560 | 0x10000000,
-    application_x_rpm=655561,
-    application_x_sdp=655562,
-    application_x_sea=655563,
-    application_x_seelogo=655564,
-    application_x_setupscript=655565,
-    application_x_shar=655566,
-    application_x_sharedlib=655567,
-    application_x_shockwave_flash=655568,
-    application_x_snappy_framed=655569,
-    application_x_sprite=655570,
-    application_x_sqlite3=655571,
-    application_x_stargallery_thm=655572,
-    application_x_stuffit=655573,
-    application_x_sv4cpio=655574,
-    application_x_sv4crc=655575,
-    application_x_tar=655576 | 0x10000000,
-    application_x_tbook=655577,
-    application_x_terminfo=655578,
-    application_x_terminfo2=655579,
-    application_x_tex_tfm=655580,
-    application_x_texinfo=655581,
-    application_x_ustar=655582,
-    application_x_visio=655583,
-    application_x_vnd_audioexplosion_mzz=655584,
-    application_x_vnd_ls_xpix=655585,
-    application_x_vrml=655586,
-    application_x_wais_source=655587,
-    application_x_wine_extension_ini=655588,
-    application_x_wintalk=655589,
-    application_x_world=655590,
-    application_x_wri=655591,
-    application_x_x509_ca_cert=655592,
-    application_x_xz=655593 | 0x08000000,
-    application_x_zip=655594,
-    application_x_zstd=655595 | 0x08000000,
-    application_xml=655596,
-    application_zip=655597 | 0x10000000,
-    application_zlib=655598,
-    audio_basic=458991 | 0x80000000,
-    audio_it=458992,
-    audio_make=458993,
-    audio_mid=458994,
-    audio_midi=458995,
-    audio_mp4=458996,
-    audio_mpeg=458997,
-    audio_ogg=458998,
-    audio_s3m=458999,
-    audio_tsp_audio=459000,
-    audio_tsplayer=459001,
-    audio_vnd_qcelp=459002,
-    audio_voxware=459003,
-    audio_x_aiff=459004,
-    audio_x_flac=459005,
-    audio_x_gsm=459006,
-    audio_x_hx_aac_adts=459007,
-    audio_x_jam=459008,
-    audio_x_liveaudio=459009,
-    audio_x_m4a=459010,
-    audio_x_midi=459011,
-    audio_x_mod=459012,
-    audio_x_mp4a_latm=459013,
-    audio_x_mpeg_3=459014,
-    audio_x_mpequrl=459015,
-    audio_x_nspaudio=459016,
-    audio_x_pn_realaudio=459017,
-    audio_x_psid=459018,
-    audio_x_realaudio=459019,
-    audio_x_s3m=459020,
-    audio_x_twinvq=459021,
-    audio_x_twinvq_plugin=459022,
-    audio_x_voc=459023,
-    audio_x_wav=459024,
-    audio_x_xbox_executable=459025 | 0x80000000,
-    audio_x_xbox360_executable=459026 | 0x80000000,
-    audio_xm=459027,
-    font_otf=327956 | 0x20000000,
-    font_sfnt=327957 | 0x20000000,
-    font_woff=327958 | 0x20000000,
-    font_woff2=327959 | 0x20000000,
-    image_bmp=524568,
-    image_cmu_raster=524569,
-    image_fif=524570,
-    image_florian=524571,
-    image_g3fax=524572,
-    image_gif=524573,
-    image_heic=524574,
-    image_ief=524575,
-    image_jpeg=524576,
-    image_jutvision=524577,
-    image_naplps=524578,
-    image_pict=524579,
-    image_png=524580,
-    image_svg=524581 | 0x80000000,
-    image_svg_xml=524582 | 0x80000000,
-    image_tiff=524583,
-    image_vnd_adobe_photoshop=524584 | 0x80000000,
-    image_vnd_djvu=524585 | 0x80000000,
-    image_vnd_fpx=524586,
-    image_vnd_microsoft_icon=524587,
-    image_vnd_rn_realflash=524588,
-    image_vnd_rn_realpix=524589,
-    image_vnd_wap_wbmp=524590,
-    image_vnd_xiff=524591,
-    image_webp=524592,
-    image_wmf=524593,
-    image_x_3ds=524594,
-    image_x_award_bioslogo=524595,
-    image_x_cmu_raster=524596,
-    image_x_cur=524597,
-    image_x_dwg=524598,
-    image_x_eps=524599,
-    image_x_exr=524600,
-    image_x_gem=524601,
-    image_x_icns=524602,
-    image_x_icon=524603 | 0x80000000,
-    image_x_jg=524604,
-    image_x_jps=524605,
-    image_x_ms_bmp=524606,
-    image_x_niff=524607,
-    image_x_pcx=524608,
-    image_x_pict=524609,
-    image_x_portable_bitmap=524610,
-    image_x_portable_graymap=524611,
-    image_x_portable_pixmap=524612,
-    image_x_quicktime=524613,
-    image_x_rgb=524614,
-    image_x_tga=524615,
-    image_x_tiff=524616,
-    image_x_win_bitmap=524617,
-    image_x_xcf=524618 | 0x80000000,
-    image_x_xpixmap=524619 | 0x80000000,
-    image_x_xwindowdump=524620,
-    message_news=196941,
-    message_rfc822=196942,
-    model_vnd_dwf=65871,
-    model_vnd_gdl=65872,
-    model_vnd_gs_gdl=65873,
-    model_vrml=65874,
-    model_x_pov=65875,
-    text_PGP=590164,
-    text_asp=590165,
-    text_css=590166,
-    text_html=590167,
-    text_javascript=590168,
-    text_mcf=590169,
-    text_pascal=590170,
-    text_plain=590171,
-    text_richtext=590172,
-    text_rtf=590173,
-    text_scriplet=590174,
-    text_tab_separated_values=590175,
-    text_troff=590176,
-    text_uri_list=590177,
-    text_vnd_abc=590178,
-    text_vnd_fmi_flexstor=590179,
-    text_vnd_wap_wml=590180,
-    text_vnd_wap_wmlscript=590181,
-    text_webviewhtml=590182,
-    text_x_Algol68=590183,
-    text_x_asm=590184,
-    text_x_audiosoft_intra=590185,
-    text_x_awk=590186,
-    text_x_bcpl=590187,
-    text_x_c=590188,
-    text_x_c__=590189,
-    text_x_component=590190,
-    text_x_diff=590191,
-    text_x_fortran=590192,
-    text_x_java=590193,
-    text_x_la_asf=590194,
-    text_x_lisp=590195,
-    text_x_m=590196,
-    text_x_m4=590197,
-    text_x_makefile=590198,
-    text_x_ms_regedit=590199,
-    text_x_msdos_batch=590200,
-    text_x_objective_c=590201,
-    text_x_pascal=590202,
-    text_x_perl=590203,
-    text_x_php=590204,
-    text_x_po=590205,
-    text_x_python=590206,
-    text_x_ruby=590207,
-    text_x_sass=590208,
-    text_x_scss=590209,
-    text_x_server_parsed_html=590210,
-    text_x_setext=590211,
-    text_x_sgml=590212,
-    text_x_shellscript=590213,
-    text_x_speech=590214,
-    text_x_tcl=590215,
-    text_x_tex=590216,
-    text_x_uil=590217,
-    text_x_uuencode=590218,
-    text_x_vcalendar=590219,
-    text_x_vcard=590220,
-    text_xml=590221,
-    video_MP2T=393614,
-    video_animaflex=393615,
-    video_avi=393616,
-    video_avs_video=393617,
-    video_mp4=393618,
-    video_mpeg=393619,
-    video_quicktime=393620,
-    video_vdo=393621,
-    video_vivo=393622,
-    video_vnd_rn_realvideo=393623,
-    video_vosaic=393624,
-    video_webm=393625,
-    video_x_amt_demorun=393626,
-    video_x_amt_showrun=393627,
-    video_x_atomic3d_feature=393628,
-    video_x_dl=393629,
-    video_x_dv=393630,
-    video_x_fli=393631,
-    video_x_flv=393632,
-    video_x_isvideo=393633,
-    video_x_jng=393634 | 0x80000000,
-    video_x_m4v=393635,
-    video_x_matroska=393636,
-    video_x_mng=393637,
-    video_x_motion_jpeg=393638,
-    video_x_ms_asf=393639,
-    video_x_msvideo=393640,
-    video_x_qtc=393641,
-    video_x_sgi_movie=393642,
-    x_epoc_x_sisx_app=721323,
+    application_x_envoy=655483,
+    application_x_esrehber=655484,
+    application_x_excel=655485,
+    application_x_executable=655486,
+    application_x_font_gdos=655487,
+    application_x_font_pf2=655488,
+    application_x_font_pfm=655489,
+    application_x_font_sfn=655490,
+    application_x_font_ttf=655491 | 0x20000000,
+    application_x_fptapplication_x_dbt=655492,
+    application_x_freelance=655493,
+    application_x_gamecube_rom=655494,
+    application_x_gdbm=655495,
+    application_x_gettext_translation=655496,
+    application_x_git=655497,
+    application_x_gsp=655498,
+    application_x_gss=655499,
+    application_x_gtar=655500,
+    application_x_gzip=655501,
+    application_x_hdf=655502,
+    application_x_helpfile=655503,
+    application_x_httpd_imap=655504,
+    application_x_ima=655505,
+    application_x_innosetup=655506,
+    application_x_internett_signup=655507,
+    application_x_inventor=655508,
+    application_x_ip2=655509,
+    application_x_java_applet=655510,
+    application_x_java_commerce=655511,
+    application_x_java_image=655512,
+    application_x_java_jmod=655513,
+    application_x_java_keystore=655514,
+    application_x_kdelnk=655515,
+    application_x_koan=655516,
+    application_x_latex=655517,
+    application_x_livescreen=655518,
+    application_x_lotus=655519,
+    application_x_lz4=655520 | 0x08000000,
+    application_x_lz4_json=655521,
+    application_x_lzh=655522,
+    application_x_lzh_compressed=655523,
+    application_x_lzip=655524 | 0x08000000,
+    application_x_lzma=655525 | 0x08000000,
+    application_x_lzop=655526 | 0x08000000,
+    application_x_lzx=655527,
+    application_x_mach_binary=655528,
+    application_x_mach_executable=655529,
+    application_x_magic_cap_package_1_0=655530,
+    application_x_mathcad=655531,
+    application_x_maxis_dbpf=655532,
+    application_x_meme=655533,
+    application_x_midi=655534,
+    application_x_mif=655535,
+    application_x_mix_transfer=655536,
+    application_x_mobipocket_ebook=655537 | 0x02000000,
+    application_x_ms_compress_szdd=655538,
+    application_x_ms_pdb=655539,
+    application_x_ms_reader=655540,
+    application_x_msaccess=655541,
+    application_x_n64_rom=655542,
+    application_x_navi_animation=655543,
+    application_x_navidoc=655544,
+    application_x_navimap=655545,
+    application_x_navistyle=655546,
+    application_x_nes_rom=655547,
+    application_x_netcdf=655548,
+    application_x_newton_compatible_pkg=655549,
+    application_x_nintendo_ds_rom=655550,
+    application_x_object=655551,
+    application_x_omc=655552,
+    application_x_omcdatamaker=655553,
+    application_x_omcregerator=655554,
+    application_x_pagemaker=655555,
+    application_x_pcl=655556,
+    application_x_pgp_keyring=655557,
+    application_x_pixclscript=655558,
+    application_x_pkcs7_certreqresp=655559,
+    application_x_pkcs7_signature=655560,
+    application_x_project=655561,
+    application_x_qpro=655562,
+    application_x_rar=655563 | 0x10000000,
+    application_x_rpm=655564,
+    application_x_sdp=655565,
+    application_x_sea=655566,
+    application_x_seelogo=655567,
+    application_x_setupscript=655568,
+    application_x_shar=655569,
+    application_x_sharedlib=655570,
+    application_x_shockwave_flash=655571,
+    application_x_snappy_framed=655572,
+    application_x_sprite=655573,
+    application_x_sqlite3=655574,
+    application_x_stargallery_thm=655575,
+    application_x_stuffit=655576,
+    application_x_sv4cpio=655577,
+    application_x_sv4crc=655578,
+    application_x_tar=655579 | 0x10000000,
+    application_x_tbook=655580,
+    application_x_terminfo=655581,
+    application_x_terminfo2=655582,
+    application_x_tex_tfm=655583,
+    application_x_texinfo=655584,
+    application_x_ustar=655585,
+    application_x_visio=655586,
+    application_x_vnd_audioexplosion_mzz=655587,
+    application_x_vnd_ls_xpix=655588,
+    application_x_vrml=655589,
+    application_x_wais_source=655590,
+    application_x_wine_extension_ini=655591,
+    application_x_wintalk=655592,
+    application_x_world=655593,
+    application_x_wri=655594,
+    application_x_x509_ca_cert=655595,
+    application_x_xz=655596 | 0x08000000,
+    application_x_zip=655597,
+    application_x_zstd=655598 | 0x08000000,
+    application_x_zstd_dictionary=655599,
+    application_xml=655600,
+    application_zip=655601 | 0x10000000,
+    application_zlib=655602,
+    audio_basic=458995 | 0x80000000,
+    audio_it=458996,
+    audio_make=458997,
+    audio_mid=458998,
+    audio_midi=458999,
+    audio_mp4=459000,
+    audio_mpeg=459001,
+    audio_ogg=459002,
+    audio_s3m=459003,
+    audio_tsp_audio=459004,
+    audio_tsplayer=459005,
+    audio_vnd_qcelp=459006,
+    audio_voxware=459007,
+    audio_x_aiff=459008,
+    audio_x_flac=459009,
+    audio_x_gsm=459010,
+    audio_x_hx_aac_adts=459011,
+    audio_x_jam=459012,
+    audio_x_liveaudio=459013,
+    audio_x_m4a=459014,
+    audio_x_midi=459015,
+    audio_x_mod=459016,
+    audio_x_mp4a_latm=459017,
+    audio_x_mpeg_3=459018,
+    audio_x_mpequrl=459019,
+    audio_x_nspaudio=459020,
+    audio_x_pn_realaudio=459021,
+    audio_x_psid=459022,
+    audio_x_realaudio=459023,
+    audio_x_s3m=459024,
+    audio_x_twinvq=459025,
+    audio_x_twinvq_plugin=459026,
+    audio_x_voc=459027,
+    audio_x_wav=459028,
+    audio_x_xbox_executable=459029 | 0x80000000,
+    audio_x_xbox360_executable=459030 | 0x80000000,
+    audio_xm=459031,
+    font_otf=327960 | 0x20000000,
+    font_sfnt=327961 | 0x20000000,
+    font_woff=327962 | 0x20000000,
+    font_woff2=327963 | 0x20000000,
+    image_bmp=524572,
+    image_cmu_raster=524573,
+    image_fif=524574,
+    image_florian=524575,
+    image_g3fax=524576,
+    image_gif=524577,
+    image_heic=524578,
+    image_ief=524579,
+    image_jpeg=524580,
+    image_jutvision=524581,
+    image_naplps=524582,
+    image_pict=524583,
+    image_png=524584,
+    image_svg=524585 | 0x80000000,
+    image_svg_xml=524586 | 0x80000000,
+    image_tiff=524587,
+    image_vnd_adobe_photoshop=524588 | 0x80000000,
+    image_vnd_djvu=524589 | 0x80000000,
+    image_vnd_fpx=524590,
+    image_vnd_microsoft_icon=524591,
+    image_vnd_rn_realflash=524592,
+    image_vnd_rn_realpix=524593,
+    image_vnd_wap_wbmp=524594,
+    image_vnd_xiff=524595,
+    image_webp=524596,
+    image_wmf=524597,
+    image_x_3ds=524598,
+    image_x_adobe_dng=524599 | 0x00800000,
+    image_x_award_bioslogo=524600,
+    image_x_canon_cr2=524601 | 0x00800000,
+    image_x_canon_crw=524602 | 0x00800000,
+    image_x_cmu_raster=524603,
+    image_x_cur=524604,
+    image_x_dcraw=524605 | 0x00800000,
+    image_x_dwg=524606,
+    image_x_eps=524607,
+    image_x_epson_erf=524608 | 0x00800000,
+    image_x_exr=524609,
+    image_x_fuji_raf=524610 | 0x00800000,
+    image_x_gem=524611,
+    image_x_icns=524612,
+    image_x_icon=524613 | 0x80000000,
+    image_x_jg=524614,
+    image_x_jps=524615,
+    image_x_kodak_dcr=524616 | 0x00800000,
+    image_x_kodak_k25=524617 | 0x00800000,
+    image_x_kodak_kdc=524618 | 0x00800000,
+    image_x_minolta_mrw=524619 | 0x00800000,
+    image_x_ms_bmp=524620,
+    image_x_niff=524621,
+    image_x_nikon_nef=524622 | 0x00800000,
+    image_x_olympus_orf=524623 | 0x00800000,
+    image_x_panasonic_raw=524624 | 0x00800000,
+    image_x_pcx=524625,
+    image_x_pentax_pef=524626 | 0x00800000,
+    image_x_pict=524627,
+    image_x_portable_bitmap=524628,
+    image_x_portable_graymap=524629,
+    image_x_portable_pixmap=524630,
+    image_x_quicktime=524631,
+    image_x_rgb=524632,
+    image_x_sigma_x3f=524633 | 0x00800000,
+    image_x_sony_arw=524634 | 0x00800000,
+    image_x_sony_sr2=524635 | 0x00800000,
+    image_x_sony_srf=524636 | 0x00800000,
+    image_x_tga=524637,
+    image_x_tiff=524638,
+    image_x_win_bitmap=524639,
+    image_x_xcf=524640 | 0x80000000,
+    image_x_xpixmap=524641 | 0x80000000,
+    image_x_xwindowdump=524642,
+    message_news=196963,
+    message_rfc822=196964,
+    model_vnd_dwf=65893,
+    model_vnd_gdl=65894,
+    model_vnd_gs_gdl=65895,
+    model_vrml=65896,
+    model_x_pov=65897,
+    text_PGP=590186,
+    text_asp=590187,
+    text_css=590188,
+    text_html=590189 | 0x01000000,
+    text_javascript=590190,
+    text_mcf=590191,
+    text_pascal=590192,
+    text_plain=590193,
+    text_richtext=590194,
+    text_rtf=590195,
+    text_scriplet=590196,
+    text_tab_separated_values=590197,
+    text_troff=590198,
+    text_uri_list=590199,
+    text_vnd_abc=590200,
+    text_vnd_fmi_flexstor=590201,
+    text_vnd_wap_wml=590202,
+    text_vnd_wap_wmlscript=590203,
+    text_webviewhtml=590204,
+    text_x_Algol68=590205,
+    text_x_asm=590206,
+    text_x_audiosoft_intra=590207,
+    text_x_awk=590208,
+    text_x_bcpl=590209,
+    text_x_c=590210,
+    text_x_c__=590211,
+    text_x_component=590212,
+    text_x_diff=590213,
+    text_x_fortran=590214,
+    text_x_java=590215,
+    text_x_la_asf=590216,
+    text_x_lisp=590217,
+    text_x_m=590218,
+    text_x_m4=590219,
+    text_x_makefile=590220,
+    text_x_ms_regedit=590221,
+    text_x_msdos_batch=590222,
+    text_x_objective_c=590223,
+    text_x_pascal=590224,
+    text_x_perl=590225,
+    text_x_php=590226,
+    text_x_po=590227,
+    text_x_python=590228,
+    text_x_ruby=590229,
+    text_x_sass=590230,
+    text_x_scss=590231,
+    text_x_server_parsed_html=590232,
+    text_x_setext=590233,
+    text_x_sgml=590234 | 0x01000000,
+    text_x_shellscript=590235,
+    text_x_speech=590236,
+    text_x_tcl=590237,
+    text_x_tex=590238,
+    text_x_uil=590239,
+    text_x_uuencode=590240,
+    text_x_vcalendar=590241,
+    text_x_vcard=590242,
+    text_xml=590243 | 0x01000000,
+    video_MP2T=393636,
+    video_animaflex=393637,
+    video_avi=393638,
+    video_avs_video=393639,
+    video_mp4=393640,
+    video_mpeg=393641,
+    video_quicktime=393642,
+    video_vdo=393643,
+    video_vivo=393644,
+    video_vnd_rn_realvideo=393645,
+    video_vosaic=393646,
+    video_webm=393647,
+    video_x_amt_demorun=393648,
+    video_x_amt_showrun=393649,
+    video_x_atomic3d_feature=393650,
+    video_x_dl=393651,
+    video_x_dv=393652,
+    video_x_fli=393653,
+    video_x_flv=393654,
+    video_x_isvideo=393655,
+    video_x_jng=393656 | 0x80000000,
+    video_x_m4v=393657,
+    video_x_matroska=393658,
+    video_x_mng=393659,
+    video_x_motion_jpeg=393660,
+    video_x_ms_asf=393661,
+    video_x_msvideo=393662,
+    video_x_qtc=393663,
+    video_x_sgi_movie=393664,
+    x_epoc_x_sisx_app=721345,
 };
 char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
 case application_arj: return "application/arj";
@@ -610,6 +632,7 @@ case application_x_mif: return "application/x-mif";
 case application_x_mix_transfer: return "application/x-mix-transfer";
 case application_xml: return "application/xml";
 case application_x_mobipocket_ebook: return "application/x-mobipocket-ebook";
+case application_vnd_amazon_mobi8_ebook: return "application/vnd.amazon.mobi8-ebook";
 case application_x_msaccess: return "application/x-msaccess";
 case application_x_ms_compress_szdd: return "application/x-ms-compress-szdd";
 case application_x_ms_pdb: return "application/x-ms-pdb";
@@ -782,6 +805,7 @@ case text_mcf: return "text/mcf";
 case text_pascal: return "text/pascal";
 case text_PGP: return "text/PGP";
 case text_plain: return "text/plain";
+case application_vnd_coffeescript: return "application/vnd.coffeescript";
 case text_richtext: return "text/richtext";
 case text_rtf: return "text/rtf";
 case text_scriplet: return "text/scriplet";
@@ -862,6 +886,26 @@ case video_x_msvideo: return "video/x-msvideo";
 case video_x_qtc: return "video/x-qtc";
 case video_x_sgi_movie: return "video/x-sgi-movie";
 case x_epoc_x_sisx_app: return "x-epoc/x-sisx-app";
+case application_x_zstd_dictionary: return "application/x-zstd-dictionary";
+case application_vnd_ms_outlook: return "application/vnd.ms-outlook";
+case image_x_olympus_orf: return "image/x-olympus-orf";
+case image_x_nikon_nef: return "image/x-nikon-nef";
+case image_x_fuji_raf: return "image/x-fuji-raf";
+case image_x_panasonic_raw: return "image/x-panasonic-raw";
+case image_x_adobe_dng: return "image/x-adobe-dng";
+case image_x_canon_cr2: return "image/x-canon-cr2";
+case image_x_canon_crw: return "image/x-canon-crw";
+case image_x_dcraw: return "image/x-dcraw";
+case image_x_kodak_dcr: return "image/x-kodak-dcr";
+case image_x_kodak_k25: return "image/x-kodak-k25";
+case image_x_kodak_kdc: return "image/x-kodak-kdc";
+case image_x_minolta_mrw: return "image/x-minolta-mrw";
+case image_x_pentax_pef: return "image/x-pentax-pef";
+case image_x_sigma_x3f: return "image/x-sigma-x3f";
+case image_x_sony_arw: return "image/x-sony-arw";
+case image_x_sony_sr2: return "image/x-sony-sr2";
+case image_x_sony_srf: return "image/x-sony-srf";
+case image_x_epson_erf: return "image/x-epson-erf";
 default: return NULL;}}
 GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -879,6 +923,7 @@ g_hash_table_insert(ext_table, "frl", (gpointer)application_freeloader);
 g_hash_table_insert(ext_table, "spl", (gpointer)application_futuresplash);
 g_hash_table_insert(ext_table, "vew", (gpointer)application_groupwise);
 g_hash_table_insert(ext_table, "gz", (gpointer)application_gzip);
+g_hash_table_insert(ext_table, "tgz", (gpointer)application_gzip);
 g_hash_table_insert(ext_table, "hta", (gpointer)application_hta);
 g_hash_table_insert(ext_table, "unv", (gpointer)application_i_deas);
 g_hash_table_insert(ext_table, "iges", (gpointer)application_iges);
@@ -993,7 +1038,6 @@ g_hash_table_insert(ext_table, "cpio", (gpointer)application_x_cpio);
 g_hash_table_insert(ext_table, "dbf", (gpointer)application_x_dbf);
 g_hash_table_insert(ext_table, "deb", (gpointer)application_x_debian_package);
 g_hash_table_insert(ext_table, "deepv", (gpointer)application_x_deepv);
-g_hash_table_insert(ext_table, "dcr", (gpointer)application_x_director);
 g_hash_table_insert(ext_table, "dir", (gpointer)application_x_director);
 g_hash_table_insert(ext_table, "dxr", (gpointer)application_x_director);
 g_hash_table_insert(ext_table, "dmp", (gpointer)application_x_dmp);
@@ -1052,6 +1096,8 @@ g_hash_table_insert(ext_table, "mif", (gpointer)application_x_mif);
 g_hash_table_insert(ext_table, "nix", (gpointer)application_x_mix_transfer);
 g_hash_table_insert(ext_table, "opf", (gpointer)application_xml);
 g_hash_table_insert(ext_table, "mobi", (gpointer)application_x_mobipocket_ebook);
+g_hash_table_insert(ext_table, "azw", (gpointer)application_vnd_amazon_mobi8_ebook);
+g_hash_table_insert(ext_table, "azw3", (gpointer)application_vnd_amazon_mobi8_ebook);
 g_hash_table_insert(ext_table, "accdb", (gpointer)application_x_msaccess);
 g_hash_table_insert(ext_table, "fon", (gpointer)application_x_ms_compress_szdd);
 g_hash_table_insert(ext_table, "pdb", (gpointer)application_x_ms_pdb);
@@ -1271,6 +1317,11 @@ g_hash_table_insert(ext_table, "sfv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "m3u", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "csv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "eml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "make", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "log", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "markdown", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "yaml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "coffee", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(ext_table, "rt", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtf", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtx", (gpointer)text_richtext);
@@ -1379,6 +1430,25 @@ g_hash_table_insert(ext_table, "divx", (gpointer)video_x_msvideo);
 g_hash_table_insert(ext_table, "qtc", (gpointer)video_x_qtc);
 g_hash_table_insert(ext_table, "movie", (gpointer)video_x_sgi_movie);
 g_hash_table_insert(ext_table, "mv", (gpointer)video_x_sgi_movie);
+g_hash_table_insert(ext_table, "msg", (gpointer)application_vnd_ms_outlook);
+g_hash_table_insert(ext_table, "orf", (gpointer)image_x_olympus_orf);
+g_hash_table_insert(ext_table, "nef", (gpointer)image_x_nikon_nef);
+g_hash_table_insert(ext_table, "raf", (gpointer)image_x_fuji_raf);
+g_hash_table_insert(ext_table, "rw2", (gpointer)image_x_panasonic_raw);
+g_hash_table_insert(ext_table, "raw", (gpointer)image_x_panasonic_raw);
+g_hash_table_insert(ext_table, "dng", (gpointer)image_x_adobe_dng);
+g_hash_table_insert(ext_table, "cr2", (gpointer)image_x_canon_cr2);
+g_hash_table_insert(ext_table, "crw", (gpointer)image_x_canon_crw);
+g_hash_table_insert(ext_table, "dcr", (gpointer)image_x_kodak_dcr);
+g_hash_table_insert(ext_table, "k25", (gpointer)image_x_kodak_k25);
+g_hash_table_insert(ext_table, "kdc", (gpointer)image_x_kodak_kdc);
+g_hash_table_insert(ext_table, "mrw", (gpointer)image_x_minolta_mrw);
+g_hash_table_insert(ext_table, "pef", (gpointer)image_x_pentax_pef);
+g_hash_table_insert(ext_table, "xf3", (gpointer)image_x_sigma_x3f);
+g_hash_table_insert(ext_table, "arw", (gpointer)image_x_sony_arw);
+g_hash_table_insert(ext_table, "sr2", (gpointer)image_x_sony_sr2);
+g_hash_table_insert(ext_table, "srf", (gpointer)image_x_sony_srf);
+g_hash_table_insert(ext_table, "erf", (gpointer)image_x_epson_erf);
 return ext_table;}
 GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj);
@@ -1556,6 +1626,7 @@ g_hash_table_insert(mime_table, "application/x-mif", (gpointer)application_x_mif
 g_hash_table_insert(mime_table, "application/x-mix-transfer", (gpointer)application_x_mix_transfer);
 g_hash_table_insert(mime_table, "application/xml", (gpointer)application_xml);
 g_hash_table_insert(mime_table, "application/x-mobipocket-ebook", (gpointer)application_x_mobipocket_ebook);
+g_hash_table_insert(mime_table, "application/vnd.amazon.mobi8-ebook", (gpointer)application_vnd_amazon_mobi8_ebook);
 g_hash_table_insert(mime_table, "application/x-msaccess", (gpointer)application_x_msaccess);
 g_hash_table_insert(mime_table, "application/x-ms-compress-szdd", (gpointer)application_x_ms_compress_szdd);
 g_hash_table_insert(mime_table, "application/x-ms-pdb", (gpointer)application_x_ms_pdb);
@@ -1728,6 +1799,7 @@ g_hash_table_insert(mime_table, "text/mcf", (gpointer)text_mcf);
 g_hash_table_insert(mime_table, "text/pascal", (gpointer)text_pascal);
 g_hash_table_insert(mime_table, "text/PGP", (gpointer)text_PGP);
 g_hash_table_insert(mime_table, "text/plain", (gpointer)text_plain);
+g_hash_table_insert(mime_table, "application/vnd.coffeescript", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(mime_table, "text/richtext", (gpointer)text_richtext);
 g_hash_table_insert(mime_table, "text/rtf", (gpointer)text_rtf);
 g_hash_table_insert(mime_table, "text/scriplet", (gpointer)text_scriplet);
@@ -1808,5 +1880,25 @@ g_hash_table_insert(mime_table, "video/x-msvideo", (gpointer)video_x_msvideo);
 g_hash_table_insert(mime_table, "video/x-qtc", (gpointer)video_x_qtc);
 g_hash_table_insert(mime_table, "video/x-sgi-movie", (gpointer)video_x_sgi_movie);
 g_hash_table_insert(mime_table, "x-epoc/x-sisx-app", (gpointer)x_epoc_x_sisx_app);
+g_hash_table_insert(mime_table, "application/x-zstd-dictionary", (gpointer)application_x_zstd_dictionary);
+g_hash_table_insert(mime_table, "application/vnd.ms-outlook", (gpointer)application_vnd_ms_outlook);
+g_hash_table_insert(mime_table, "image/x-olympus-orf", (gpointer)image_x_olympus_orf);
+g_hash_table_insert(mime_table, "image/x-nikon-nef", (gpointer)image_x_nikon_nef);
+g_hash_table_insert(mime_table, "image/x-fuji-raf", (gpointer)image_x_fuji_raf);
+g_hash_table_insert(mime_table, "image/x-panasonic-raw", (gpointer)image_x_panasonic_raw);
+g_hash_table_insert(mime_table, "image/x-adobe-dng", (gpointer)image_x_adobe_dng);
+g_hash_table_insert(mime_table, "image/x-canon-cr2", (gpointer)image_x_canon_cr2);
+g_hash_table_insert(mime_table, "image/x-canon-crw", (gpointer)image_x_canon_crw);
+g_hash_table_insert(mime_table, "image/x-dcraw", (gpointer)image_x_dcraw);
+g_hash_table_insert(mime_table, "image/x-kodak-dcr", (gpointer)image_x_kodak_dcr);
+g_hash_table_insert(mime_table, "image/x-kodak-k25", (gpointer)image_x_kodak_k25);
+g_hash_table_insert(mime_table, "image/x-kodak-kdc", (gpointer)image_x_kodak_kdc);
+g_hash_table_insert(mime_table, "image/x-minolta-mrw", (gpointer)image_x_minolta_mrw);
+g_hash_table_insert(mime_table, "image/x-pentax-pef", (gpointer)image_x_pentax_pef);
+g_hash_table_insert(mime_table, "image/x-sigma-x3f", (gpointer)image_x_sigma_x3f);
+g_hash_table_insert(mime_table, "image/x-sony-arw", (gpointer)image_x_sony_arw);
+g_hash_table_insert(mime_table, "image/x-sony-sr2", (gpointer)image_x_sony_sr2);
+g_hash_table_insert(mime_table, "image/x-sony-srf", (gpointer)image_x_sony_srf);
+g_hash_table_insert(mime_table, "image/x-epson-erf", (gpointer)image_x_epson_erf);
 return mime_table;}
 #endif
--- a/src/parsing/parse.c
+++ b/src/parsing/parse.c
@@ -1,7 +1,15 @@
+#include "parse.h"
+
 #include "src/sist.h"
 #include "src/ctx.h"
+#include "mime.h"
+#include "src/io/serialize.h"

-__thread magic_t Magic = NULL;
+#include <magic.h>
+
+
+#define MIN_VIDEO_SIZE 1024 * 64
+#define MIN_IMAGE_SIZE 1024 * 2

 int fs_read(struct vfile *f, void *buf, size_t size) {

@@ -24,61 +32,37 @@ void fs_close(struct vfile *f) {
    }
 }

-void *read_all(parse_job_t *job, const char *buf, int bytes_read) {
-
-    void *full_buf;
-
-    if (job->info.st_size <= bytes_read) {
-        full_buf = malloc(job->info.st_size);
-        memcpy(full_buf, buf, job->info.st_size);
-    } else {
-        full_buf = malloc(job->info.st_size);
-        memcpy(full_buf, buf, bytes_read);
-
-        int ret = job->vfile.read(&job->vfile, full_buf + bytes_read, job->info.st_size - bytes_read);
-        if (ret < 0) {
-            free(full_buf);
-
-            if (job->vfile.is_fs_file) {
-                LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno))
-            } else {
-                LOG_ERRORF(job->filepath, "(virtual) read(): [%d] %s", ret, archive_error_string(job->vfile.arc))
-            }
-            return NULL;
-        }
+void fs_reset(struct vfile *f) {
+    if (f->fd != -1) {
+        lseek(f->fd, 0, SEEK_SET);
    }
-
-    return full_buf;
 }

+#define IS_GIT_OBJ (strlen(doc.filepath + doc.base) == 38 && (strstr(doc.filepath, "objects") != NULL))
+
 void parse(void *arg) {

    parse_job_t *job = arg;
    document_t doc;

-    int inc_ts = incremental_get(ScanCtx.original_table, job->info.st_ino);
-    if (inc_ts != 0 && inc_ts == job->info.st_mtim.tv_sec) {
-        incremental_mark_file_for_copy(ScanCtx.copy_table, job->info.st_ino);
+    int inc_ts = incremental_get(ScanCtx.original_table, job->vfile.info.st_ino);
+    if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
+        incremental_mark_file_for_copy(ScanCtx.copy_table, job->vfile.info.st_ino);
        return;
    }

-    if (Magic == NULL) {
-        Magic = magic_open(MAGIC_MIME_TYPE);
-        magic_load(Magic, NULL);
-    }
-
    doc.filepath = job->filepath;
    doc.ext = (short) job->ext;
    doc.base = (short) job->base;
    doc.meta_head = NULL;
    doc.meta_tail = NULL;
    doc.mime = 0;
-    doc.size = job->info.st_size;
-    doc.ino = job->info.st_ino;
-    doc.mtime = job->info.st_mtim.tv_sec;
+    doc.size = job->vfile.info.st_size;
+    doc.ino = job->vfile.info.st_ino;
+    doc.mtime = job->vfile.info.st_mtim.tv_sec;

    uuid_generate(doc.uuid);
-    char *buf[PARSE_BUF_SIZE];
+    char *buf[MAGIC_BUF_SIZE];

    if (LogCtx.very_verbose) {
        char uuid_str[UUID_STR_LEN];
@@ -86,7 +70,7 @@ void parse(void *arg) {
        LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str)
    }

-    if (job->info.st_size == 0) {
+    if (job->vfile.info.st_size == 0) {
        doc.mime = MIME_EMPTY;
    } else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) {
        doc.mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext);
@@ -95,8 +79,17 @@ void parse(void *arg) {
    int bytes_read = 0;

    if (doc.mime == 0 && !ScanCtx.fast) {
+        if (IS_GIT_OBJ) {
+            goto abort;
+        }
+
        // Get mime type with libmagic
-        bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
+        if (!job->vfile.is_fs_file) {
+            LOG_WARNING(job->filepath, "Guessing mime type with libmagic inside archive files is not currently supported");
+            goto abort;
+        }
+
+        bytes_read = job->vfile.read(&job->vfile, buf, MAGIC_BUF_SIZE);
        if (bytes_read < 0) {

            if (job->vfile.is_fs_file) {
@@ -109,7 +102,10 @@ void parse(void *arg) {
            return;
        }

-        const char *magic_mime_str = magic_buffer(Magic, buf, bytes_read);
+        magic_t magic = magic_open(MAGIC_MIME_TYPE);
+        magic_load(magic, NULL);
+
+        const char *magic_mime_str = magic_buffer(magic, buf, bytes_read);
        if (magic_mime_str != NULL) {
            doc.mime = mime_get_mime_by_string(ScanCtx.mime_table, magic_mime_str);

@@ -120,71 +116,56 @@ void parse(void *arg) {
            }
        }

-        magic_close(Magic);
-        Magic = NULL;
+        job->vfile.reset(&job->vfile);
+
+        magic_close(magic);
    }

    int mmime = MAJOR_MIME(doc.mime);

    if (!(SHOULD_PARSE(doc.mime))) {

+    } else if (IS_RAW(doc.mime)) {
+        parse_raw(&ScanCtx.raw_ctx, &job->vfile, &doc);
    } else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) ||
               (mmime == MimeImage && doc.size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) {

-        if (job->vfile.is_fs_file) {
-            parse_media_filename(job->filepath, &doc);
-        } else {
-            parse_media_vfile(&job->vfile, &doc);
-        }
+        parse_media(&ScanCtx.media_ctx, &job->vfile, &doc);

    } else if (IS_PDF(doc.mime)) {
-        void *pdf_buf = read_all(job, (char *) buf, bytes_read);
-        parse_pdf(pdf_buf, doc.size, &doc);
+        parse_ebook(&ScanCtx.ebook_ctx, &job->vfile, mime_get_mime_text(doc.mime), &doc);

-        if (pdf_buf != buf && pdf_buf != NULL) {
-            free(pdf_buf);
+    } else if (mmime == MimeText && ScanCtx.text_ctx.content_size > 0) {
+        if (IS_MARKUP(doc.mime)) {
+            parse_markup(&ScanCtx.text_ctx, &job->vfile, &doc);
+        } else {
+            parse_text(&ScanCtx.text_ctx, &job->vfile, &doc);
        }

-    } else if (mmime == MimeText && ScanCtx.content_size > 0) {
-        parse_text(bytes_read, &job->vfile, (char *) buf, &doc);
-
    } else if (IS_FONT(doc.mime)) {
-        void *font_buf = read_all(job, (char *) buf, bytes_read);
-        parse_font(font_buf, doc.size, &doc);
+        parse_font(&ScanCtx.font_ctx, &job->vfile, &doc);

-        if (font_buf != buf && font_buf != NULL) {
-            free(font_buf);
-        }
    } else if (
-            ScanCtx.archive_mode != ARC_MODE_SKIP && (
+            ScanCtx.arc_ctx.mode != ARC_MODE_SKIP && (
                    IS_ARC(doc.mime) ||
                    (IS_ARC_FILTER(doc.mime) && should_parse_filtered_file(doc.filepath, doc.ext))
            )) {
-        parse_archive(&job->vfile, &doc);
-    } else if (ScanCtx.content_size > 0 && IS_DOC(doc.mime)) {
-        void *doc_buf = read_all(job, (char *) buf, bytes_read);
-        parse_doc(doc_buf, doc.size, &doc);
-
-        if (doc_buf != buf && doc_buf != NULL) {
-            free(doc_buf);
-        }
-    } else if (is_cbr(doc.mime)) {
-        void *cbr_buf = read_all(job, (char *) buf, bytes_read);
-        parse_cbr(cbr_buf, doc.size, &doc);
-
-        if (cbr_buf != buf && cbr_buf != NULL) {
-            free(cbr_buf);
-        }
+        parse_archive(&ScanCtx.arc_ctx, &job->vfile, &doc);
+    } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(doc.mime)) {
+        parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, &doc);
+    } else if (is_cbr(&ScanCtx.comic_ctx, doc.mime) || is_cbz(&ScanCtx.comic_ctx, doc.mime)) {
+        parse_comic(&ScanCtx.comic_ctx, &job->vfile, &doc);
+    } else if (IS_MOBI(doc.mime)) {
+        parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, &doc);
    }

+    abort:
+
    //Parent meta
    if (!uuid_is_null(job->parent)) {
-        char tmp[UUID_STR_LEN];
-        uuid_unparse(job->parent, tmp);
-
        meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + UUID_STR_LEN + 1);
        meta_parent->key = MetaParent;
-        strcpy(meta_parent->strval, tmp);
+        uuid_unparse(job->parent, meta_parent->str_val);
        APPEND_META((&doc), meta_parent)
    }

@@ -194,7 +175,5 @@ void parse(void *arg) {
 }

 void cleanup_parse() {
-    if (Magic != NULL) {
-        magic_close(Magic);
-    }
+    // noop
 }
--- a/src/parsing/parse.h
+++ b/src/parsing/parse.h
@@ -1,12 +1,13 @@
 #ifndef SIST2_PARSE_H
 #define SIST2_PARSE_H

-#include "src/sist.h"
+#include "../sist.h"

-#define PARSE_BUF_SIZE 4096
+#define MAGIC_BUF_SIZE 4096 * 6

 int fs_read(struct vfile *f, void *buf, size_t size);
 void fs_close(struct vfile *f);
+void fs_reset(struct vfile *f);

 void parse(void *arg);

--- a/src/parsing/pdf.c
+++ b/src/parsing/pdf.c
@@ -1,331 +0,0 @@
-#include "pdf.h"
-#include "src/ctx.h"
-
-#define MIN_OCR_SIZE 350
-#define MIN_OCR_LEN 10
-__thread text_buffer_t thread_buffer;
-
-
-int render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
-
-    int err = 0;
-    fz_page *cover = NULL;
-
-    fz_var(cover);
-    fz_var(err);
-    fz_try(ctx)
-        cover = fz_load_page(ctx, fzdoc, 0);
-    fz_catch(ctx)
-        err = 1;
-
-    if (err != 0) {
-        fz_drop_page(ctx, cover);
-        LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
-        return FALSE;
-    }
-
-    fz_rect bounds = fz_bound_page(ctx, cover);
-
-    float scale;
-    float w = (float) bounds.x1 - bounds.x0;
-    float h = (float) bounds.y1 - bounds.y0;
-    if (w > h) {
-        scale = (float) ScanCtx.tn_size / w;
-    } else {
-        scale = (float) ScanCtx.tn_size / h;
-    }
-    fz_matrix m = fz_scale(scale, scale);
-
-    bounds = fz_transform_rect(bounds, m);
-    fz_irect bbox = fz_round_rect(bounds);
-    fz_pixmap *pixmap = fz_new_pixmap_with_bbox(ctx, ctx->colorspace->rgb, bbox, NULL, 0);
-
-    fz_clear_pixmap_with_value(ctx, pixmap, 0xFF);
-    fz_device *dev = fz_new_draw_device(ctx, m, pixmap);
-
-    fz_var(err);
-    fz_try(ctx)
-    {
-        pthread_mutex_lock(&ScanCtx.mupdf_mu);
-        fz_run_page(ctx, cover, dev, fz_identity, NULL);
-    }
-    fz_always(ctx)
-    {
-        fz_close_device(ctx, dev);
-        fz_drop_device(ctx, dev);
-        pthread_mutex_unlock(&ScanCtx.mupdf_mu);
-    }
-    fz_catch(ctx)
-        err = ctx->error.errcode;
-
-    if (err != 0) {
-        LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
-        fz_drop_page(ctx, cover);
-        fz_drop_pixmap(ctx, pixmap);
-        return FALSE;
-    }
-
-    fz_buffer *fzbuf = NULL;
-    fz_var(fzbuf);
-    fz_var(err);
-
-    fz_try(ctx)
-        fzbuf = fz_new_buffer_from_pixmap_as_png(ctx, pixmap, fz_default_color_params);
-    fz_catch(ctx)
-        err = ctx->error.errcode;
-
-    if (err == 0) {
-        unsigned char *tn_buf;
-        size_t tn_len = fz_buffer_storage(ctx, fzbuf, &tn_buf);
-        store_write(ScanCtx.index.store, (char *) doc->uuid, sizeof(doc->uuid), (char *) tn_buf, tn_len);
-    }
-
-    fz_drop_buffer(ctx, fzbuf);
-    fz_drop_pixmap(ctx, pixmap);
-    fz_drop_page(ctx, cover);
-
-    if (err != 0) {
-        LOG_WARNINGF(doc->filepath, "fz_new_buffer_from_pixmap_as_png() returned error code [%d] %s", err,
-                     ctx->error.message)
-        return FALSE;
-    }
-
-    return TRUE;
-}
-
-void fz_err_callback(void *user, UNUSED(const char *message)) {
-    if (LogCtx.verbose) {
-        document_t *doc = (document_t *) user;
-        LOG_WARNINGF(doc->filepath, "FZ: %s", message)
-    }
-}
-
-__always_inline
-static void init_ctx(fz_context *ctx, document_t *doc) {
-    fz_disable_icc(ctx);
-    fz_register_document_handlers(ctx);
-
-    ctx->warn.print_user = doc;
-    ctx->warn.print = fz_err_callback;
-    ctx->error.print_user = doc;
-    ctx->error.print = fz_err_callback;
-}
-
-__always_inline
-static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
-    if (block->type != FZ_STEXT_BLOCK_TEXT) {
-        return 0;
-    }
-
-    fz_stext_line *line = block->u.t.first_line;
-    while (line != NULL) {
-        fz_stext_char *c = line->first_char;
-        while (c != NULL) {
-            if (text_buffer_append_char(tex, c->c) == TEXT_BUF_FULL) {
-                return TEXT_BUF_FULL;
-            }
-            c = c->next;
-        }
-        line = line->next;
-    }
-    return 0;
-}
-
-#define IS_VALID_BPP(d) (d==1 || d==2 || d==4 || d==8 || d==16 || d==24 || d==32)
-
-void fill_image(fz_context *ctx, UNUSED(fz_device *dev),
-                fz_image *img, UNUSED(fz_matrix ctm), UNUSED(float alpha),
-                UNUSED(fz_color_params color_params)) {
-
-    int l2factor = 0;
-
-    if (img->w > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE && IS_VALID_BPP(img->n)) {
-
-        fz_pixmap *pix = img->get_pixmap(ctx, img, NULL, img->w, img->h, &l2factor);
-
-        if (pix->h > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE && img->xres != 0) {
-            TessBaseAPI *api = TessBaseAPICreate();
-            TessBaseAPIInit3(api, ScanCtx.tesseract_path, ScanCtx.tesseract_lang);
-
-            TessBaseAPISetImage(api, pix->samples, pix->w, pix->h, pix->n, pix->stride);
-            TessBaseAPISetSourceResolution(api, pix->xres);
-
-            char *text = TessBaseAPIGetUTF8Text(api);
-            size_t len = strlen(text);
-            if (len >= MIN_OCR_LEN) {
-                text_buffer_append_string(&thread_buffer, text, len - 1);
-                LOG_DEBUGF(
-                        "pdf.c",
-                        "(OCR) %dx%d got %dB from tesseract (%s), buffer:%dB",
-                        pix->w, pix->h, len, ScanCtx.tesseract_lang, thread_buffer.dyn_buffer.cur
-                )
-            }
-
-            TessBaseAPIEnd(api);
-            TessBaseAPIDelete(api);
-        }
-        fz_drop_pixmap(ctx, pix);
-    }
-}
-
-void parse_pdf(const void *buf, size_t buf_len, document_t *doc) {
-
-    if (buf == NULL) {
-        return;
-    }
-
-    static int mu_is_initialized = 0;
-    if (!mu_is_initialized) {
-        pthread_mutex_init(&ScanCtx.mupdf_mu, NULL);
-        mu_is_initialized = 1;
-    }
-    fz_context *ctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
-
-    init_ctx(ctx, doc);
-
-    int err = 0;
-
-    fz_document *fzdoc = NULL;
-    fz_stream *stream = NULL;
-    fz_var(fzdoc);
-    fz_var(stream);
-    fz_var(err);
-
-    fz_try(ctx)
-    {
-        stream = fz_open_memory(ctx, buf, buf_len);
-        fzdoc = fz_open_document_with_stream(ctx, mime_get_mime_text(doc->mime), stream);
-    }
-    fz_catch(ctx)
-        err = ctx->error.errcode;
-
-    if (err != 0) {
-        fz_drop_stream(ctx, stream);
-        fz_drop_document(ctx, fzdoc);
-        fz_drop_context(ctx);
-        return;
-    }
-
-    char title[4096] = {'\0',};
-    fz_try(ctx)
-        fz_lookup_metadata(ctx, fzdoc, FZ_META_INFO_TITLE, title, sizeof(title));
-    fz_catch(ctx)
-        ;
-
-    if (strlen(title) > 0) {
-        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + strlen(title));
-        meta_content->key = MetaTitle;
-        strcpy(meta_content->strval, title);
-        APPEND_META(doc, meta_content)
-    }
-
-    int page_count = -1;
-    fz_var(err);
-    fz_try(ctx)
-        page_count = fz_count_pages(ctx, fzdoc);
-    fz_catch(ctx)
-        err = ctx->error.errcode;
-
-    if (err) {
-        LOG_WARNINGF(doc->filepath, "fz_count_pages() returned error code [%d] %s", err, ctx->error.message)
-        fz_drop_stream(ctx, stream);
-        fz_drop_document(ctx, fzdoc);
-        fz_drop_context(ctx);
-        return;
-    }
-
-    if (ScanCtx.tn_size > 0) {
-        err = render_cover(ctx, doc, fzdoc);
-    }
-
-    if (err == TRUE) {
-        fz_drop_stream(ctx, stream);
-        fz_drop_document(ctx, fzdoc);
-        fz_drop_context(ctx);
-        return;
-    }
-
-    if (ScanCtx.content_size > 0) {
-        fz_stext_options opts = {0};
-        thread_buffer = text_buffer_create(ScanCtx.content_size);
-
-        for (int current_page = 0; current_page < page_count; current_page++) {
-            fz_page *page = NULL;
-            fz_var(err);
-            fz_try(ctx)
-                page = fz_load_page(ctx, fzdoc, current_page);
-            fz_catch(ctx)
-                err = ctx->error.errcode;
-            if (err != 0) {
-                LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
-                text_buffer_destroy(&thread_buffer);
-                fz_drop_page(ctx, page);
-                fz_drop_stream(ctx, stream);
-                fz_drop_document(ctx, fzdoc);
-                fz_drop_context(ctx);
-                return;
-            }
-
-            fz_stext_page *stext = fz_new_stext_page(ctx, fz_bound_page(ctx, page));
-            fz_device *dev = fz_new_stext_device(ctx, stext, &opts);
-            dev->stroke_path = NULL;
-            dev->stroke_text = NULL;
-            dev->clip_text = NULL;
-            dev->clip_stroke_path = NULL;
-            dev->clip_stroke_text = NULL;
-
-            if (ScanCtx.tesseract_lang != NULL) {
-                dev->fill_image = fill_image;
-            }
-
-            fz_var(err);
-            fz_try(ctx)
-                fz_run_page(ctx, page, dev, fz_identity, NULL);
-            fz_always(ctx)
-            {
-                fz_close_device(ctx, dev);
-                fz_drop_device(ctx, dev);
-            }
-            fz_catch(ctx)
-                err = ctx->error.errcode;
-
-            if (err != 0) {
-                LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
-                text_buffer_destroy(&thread_buffer);
-                fz_drop_page(ctx, page);
-                fz_drop_stext_page(ctx, stext);
-                fz_drop_stream(ctx, stream);
-                fz_drop_document(ctx, fzdoc);
-                fz_drop_context(ctx);
-                return;
-            }
-
-            fz_stext_block *block = stext->first_block;
-            while (block != NULL) {
-                int ret = read_stext_block(block, &thread_buffer);
-                if (ret == TEXT_BUF_FULL) {
-                    break;
-                }
-                block = block->next;
-            }
-            fz_drop_stext_page(ctx, stext);
-            fz_drop_page(ctx, page);
-
-            if (thread_buffer.dyn_buffer.cur >= thread_buffer.dyn_buffer.size) {
-                break;
-            }
-        }
-        text_buffer_terminate_string(&thread_buffer);
-
-        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + thread_buffer.dyn_buffer.cur);
-        meta_content->key = MetaContent;
-        memcpy(meta_content->strval, thread_buffer.dyn_buffer.buf, thread_buffer.dyn_buffer.cur);
-        APPEND_META(doc, meta_content)
-
-        text_buffer_destroy(&thread_buffer);
-    }
-
-    fz_drop_stream(ctx, stream);
-    fz_drop_document(ctx, fzdoc);
-    fz_drop_context(ctx);
-}
--- a/src/parsing/pdf.h
+++ b/src/parsing/pdf.h
@@ -1,9 +0,0 @@
-#ifndef SIST2_PDF_H
-#define SIST2_PDF_H
-
-#include "src/sist.h"
-
-
-void parse_pdf(const void *buf, size_t buf_len, document_t *doc);
-
-#endif
--- a/src/parsing/text.c
+++ b/src/parsing/text.c
@@ -1,37 +0,0 @@
-#include "text.h"
-#include "src/ctx.h"
-
-void parse_text(int bytes_read, struct vfile *f, char *buf, document_t *doc) {
-
-    char *intermediate_buf;
-    int intermediate_buf_len;
-
-    if (bytes_read == doc->size || bytes_read >= ScanCtx.content_size) {
-        int to_copy = MIN(bytes_read, ScanCtx.content_size);
-        intermediate_buf = malloc(to_copy);
-        intermediate_buf_len = to_copy;
-        memcpy(intermediate_buf, buf, to_copy);
-
-    } else {
-        int to_read = MIN(ScanCtx.content_size, doc->size) - bytes_read;
-
-        intermediate_buf = malloc(to_read + bytes_read);
-        intermediate_buf_len = to_read + bytes_read;
-        if (bytes_read != 0) {
-            memcpy(intermediate_buf, buf, bytes_read);
-        }
-
-        f->read(f, intermediate_buf + bytes_read, to_read);
-    }
-    text_buffer_t tex = text_buffer_create(ScanCtx.content_size);
-    text_buffer_append_string(&tex, intermediate_buf, intermediate_buf_len);
-    text_buffer_terminate_string(&tex);
-
-    meta_line_t *meta = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur);
-    meta->key = MetaContent;
-    strcpy(meta->strval, tex.dyn_buffer.buf);
-    APPEND_META(doc, meta)
-
-    free(intermediate_buf);
-    text_buffer_destroy(&tex);
-}
--- a/src/parsing/text.h
+++ b/src/parsing/text.h
@@ -1,8 +0,0 @@
-#ifndef SIST2_TEXT_H
-#define SIST2_TEXT_H
-
-#include "src/sist.h"
-
-void parse_text(int bytes_read, struct vfile *f, char *buf, document_t *doc);
-
-#endif
--- a/src/sist.h
+++ b/src/sist.h
@@ -1,75 +1,51 @@
 #ifndef SIST_H
 #define SIST_H

+#ifndef	FALSE
+#define	FALSE	(0)
+#define BOOL int
+#endif
+
+#ifndef	TRUE
+#define	TRUE	(!FALSE)
+#endif
+
+#undef	MAX
+#define MAX(a, b)  (((a) > (b)) ? (a) : (b))
+
+#undef	MIN
+#define MIN(a, b)  (((a) < (b)) ? (a) : (b))
+
+#ifndef PATH_MAX
+#define PATH_MAX 4096
+#endif
+
+#undef ABS
+#define ABS(a)	   (((a) < 0) ? -(a) : (a))
+
 #define UUID_STR_LEN 37
 #define UNUSED(x) __attribute__((__unused__))  x

-#include <glib-2.0/glib.h>
-#include <unistd.h>
+#include "util.h"
+#include "log.h"
+#include "types.h"
+
+#include "libscan/scan.h"
+
+#include <cjson/cJSON.h>
+
+#include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
+#include <stdarg.h>
+
+#include <unistd.h>
 #include <fcntl.h>
-#include <ftw.h>
-#include <uuid.h>
-#include <magic.h>
-#include "libavformat/avformat.h"
-#include "libswscale/swscale.h"
-#include "libswresample/swresample.h"
-#include "libavcodec/avcodec.h"
-#include "libavutil/imgutils.h"
-#include <ctype.h>
-#include <mupdf/fitz.h>
-#include <mupdf/pdf.h>
-#include "argparse/argparse.h"
-#include <time.h>
-#include <limits.h>
-#include <pthread.h>
+#include <dirent.h>
 #include <sys/stat.h>
-#include <wordexp.h>
-#include "ft2build.h"
-#include "freetype/freetype.h"
-#include <archive.h>
-#include <archive_entry.h>
-#include <libxml/xmlstring.h>
-#include <libxml/parser.h>
-#define BOOL int
-#include <tesseract/capi.h>
-#include <pcre.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <ctype.h>

-#include <onion/onion.h>
-#include <onion/handler.h>
-#include <onion/block.h>
-#include <onion/shortcuts.h>
-#include <onion/codecs.h>
-#include <curl/curl.h>
-
-#include "cJSON/cJSON.h"
-
-#include "types.h"
-#include "tpool.h"
-#include "utf8.h/utf8.h"
-#include "util.h"
-#include "io/store.h"
-#include "io/serialize.h"
-#include "io/walk.h"
-#include "parsing/parse.h"
-#include "parsing/mime.h"
-#include "parsing/text.h"
-#include "parsing/pdf.h"
-#include "parsing/media.h"
-#include "parsing/font.h"
-#include "parsing/arc.h"
-#include "parsing/doc.h"
-#include "parsing/cbr.h"
-#include "cli.h"
-#include "log.h"
-
-#include "src/index/elastic.h"
-#include "index/web.h"
-#include "web/serve.h"
-#include "web/auth_basic.h"
-
-;

 #endif
--- a/src/static/css/autocomplete.min.css
+++ b/src/static/css/autocomplete.min.css
@@ -0,0 +1,4 @@
+.autocomplete-suggestions { text-align: left; cursor: default; border: 1px solid #ccc; border-top: 0; background: #fff; box-shadow: -1px 1px 3px rgba(0,0,0,.1); position: absolute; display: none; z-index: 9999; max-height: 254px; overflow: hidden; overflow-y: auto; box-sizing: border-box; }
+.autocomplete-suggestion { position: relative; padding: 0 .6em; line-height: 23px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; font-size: 1.02em; color: #333; }
+.autocomplete-suggestion b { font-weight: normal; color: #1f8dd6; }
+.autocomplete-suggestion.selected { background: #f0f0f0; }
--- a/src/static/css/bootstrap-colorpicker.min.css
+++ b/src/static/css/bootstrap-colorpicker.min.css
--- a/src/static/css/bootstrap.min.css
+++ b/src/static/css/bootstrap.min.css
--- a/src/static/css/bricklayer.min.css
+++ b/src/static/css/bricklayer.min.css
@@ -0,0 +1 @@
+.bricklayer{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:start;-webkit-align-items:flex-start;-ms-flex-align:start;align-items:flex-start;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;-webkit-flex-wrap:wrap;-ms-flex-wrap:wrap;flex-wrap:wrap}.bricklayer-column-sizer{width:100%;display:none}@media screen and (min-width:640px){.bricklayer-column-sizer{width:50%}}@media screen and (min-width:980px){.bricklayer-column-sizer{width:33.333%}}@media screen and (min-width:1200px){.bricklayer-column-sizer{width:25%}}.bricklayer-column{-webkit-box-flex:1;-webkit-flex:1;-ms-flex:1;flex:1;padding-left:5px;padding-right:5px}
--- a/src/static/css/dark.css
+++ b/src/static/css/dark.css
@@ -6,9 +6,10 @@
    width: 1rem;
    margin-right: 0.2rem;
    cursor: pointer;
-    color: #757575;
    line-height: 1rem;
-    height: 1.1rem;
+    height: 1rem;
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+    filter: brightness(65%);
 }

 .info-icon:hover {
@@ -37,6 +38,7 @@
    color: #BBB;
    text-overflow: ellipsis;
    overflow: hidden;
+    white-space: nowrap;
 }

 a {
@@ -119,7 +121,7 @@ body {
    background: #546b7a;
 }

-.navbar a:hover {
+a:hover, .btn:hover {
    color: #fff;
 }

@@ -128,7 +130,11 @@ body {
 }

 .document {
-    padding: 0.5rem;
+    padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }

 .document p {
@@ -164,6 +170,12 @@ body {
    background-color: #FAAB3C;
 }

+.add-tag-button {
+    cursor: pointer;
+    color: #212529;
+    background-color: #e0e0e0;
+}
+
 .card-img-overlay {
    pointer-events: none;
    padding: 0.75rem;
@@ -189,22 +201,41 @@ body {
    margin-right: 3px;
 }

+.badge-delete {
+    margin-right: -2px;
+    margin-left: 2px;
+    margin-top: -1px;
+    font-family: monospace;
+    font-size: 90%;
+    background: rgba(0, 0, 0, 0.2);
+    padding: 0.1em 0.4em;
+    color: white;
+    cursor: pointer;
+}
+
 .badge-user {
    color: #212529;
    background-color: #e0e0e0;
 }

+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}
+
 .fit {
    display: block;
    min-width: 64px;
    max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
    margin: 0 auto 0;
-    padding: 3px 3px 0;
-    width: auto;
    height: auto;
 }

+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
    display: block;
    max-width: 64px;
@@ -221,23 +252,7 @@ body {
    width: 100%;
 }

-@media (min-width: 1200px) {
-    .card-columns {
-        column-count: 4;
-    }
-}
-
-@media (min-width: 1500px) {
-    .container {
-        max-width: 1440px;
-    }
-
-    .card-columns {
-        column-count: 5;
-    }
-}
-
-@media (min-width: 1800px) {
+@media screen and (min-width: 1800px) {
    .container {
        max-width: 1550px;
    }
@@ -266,6 +281,7 @@ mark {
    margin: 3px;
    white-space: normal;
    color: rgb(224, 224, 224);
+    overflow: hidden;
 }

 .irs-single, .irs-from, .irs-to {
@@ -411,7 +427,7 @@ option {
    max-height: 250px;
 }

-@media (max-width: 600px) {
+@media (max-width: 650px) {
    .media-expanded .fit {
        max-height: none;
    }
@@ -432,6 +448,7 @@ option {
    .small-btn {
        display: none;
    }
+
    .large-btn {
        display: inherit;
    }
@@ -441,6 +458,7 @@ option {
    .small-btn {
        display: inherit;
    }
+
    .large-btn {
        display: none;
    }
@@ -457,3 +475,65 @@ option {
 svg {
    fill: white;
 }
+
+.play {
+    position: absolute;
+    width: 50px;
+    height: 50px;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    pointer-events: none;
+}
+
+.play svg {
+    fill: rgba(255, 255, 255, 0.7);
+}
+
+.img-wrapper:hover svg {
+    fill: rgba(255, 255, 255, 1);
+}
+
+.pointer {
+    cursor: pointer;
+}
+
+.stats-card {
+    text-align: center;
+    margin-top: 1em;
+    padding: 1em;
+
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
+
+    background: #212121;
+}
+
+.graph {
+    display: inline-block;
+    width: 40%;
+}
+
+.full-screen {
+    position: absolute;
+    left: 0;
+    width: 100%;
+}
+
+.stats-btn {
+    float: right;
+    margin-bottom: 10px;
+}
+
+#graphs-card svg text {
+    fill: #eee;
+}
+
+.wholerow {
+    outline: none !important;
+}
+
+.stat > .card-body {
+    padding: 0.7em 1.25em;
+}
--- a/src/static/css/inspire-tree-light.min.css
+++ b/src/static/css/inspire-tree-light.min.css
--- a/src/static/css/ion.rangeSlider.min.css
+++ b/src/static/css/ion.rangeSlider.min.css
--- a/src/static/css/ion.rangeSlider.skinFlat.min.css
+++ b/src/static/css/ion.rangeSlider.skinFlat.min.css
--- a/src/static/css/jquery.toast.min.css
+++ b/src/static/css/jquery.toast.min.css
--- a/src/static/css/light.css
+++ b/src/static/css/light.css
@@ -6,9 +6,10 @@
    width: 1rem;
    margin-right: 0.2rem;
    cursor: pointer;
-    color: #757575;
    line-height: 1rem;
    height: 1rem;
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+    filter: brightness(45%);
 }

 .info-icon:hover {
@@ -36,6 +37,7 @@
    color: #444;
    text-overflow: ellipsis;
    overflow: hidden;
+    white-space: nowrap;
 }

 body {
@@ -48,7 +50,9 @@ body {

 .card {
    margin-top: 1em;
-    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .075) !important;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }

 .sub-document {
@@ -66,7 +70,11 @@ body {
 }

 .document {
-    padding: 0.5rem;
+   padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }

 .document p {
@@ -102,11 +110,33 @@ body {
    background-color: #e0e0e0;
 }

+.badge {
+    margin-right: 3px;
+}
+
+.badge-delete {
+    margin-right: -2px;
+    margin-left: 2px;
+    margin-top: -1px;
+    font-family: monospace;
+    font-size: 90%;
+    background: rgba(0,0,0,0.2);
+    padding: 0.1em 0.4em;
+    color: white;
+    cursor: pointer;
+}
+
 .badge-text {
    color: #FFFFFF;
    background-color: #FAAB3C;
 }

+.add-tag-button {
+    cursor: pointer;
+    color: #212529;
+    background-color: #e0e0e0;
+}
+
 .card-img-overlay {
    pointer-events: none;
    padding: 0.75rem;
@@ -127,21 +157,25 @@ body {
    overflow: hidden;
 }

-.badge {
-    margin-right: 3px;
+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
 }

 .fit {
    display: block;
    min-width: 64px;
    max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
    margin: 0 auto 0;
-    padding: 3px 3px 0 3px;
    width: auto;
    height: auto;
 }

+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
    display: block;
    max-width: 64px;
@@ -158,23 +192,31 @@ body {
    width: 100%;
 }

-@media (min-width: 1200px) {
-    .card-columns {
-        column-count: 4;
+.bricklayer {
+    /*max-width: 100%;*/
+}
+
+@media screen and (max-width: 1200px) {
+    .bricklayer-column {
+        max-width: 100%;
    }
 }

-@media (min-width: 1500px) {
+@media screen and (min-width: 1500px) {
    .container {
        max-width: 1440px;
    }

-    .card-columns {
-        column-count: 5;
+    .bricklayer-column-sizer {
+        width: 20% !important;
+    }
+
+    .bricklayer-column {
+        max-width: 20%;
    }
 }

-@media (min-width: 1800px) {
+@media screen and (min-width: 1800px) {
    .container {
        max-width: 1550px;
    }
@@ -197,6 +239,7 @@ mark {
    margin: 3px;
    white-space: normal;
    color: #000;
+    overflow: hidden;
 }

 .irs-single, .irs-from, .irs-to {
@@ -272,7 +315,7 @@ mark {
    max-height: 250px;
 }

-@media (max-width: 600px) {
+@media (max-width: 650px) {
    .media-expanded .fit {
        max-height: none;
    }
@@ -293,6 +336,7 @@ mark {
    .small-btn {
        display: none;
    }
+
    .large-btn {
        display: inherit;
    }
@@ -302,6 +346,7 @@ mark {
    .small-btn {
        display: inherit;
    }
+
    .large-btn {
        display: none;
    }
@@ -314,3 +359,61 @@ mark {
 #pathTree .title {
    cursor: pointer;
 }
+
+.play {
+    position: absolute;
+    width: 50px;
+    height: 50px;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    pointer-events: none;
+}
+
+.play svg {
+    fill: rgba(0, 0, 0, 0.7);
+}
+
+.img-wrapper:hover svg {
+    fill: rgba(0, 0, 0, 1);
+}
+
+.pointer {
+    cursor: pointer;
+}
+
+.stats-card {
+    text-align: center;
+    margin-top: 1em;
+    padding: 1em;
+
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
+
+    background: #fff;
+}
+
+.graph {
+    display: inline-block;
+    width: 40%;
+}
+
+.full-screen {
+    position: absolute;
+    left: 0;
+    width: 100%;
+}
+
+.stats-btn {
+    float: right;
+    margin-bottom: 10px;
+}
+
+.wholerow {
+    outline: none !important;
+}
+
+.stat > .card-body {
+    padding: 0.7em 1.25em;
+}
--- a/src/static/css/lity.min.css
+++ b/src/static/css/lity.min.css
@@ -0,0 +1,3 @@
+/*! Lity - v2.4.0 - 2019-08-10
+* http://sorgalla.com/lity/
+* Copyright (c) 2015-2019 Jan Sorgalla; Licensed MIT */.lity{z-index:9990;position:fixed;top:0;right:0;bottom:0;left:0;white-space:nowrap;background:#0b0b0b;background:rgba(0,0,0,0.9);outline:none !important;opacity:0;-webkit-transition:opacity .3s ease;-o-transition:opacity .3s ease;transition:opacity .3s ease}.lity.lity-opened{opacity:1}.lity.lity-closed{opacity:0}.lity *{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.lity-wrap{z-index:9990;position:fixed;top:0;right:0;bottom:0;left:0;text-align:center;outline:none !important}.lity-wrap:before{content:'';display:inline-block;height:100%;vertical-align:middle;margin-right:-0.25em}.lity-loader{z-index:9991;color:#fff;position:absolute;top:50%;margin-top:-0.8em;width:100%;text-align:center;font-size:14px;font-family:Arial,Helvetica,sans-serif;opacity:0;-webkit-transition:opacity .3s ease;-o-transition:opacity .3s ease;transition:opacity .3s ease}.lity-loading .lity-loader{opacity:1}.lity-container{z-index:9992;position:relative;text-align:left;vertical-align:middle;display:inline-block;white-space:normal;max-width:100%;max-height:100%;outline:none !important}.lity-content{z-index:9993;width:100%;-webkit-transform:scale(1);-ms-transform:scale(1);-o-transform:scale(1);transform:scale(1);-webkit-transition:-webkit-transform .3s ease;transition:-webkit-transform .3s ease;-o-transition:-o-transform .3s ease;transition:transform .3s ease;transition:transform .3s ease, -webkit-transform .3s ease, -o-transform .3s ease}.lity-loading .lity-content,.lity-closed .lity-content{-webkit-transform:scale(.8);-ms-transform:scale(.8);-o-transform:scale(.8);transform:scale(.8)}.lity-content:after{content:'';position:absolute;left:0;top:0;bottom:0;display:block;right:0;width:auto;height:auto;z-index:-1;-webkit-box-shadow:0 0 8px rgba(0,0,0,0.6);box-shadow:0 0 8px rgba(0,0,0,0.6)}.lity-close{z-index:9994;width:35px;height:35px;position:fixed;right:0;top:0;-webkit-appearance:none;cursor:pointer;text-decoration:none;text-align:center;padding:0;color:#fff;font-style:normal;font-size:35px;font-family:Arial,Baskerville,monospace;line-height:35px;text-shadow:0 1px 2px rgba(0,0,0,0.6);border:0;background:none;outline:none;-webkit-box-shadow:none;box-shadow:none}.lity-close::-moz-focus-inner{border:0;padding:0}.lity-close:hover,.lity-close:focus,.lity-close:active,.lity-close:visited{text-decoration:none;text-align:center;padding:0;color:#fff;font-style:normal;font-size:35px;font-family:Arial,Baskerville,monospace;line-height:35px;text-shadow:0 1px 2px rgba(0,0,0,0.6);border:0;background:none;outline:none;-webkit-box-shadow:none;box-shadow:none}.lity-close:active{top:1px}.lity-image img{max-width:100%;display:block;line-height:0;border:0}.lity-iframe .lity-container,.lity-youtube .lity-container,.lity-vimeo .lity-container,.lity-facebookvideo .lity-container,.lity-googlemaps .lity-container{width:100%;max-width:964px}.lity-iframe-container{width:100%;height:0;padding-top:56.25%;overflow:hidden;pointer-events:all;-webkit-transform:translateZ(0);transform:translateZ(0);-webkit-overflow-scrolling:touch}.lity-iframe-container iframe{position:absolute;display:block;top:0;left:0;width:100%;height:100%;-webkit-box-shadow:0 0 8px rgba(0,0,0,0.6);box-shadow:0 0 8px rgba(0,0,0,0.6);background:#000}.lity-hide{display:none}
--- a/src/static/img/sprite-skin-flat-dark.png
+++ b/src/static/img/sprite-skin-flat-dark.png
--- a/src/static/img/sprite-skin-flat.png
+++ b/src/static/img/sprite-skin-flat.png
--- a/src/static/js/2_jquery-3.4.1.min.js
+++ b/src/static/js/2_jquery-3.4.1.min.js
--- a/src/static/js/3_bootstrap.bundle.min.js
+++ b/src/static/js/3_bootstrap.bundle.min.js
--- a/src/static/js/4_lodash.min.js
+++ b/src/static/js/4_lodash.min.js
--- a/src/static/js/5_inspire-tree.min.js
+++ b/src/static/js/5_inspire-tree.min.js
--- a/src/static/js/6_inspire-tree-dom.min.js
+++ b/src/static/js/6_inspire-tree-dom.min.js
--- a/src/static/js/7_jquery.toast.min.js
+++ b/src/static/js/7_jquery.toast.min.js
--- a/src/static/js/auto-complete.min.js
+++ b/src/static/js/auto-complete.min.js
--- a/src/static/js/bootstrap-colorpicker.min.js
+++ b/src/static/js/bootstrap-colorpicker.min.js
--- a/src/static/js/bricklayer.min.js
+++ b/src/static/js/bricklayer.min.js
@@ -0,0 +1 @@
+!function t(e,n,r){function o(s,u){if(!n[s]){if(!e[s]){var l="function"==typeof require&&require;if(!u&&l)return l(s,!0);if(i)return i(s,!0);var a=new Error("Cannot find module '"+s+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[s]={exports:{}};e[s][0].call(p.exports,function(t){var n=e[s][1][t];return o(n?n:t)},p,p.exports,t,e,n,r)}return n[s].exports}for(var i="function"==typeof require&&require,s=0;s<r.length;s++)o(r[s]);return o}({1:[function(t,e,n){var r,o=this&&this.__extends||function(t,e){function n(){this.constructor=t}for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r]);t.prototype=null===e?Object.create(e):(n.prototype=e.prototype,new n)};!function(t){function e(t){return[].slice.call(t)}function n(t,e,n){if(window.CustomEvent)var r=new CustomEvent(e,{detail:n});else{var r=document.createEvent("CustomEvent");r.initCustomEvent(e,!0,!0,n)}return t.dispatchEvent(r)}var r={rulerClassName:"bricklayer-column-sizer",columnClassName:"bricklayer-column"},i=function(){function t(t){this.element=document.createElement("div"),this.element.className=t}return t.prototype.destroy=function(){this.element.parentNode.removeChild(this.element)},t}(),s=function(t){function e(){t.apply(this,arguments)}return o(e,t),e.prototype.getWidth=function(){this.element.setAttribute("style","\n        display: block;\n        visibility: hidden !important;\n        top: -1000px !important;\n      ");var t=this.element.offsetWidth;return this.element.removeAttribute("style"),t},e}(i),u=function(t){function e(){t.apply(this,arguments)}return o(e,t),e}(i),l=function(){function t(t,e){void 0===e&&(e=r),this.element=t,this.options=e,this.build(),this.buildResponsive()}return t.prototype.append=function(t){var n=this;if(Array.isArray(t))return void t.forEach(function(t){return n.append(t)});var r=this.findMinHeightColumn();this.elements=e(this.elements).concat([t]),this.applyPosition("append",r,t)},t.prototype.prepend=function(t){var n=this;if(Array.isArray(t))return void t.forEach(function(t){return n.prepend(t)});var r=this.findMinHeightColumn();this.elements=[t].concat(e(this.elements)),this.applyPosition("prepend",r,t)},t.prototype.on=function(t,e){return this.element.addEventListener("bricklayer."+t,e),this},t.prototype.redraw=function(){var t=this.columnCount;this.checkColumnCount(!1),this.reorderElements(t),n(this.element,"bricklayer.redraw",{columnCount:t})},t.prototype.destroy=function(){var t=this;this.ruler.destroy(),e(this.elements).forEach(function(e){return t.element.appendChild(e)}),e(this.getColumns()).forEach(function(t){return t.parentNode.removeChild(t)}),n(this.element,"bricklayer.destroy",{})},t.prototype.build=function(){this.ruler=new s(this.options.rulerClassName),this.elements=this.getElementsInOrder(),this.element.insertBefore(this.ruler.element,this.element.firstChild)},t.prototype.buildResponsive=function(){var t=this;window.addEventListener("resize",function(e){return t.checkColumnCount()}),this.checkColumnCount(),this.on("breakpoint",function(e){return t.reorderElements(e.detail.columnCount)}),this.columnCount>=1&&this.reorderElements(this.columnCount)},t.prototype.getColumns=function(){return this.element.querySelectorAll(":scope > ."+this.options.columnClassName)},t.prototype.findMinHeightColumn=function(){var t=e(this.getColumns()),n=t.map(function(t){return t.offsetHeight}),r=Math.min.apply(null,n);return t[n.indexOf(r)]},t.prototype.getElementsInOrder=function(){return this.element.querySelectorAll(":scope > *:not(."+this.options.columnClassName+"):not(."+this.options.rulerClassName+")")},t.prototype.checkColumnCount=function(t){void 0===t&&(t=!0);var e=this.getColumnCount();this.columnCount!==e&&(t&&n(this.element,"bricklayer.breakpoint",{columnCount:e}),this.columnCount=e)},t.prototype.reorderElements=function(t){var n=this;void 0===t&&(t=1),(t==1/0||1>t)&&(t=1);for(var r=e(this.elements).map(function(t){var e=t.parentNode?t.parentNode.removeChild(t):t;return e}),o=this.getColumns(),i=0;i<o.length;i++)o[i].parentNode.removeChild(o[i]);for(var i=0;t>i;i++){var s=new u(this.options.columnClassName).element;this.element.appendChild(s)}r.forEach(function(t){var e=n.findMinHeightColumn();e.appendChild(t)})},t.prototype.getColumnCount=function(){var t=this.element.offsetWidth,e=this.ruler.getWidth();return Math.round(t/e)},t.prototype.applyPosition=function(t,e,r){var o=this,i=function(i){var s=i+t.charAt(0).toUpperCase()+t.substr(1);n(o.element,"bricklayer."+s,{item:r,column:e})};switch(i("before"),t){case"append":e.appendChild(r);break;case"prepend":e.insertBefore(r,e.firstChild)}i("after")},t}();t.Container=l}(r||(r={})),function(t,n){"function"==typeof define&&define.amd?define(function(){return n()}):"undefined"!=typeof window&&t===window?t.Bricklayer=n():"object"==typeof e&&e.exports&&(e.exports=n())}("undefined"!=typeof window?window:this,function(){return r.Container})},{}]},{},[1]);
--- a/src/static/js/d3.v5.min.js
+++ b/src/static/js/d3.v5.min.js
--- a/src/static/js/dom-to-image.min.js
+++ b/src/static/js/dom-to-image.min.js
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
simon987	0e517d5e2b	Fix #81	2020-08-03 20:09:07 -04:00
simon987	8223ef3860	Update USAGE.md	2020-08-03 19:48:49 -04:00
simon987	995a196690	Log user script task, add async arg	2020-08-03 19:44:43 -04:00
simon987	465d017e18	CSS tweaks, fix #87	2020-08-03 19:15:12 -04:00
simon987	ca994d3914	Fix bug with media files, don't encode thumbnail when not necessary	2020-07-26 11:52:48 -04:00
simon987	db2285973f	Configurable column count	2020-07-26 11:50:21 -04:00
simon987	61de9e9f14	Set timeout for HTTP get request	2020-07-25 19:55:27 -04:00
simon987	3015ef0ff4	Increase file preview file	2020-07-25 17:26:17 -04:00
simon987	b55d432841	Fix #65	2020-07-25 09:37:37 -04:00
simon987	ed90a140ce	Update README.md	2020-07-19 14:53:03 -04:00
simon987	052df82373	Fix #83	2020-07-19 13:10:30 -04:00
simon987	5676136777	Remove println that was left accidentally	2020-07-18 20:55:12 -04:00
simon987	c061613302	Fix #76	2020-07-18 19:23:43 -04:00
simon987	d0325fd9b9	Fix for simon987/sist2#85	2020-07-18 18:48:54 -04:00
simon987	e05a6f3863	Fix for #75	2020-07-18 18:46:52 -04:00
simon987	f1690a9cca	Mobi build fix	2020-07-18 13:10:45 -04:00
simon987	100a264413	Don't show MuPDF warnings unless --very-verbose is specified	2020-07-18 10:28:05 -04:00
simon987	29390bb454	Update README	2020-07-18 09:54:36 -04:00
simon987	4d43036ded	Fix simon987/sist2#78	2020-07-18 09:41:39 -04:00
simon987	0b5cdbd130	Fix #79	2020-07-18 09:36:10 -04:00
simon987	53d7695f66	Read .raw thumbnails #80 , fix media probing for some formats	2020-07-18 09:31:42 -04:00
simon987	8d53456404	fix libscan submodule	2020-07-17 20:33:50 -04:00
simon987	cbc08a7cc9	Save ebook renders as jpeg	2020-07-17 20:18:21 -04:00
simon987	e629b4d7d3	Faster comic book parsing, probably fixes #77	2020-07-17 19:10:18 -04:00
simon987	22f7073b39	mobi reading bugfix	2020-07-16 20:30:28 -04:00
simon987	1781a74960	Oops I didn't mean to push this	2020-07-16 19:23:52 -04:00
simon987	db96c95ac7	log fix #73	2020-07-16 19:19:23 -04:00
simon987	7b9fa4cc0a	Fix bad merge...	2020-07-15 21:00:51 -04:00
simon987	5cc1fa86a9	Read embedded thumbnail simon987/sist2#74	2020-07-15 20:56:25 -04:00
simon987	649689ce30	Remove warning when generating stats	2020-07-15 20:41:38 -04:00
simon987	c8536f65a8	Fix memory leak in index	2020-07-15 20:41:09 -04:00
simon987	75b5e249c1	Merge pull request #72 from dpieski/patch-1 Update USAGE.md	2020-07-15 14:37:28 -04:00
Andrew	f49e03ac79	Update USAGE.md added example for Windows to display number of logical processors. Does this same limitation apply to the new `index` threads option?	2020-07-15 13:21:02 -05:00
simon987	a6d2afc8dc	Merge pull request #71 from simon987/web-tag Web tag	2020-07-14 20:23:22 -04:00
simon987	8f8f66ba05	Update README.md	2020-07-14 20:22:03 -04:00
simon987	1d9fcf7105	Manual tagging	2020-07-13 19:18:07 -04:00
simon987	8127745f2b	wip	2020-07-13 19:16:51 -04:00
simon987	230988d6d1	frontend tags	2020-07-13 19:15:59 -04:00
simon987	13f4dbed2d	Handle 429, multi-threaded index module	2020-07-11 17:42:46 -04:00
simon987	ed15e89f45	Fix exec-script --es-url not being passed	2020-06-28 12:41:09 -04:00
simon987	c636d3d921	Set number_of_replicas to 0 by default in elasticsearch	2020-06-26 18:10:51 -04:00
simon987	7e92d4b7d1	refresh index only if user script is ran	2020-06-25 20:48:47 -04:00
simon987	8ffe780ab2	Tag tree fix for #64 , validate required argument in exec-script	2020-06-25 20:11:30 -04:00
simon987	d3c8928fe8	Update readme	2020-06-24 21:06:27 -04:00
simon987	d9f628fca4	Build fix	2020-06-21 16:53:22 -04:00
simon987	68289268c1	Add exif tag	2020-06-21 16:51:14 -04:00
simon987	649c50c465	Update README.md	2020-06-21 14:35:18 -04:00
simon987	7b49a0dc49	Build fix	2020-06-21 12:56:13 -04:00
simon987	eb559b53aa	RAW picture file support	2020-06-21 10:46:11 -04:00
simon987	6d01f9c0df	whoops	2020-06-19 22:12:19 -04:00
simon987	e724fec668	Fix web return codes	2020-06-19 21:41:17 -04:00
simon987	fe5e93b300	Update USAGE.md	2020-06-19 21:29:09 -04:00
simon987	ecad85fd7d	version bump	2020-06-19 21:10:03 -04:00
simon987	74cc898259	Fix tag display issue	2020-06-19 21:07:19 -04:00
simon987	dc2e4443c4	Add exec-script command	2020-06-19 21:07:19 -04:00
simon987	1a64431b52	Merge pull request #63 from dpieski/patch-3 Correct typos in example	2020-06-19 18:26:10 -04:00
Andrew	9bad515e06	Correct typos in example Correct typos in examples.	2020-06-19 17:22:02 -05:00
simon987	648559cedb	Update README.md	2020-06-17 13:25:20 -04:00
simon987	3e6cd9cd5c	Merge pull request #60 from dpieski/patch-2 update Usage.md	2020-06-17 13:04:46 -04:00
simon987	f249992798	Update scripting.md	2020-06-17 13:00:07 -04:00
Andrew	e9645ecdaa	update Usage.md Fixing a link.	2020-06-17 10:58:25 -05:00
simon987	046edea0e2	Handle special characters in file paths	2020-06-10 19:45:36 -04:00
simon987	a011b7e97b	Fragment size setting	2020-06-09 21:40:53 -04:00
simon987	8c1c1697e0	Fix file wordexp in some paths #59	2020-06-05 19:41:02 -04:00
simon987	018b49fa4c	Fix csv_escape #58	2020-06-05 19:13:03 -04:00
simon987	27b4e6403e	Re-enable path autocomplete #54	2020-06-02 19:46:58 -04:00
simon987	13fdbd9e69	Fix for ES 7.7 #54	2020-06-01 18:14:34 -04:00
simon987	5e7fdaf8dd	Update issue-template.md	2020-06-01 10:45:43 -04:00
simon987	19d5c8ac9f	Update issue-template.md	2020-05-29 18:19:21 -04:00
simon987	99497049a8	Merge pull request #53 from dpieski/patch-1 Update README	2020-05-29 18:16:13 -04:00
Andrew	1a3181d78b	Update README changed case of path in a link to the usage guide to fix 404 error.	2020-05-29 15:37:20 -05:00
simon987	449aa77c8f	Fix for unknown mime inside archives	2020-05-25 17:36:04 -04:00
simon987	3058c55510	Memory leak fix #37	2020-05-24 15:42:42 -04:00
simon987	dedf9287b2	Fix name separation in `--archive list` mode	2020-05-24 14:36:59 -04:00
simon987	ab199b0c0c	Remove arc_reset() function because seek() inside archive doesn't work	2020-05-24 14:18:31 -04:00
simon987	c4fbae123e	Better support for media files inside archives	2020-05-24 14:10:23 -04:00
simon987	dd2397ef5c	handle .tgz #44 , ignore files inside archives for stats page	2020-05-24 10:10:28 -04:00
simon987	ee0f71f4d3	fix compile warning	2020-05-17 15:00:56 -04:00
simon987	0bbb96b149	Merge pull request #51 from simon987/stats Stats page	2020-05-17 14:49:28 -04:00
simon987	78f6e16701	image	2020-05-17 12:47:45 -04:00
simon987	4625bca9a9	stats	2020-05-17 12:47:02 -04:00
simon987	f2ae653886	Revert "wip" This reverts commit `5686bc86`	2020-05-16 08:16:49 -04:00
simon987	5686bc864d	wip	2020-05-13 17:37:40 -04:00
simon987	cf513b4ad8	Escape invalid UTF8 characters simon987/sist2#44 , increase magic buffer size	2020-05-12 19:28:02 -04:00
simon987	013423424e	UTF-8 fix attempt w/ libarchive (#44 )	2020-05-10 19:52:42 -04:00
simon987	16514fd6b0	Option to search in path #49	2020-05-09 22:00:22 -04:00
simon987	27509f97e1	Update USAGE.md	2020-05-08 19:08:46 -04:00
simon987	4c540eae1c	Update USAGE.md	2020-05-08 19:07:45 -04:00
simon987	d2b53ff6fc	Update README.md	2020-05-08 18:32:32 -04:00
simon987	0ef4292abf	Fix duplicate tag problem (simon987/sist2#48 )	2020-05-05 20:20:10 -04:00
simon987	e6fde38c24	Load defaults when LocalStorage is outdated	2020-05-03 08:13:25 -04:00
simon987	5fa343d40f	fix version typo	2020-05-03 08:10:28 -04:00
simon987	7ee1374802	oops	2020-04-30 21:21:48 -04:00
simon987	bd9e56829c	Support for markup files	2020-04-30 20:21:09 -04:00
simon987	718169345e	gzip artifacts in CI	2020-04-21 19:34:46 -04:00
simon987	5a6aa763ca	build fix	2020-04-21 18:50:32 -04:00
simon987	695d9abd83	revert debug hard-coded listen address	2020-04-21 15:52:35 -04:00
simon987	e436af7b2a	2.0 (#46 ) * extract scan code to libscan, (wip) * submodules * replace curl with mongoose (wip) * replace onion with mongoose (wip) * replace onion with mongoose (wip) * It compiles! (I think) * Update readme * Entirely remove libonion (WIP) * unscramble submodules * recover screenshot * Update mappings * Bug fixes * update * media meta fix * memory fixes * More bug fixes... * Bug fix w/ libmagic & vfile * libmagic fix (again) * Better lightbox, better video handler, random reloads fix * Use svg for info icon * re-enable http auth * mobi support #41, fix logs * Update README & cleanup	2020-04-21 14:42:20 -04:00
simon987	4501a7810f	Update issue-template.md	2020-04-11 07:33:33 -04:00
simon987	e36761fa6a	Update issue templates	2020-04-11 07:28:48 -04:00
simon987	fe53b79d56	Fix warnings	2020-03-25 08:18:59 -04:00
simon987	09615bbed6	Update dependencies	2020-03-24 14:30:23 -04:00
simon987	a2be9b955c	Fix build errors	2020-03-24 11:49:13 -04:00
simon987	9298bd2d9d	CI fix...	2020-03-24 10:09:33 -04:00
simon987	317034ba21	teamcity automation attempt	2020-03-24 10:01:27 -04:00
simon987	0505303503	text_buffer bug fixes & Sort option	2020-03-20 20:54:22 -04:00
simon987	6e5772f13b	Errors cleanup	2020-03-20 10:05:10 -04:00
simon987	ccccdb3b78	Fixes #38	2020-03-13 16:35:11 -04:00
simon987	12d17acf4f	UI fixes	2020-03-06 12:27:38 -05:00
simon987	48b56cdb7b	I forgot to commit this somehow	2020-03-06 10:32:05 -05:00
simon987	048f707f80	Fix buffer overflow in json parse function (index module)	2020-03-06 10:17:21 -05:00
				`@@ -0,0 +1 @@`
				.bricklayer{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:start;-webkit-align-items:flex-start;-ms-flex-align:start;align-items:flex-start;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;-webkit-flex-wrap:wrap;-ms-flex-wrap:wrap;flex-wrap:wrap}.bricklayer-column-sizer{width:100%;display:none}@media screen and (min-width:640px){.bricklayer-column-sizer{width:50%}}@media screen and (min-width:980px){.bricklayer-column-sizer{width:33.333%}}@media screen and (min-width:1200px){.bricklayer-column-sizer{width:25%}}.bricklayer-column{-webkit-box-flex:1;-webkit-flex:1;-ms-flex:1;flex:1;padding-left:5px;padding-right:5px}
				`@@ -0,0 +1 @@`
				!function t(e,n,r){function o(s,u){if(!n[s]){if(!e[s]){var l="function"==typeof require&&require;if(!u&&l)return l(s,!0);if(i)return i(s,!0);var a=new Error("Cannot find module '"+s+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[s]={exports:{}};e[s][0].call(p.exports,function(t){var n=e[s][1][t];return o(n?n:t)},p,p.exports,t,e,n,r)}return n[s].exports}for(var i="function"==typeof require&&require,s=0;s<r.length;s++)o(r[s]);return o}({1:[function(t,e,n){var r,o=this&&this.__extends\|\|function(t,e){function n(){this.constructor=t}for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r]);t.prototype=null===e?Object.create(e):(n.prototype=e.prototype,new n)};!function(t){function e(t){return[].slice.call(t)}function n(t,e,n){if(window.CustomEvent)var r=new CustomEvent(e,{detail:n});else{var r=document.createEvent("CustomEvent");r.initCustomEvent(e,!0,!0,n)}return t.dispatchEvent(r)}var r={rulerClassName:"bricklayer-column-sizer",columnClassName:"bricklayer-column"},i=function(){function t(t){this.element=document.createElement("div"),this.element.className=t}return t.prototype.destroy=function(){this.element.parentNode.removeChild(this.element)},t}(),s=function(t){function e(){t.apply(this,arguments)}return o(e,t),e.prototype.getWidth=function(){this.element.setAttribute("style","\n display: block;\n visibility: hidden !important;\n top: -1000px !important;\n ");var t=this.element.offsetWidth;return this.element.removeAttribute("style"),t},e}(i),u=function(t){function e(){t.apply(this,arguments)}return o(e,t),e}(i),l=function(){function t(t,e){void 0===e&&(e=r),this.element=t,this.options=e,this.build(),this.buildResponsive()}return t.prototype.append=function(t){var n=this;if(Array.isArray(t))return void t.forEach(function(t){return n.append(t)});var r=this.findMinHeightColumn();this.elements=e(this.elements).concat([t]),this.applyPosition("append",r,t)},t.prototype.prepend=function(t){var n=this;if(Array.isArray(t))return void t.forEach(function(t){return n.prepend(t)});var r=this.findMinHeightColumn();this.elements=[t].concat(e(this.elements)),this.applyPosition("prepend",r,t)},t.prototype.on=function(t,e){return this.element.addEventListener("bricklayer."+t,e),this},t.prototype.redraw=function(){var t=this.columnCount;this.checkColumnCount(!1),this.reorderElements(t),n(this.element,"bricklayer.redraw",{columnCount:t})},t.prototype.destroy=function(){var t=this;this.ruler.destroy(),e(this.elements).forEach(function(e){return t.element.appendChild(e)}),e(this.getColumns()).forEach(function(t){return t.parentNode.removeChild(t)}),n(this.element,"bricklayer.destroy",{})},t.prototype.build=function(){this.ruler=new s(this.options.rulerClassName),this.elements=this.getElementsInOrder(),this.element.insertBefore(this.ruler.element,this.element.firstChild)},t.prototype.buildResponsive=function(){var t=this;window.addEventListener("resize",function(e){return t.checkColumnCount()}),this.checkColumnCount(),this.on("breakpoint",function(e){return t.reorderElements(e.detail.columnCount)}),this.columnCount>=1&&this.reorderElements(this.columnCount)},t.prototype.getColumns=function(){return this.element.querySelectorAll(":scope > ."+this.options.columnClassName)},t.prototype.findMinHeightColumn=function(){var t=e(this.getColumns()),n=t.map(function(t){return t.offsetHeight}),r=Math.min.apply(null,n);return t[n.indexOf(r)]},t.prototype.getElementsInOrder=function(){return this.element.querySelectorAll(":scope > *:not(."+this.options.columnClassName+"):not(."+this.options.rulerClassName+")")},t.prototype.checkColumnCount=function(t){void 0===t&&(t=!0);var e=this.getColumnCount();this.columnCount!==e&&(t&&n(this.element,"bricklayer.breakpoint",{columnCount:e}),this.columnCount=e)},t.prototype.reorderElements=function(t){var n=this;void 0===t&&(t=1),(t==1/0\|\|1>t)&&(t=1);for(var r=e(this.elements).map(function(t){var e=t.parentNode?t.parentNode.removeChild(t):t;return e}),o=this.getColumns(),i=0;i<o.length;i++)o[i].parentNode.removeChild(o[i]);for(var i=0;t>i;i++){var s=new u(this.options.columnClassName).element;this.element.appendChild(s)}r.forEach(function(t){var e=n.findMinHeightColumn();e.appendChild(t)})},t.prototype.getColumnCount=function(){var t=this.element.offsetWidth,e=this.ruler.getWidth();return Math.round(t/e)},t.prototype.applyPosition=function(t,e,r){var o=this,i=function(i){var s=i+t.charAt(0).toUpperCase()+t.substr(1);n(o.element,"bricklayer."+s,{item:r,column:e})};switch(i("before"),t){case"append":e.appendChild(r);break;case"prepend":e.insertBefore(r,e.firstChild)}i("after")},t}();t.Container=l}(r\|\|(r={})),function(t,n){"function"==typeof define&&define.amd?define(function(){return n()}):"undefined"!=typeof window&&t===window?t.Bricklayer=n():"object"==typeof e&&e.exports&&(e.exports=n())}("undefined"!=typeof window?window:this,function(){return r.Container})},{}]},{},[1]);