Fix #521

Update README.md

.dockerignore

@@ -0,0 +1,43 @@
+.idea/
+*.sist2
+docs/
+test_i/
+test_i_inc/
+Testing/
+.drone.yml
+**/cmake_install.cmake
+**/CMakeCache.txt
+**/CMakeFiles/
+.cmake
+LICENSE
+Makefile
+**/*.md
+**/*.cbp
+VERSION
+**/node_modules/
+sist2-*-linux-debug
+sist2-*-linux
+sist2_debug
+sist2
+**/libscan-test-files
+**/scan_ub_test
+**/scan_a_test
+**/scan_test
+**/ext_ffmpeg
+**/ext_libmobi
+**/ext_libwpd
+**/core
+*.a
+tmp_scan/
+Dockerfile
+Dockerfile.arm64
+docker-compose.yml
+state.db
+*-journal
+build/
+__pycache__/
+sist2-vue/dist
+sist2-admin/frontend/dist
+*.fts
+.git
+third-party/libscan/third-party/ext_*/*

.drone.yml

@@ -0,0 +1,108 @@
+kind: pipeline
+type: docker
+name: amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
+  - name: build
+    image: sist2app/sist2-build
+    depends_on:
+    - submodules
+    commands:
+      - ./scripts/build.sh
+  - name: scp files
+    depends_on:
+      - build
+    image: appleboy/drone-scp
+    settings:
+      host:
+        from_secret: SSH_HOST
+      port:
+        from_secret: SSH_PORT
+      user:
+        from_secret: SSH_USER
+      key:
+        from_secret: SSH_KEY
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      source:
+        - ./VERSION
+        - ./sist2-x64-linux
+        - ./sist2-x64-linux-debug
+
+---
+kind: pipeline
+type: docker
+name: arm64
+
+platform:
+  arch: arm64
+
+steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile.arm64
+      auto_tag: true
+      auto_tag_suffix: arm64-linux
+      when:
+        event:
+          - tag
+  - name: build
+    image: sist2app/sist2-build-arm64
+    depends_on:
+      - submodules
+    commands:
+      - ./scripts/build_arm64.sh
+  - name: scp files
+    depends_on:
+      - build
+    image: appleboy/drone-scp
+    settings:
+      host:
+        from_secret: SSH_HOST
+      port:
+        from_secret: SSH_PORT
+      user:
+        from_secret: SSH_USER
+      key:
+        from_secret: SSH_KEY
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      source:
+        - ./sist2-arm64-linux
+        - ./sist2-arm64-linux-debug

.gitattributes

@@ -1,3 +0,0 @@
-CMakeModules/* linguist-vendored
-web/js/*.min.js linguist-vendored
-web/css/*.min.css linguist-vendored

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: "🐞 Bug Report"
+about: Submit a bug report 
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Device Information (please complete the following information):**
+ - OS: `[e.g., Ubuntu 20.04, WSL2]`
+ - Deployment: `[Linux, Linux ARM64 or Docker]`
+ - Browser *(if relevant)*: `[e.g., chrome, safari]`
+ - SIST2 Version: `[e.g., v2.9.0]`
+ - Elasticsearch Version *(if relevant)* : ``
+
+**Command with arguments** 
+<!-- `ex: "scan ~/Documents -o ./i2 --threads 3 -q 1.0` -->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**Steps To Reproduce**
+Please be specific!
+1. Go to '...'
+2. Click on '....'
+3. etc.
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Actual Behavior**
+<!-- A clear and concise description of what actually happens. -->
+
+**Screenshots**
+<!-- If applicable, add screenshots to help explain your problem. -->
+
+**Additional context**
+<!-- Add any other context about the problem here. If applicable, please include why you think the bug is occurring and/or troubleshooting you have already performed. -->
+<!-- If the issue is related to the `scan` module, please attach the files necessary to reproduce the error or email them to me[at]simon987.net. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: SIST2 Documentation
+    url: https://github.com/simon987/sist2/blob/master/docs/USAGE.md
+    about: Check out the SIST2 documentation for answers to common questions

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: "🚀 Feature Request"
+about: Suggest an idea for SIST2
+title: ''
+assignees: ''
+
+---
+**Which SIST2 component is your Feature Request related to?**
+<!-- e.g., Scan, Index, or Web? -->
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. e.g., "I'm always frustrated when [...]" -->
+
+**What would you like to see happen?**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/issue-template.md

@@ -0,0 +1,18 @@
+---
+name: Issue template
+about: General
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+sist2 version:
+
+Platform (Linux or Docker, x86-64 or arm64):
+
+Elasticsearch version:
+
+Command with arguments:  `ex: "scan ~/Documents -o ./i2 --threads 3 -q 1.0`
+
+If the issue is related to the `scan` module, please attach the files necessary to reproduce the error or email them to me[at]simon987.net.

.gitignore

@@ -1,18 +1,49 @@
 .idea
 thumbs
-test
 *.cbp
 CMakeCache.txt
 CMakeFiles
+cmake-build-default-event-trace
 cmake-build-debug
 cmake_install.cmake
 Makefile
 *.out
 LOG
 sist2*
-index.sist2/
+!sist2-vue/
+!sist2-admin
+!sist2_admin
+!sist2.py
+*.sist2/
 bundle*.css
 bundle.js
 *.a
 vgcore.*
 build/
+third-party/argparse
+*.idx/
+VERSION
+git_hash.h
+Testing/
+test_i
+test_i_inc
+node_modules/
+.cmake/
+i_inc/
+state.db
+*.pyc
+!sist2-admin/frontend/dist
+*.js.map
+sist2-vue/dist
+sist2-admin/frontend/dist
+.ninja_deps
+.ninja_log
+build.ninja
+src/web/static_generated.c
+src/magic_generated.c
+src/index/static_generated.c
+*.sist2
+*-shm
+*-journal
+.vscode
+*.fts

.gitmodules

@@ -1,45 +1,15 @@
-[submodule "argparse"]
+[submodule "third-party/argparse"]
-	path = argparse
+	path = third-party/argparse
-	url = https://github.com/cofyc/argparse
+	url = https://github.com/simon987/argparse
-[submodule "cJSON"]
+[submodule "third-party/libscan/third-party/utf8.h"]
-	path = cJSON
+	path = third-party/libscan/third-party/utf8.h
-	url = https://github.com/DaveGamble/cJSON
-[submodule "lmdb"]
-	path = lmdb
-	url = https://github.com/LMDB/lmdb
-[submodule "utf8.h"]
-	path = utf8.h
 	url = https://github.com/sheredom/utf8.h
-[submodule "lib/bzip2-1.0.6"]
+[submodule "third-party/libscan/third-party/antiword"]
-	path = lib/bzip2-1.0.6
+	path = third-party/libscan/third-party/antiword
-	url = https://github.com/enthought/bzip2-1.0.6
+	url = https://github.com/simon987/antiword
-[submodule "lib/libmagic"]
+[submodule "third-party/libscan/third-party/libmobi"]
-	path = lib/libmagic
+	path = third-party/libscan/third-party/libmobi
-	url = https://github.com/threatstack/libmagic
+	url = https://github.com/bfabiszewski/libmobi
-[submodule "lib/harfbuzz"]
+[submodule "third-party/libscan/libscan-test-files"]
-	path = lib/harfbuzz
+	path = third-party/libscan/libscan-test-files
-	url = https://github.com/harfbuzz/harfbuzz
+	url = https://github.com/simon987/libscan-test-files
-[submodule "lib/openjpeg"]
-	path = lib/openjpeg
-	url = https://github.com/uclouvain/openjpeg
-[submodule "lib/ffmpeg"]
-	path = lib/ffmpeg
-	url = https://git.ffmpeg.org/ffmpeg.git
-[submodule "lib/onion"]
-	path = lib/onion
-	url = https://github.com/davidmoreno/onion
-[submodule "lib/mupdf"]
-	path = lib/mupdf
-	url = git://git.ghostscript.com/mupdf.git
-[submodule "lib/tesseract"]
-	path = lib/tesseract
-	url = https://github.com/tesseract-ocr/tesseract
-[submodule "lib/leptonica"]
-	path = lib/leptonica
-	url = https://github.com/danbloomberg/leptonica
-[submodule "lib/libtiff"]
-	path = lib/libtiff
-	url = https://gitlab.com/libtiff/libtiff
-[submodule "lib/libpng"]
-	path = lib/libpng
-	url = https://github.com/glennrp/libpng

.teamcity/settings.kts

@@ -1,69 +0,0 @@
-import jetbrains.buildServer.configs.kotlin.v2019_2.*
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ExecBuildStep
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.exec
-import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs
-import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot
-
-/*
-The settings script is an entry point for defining a TeamCity
-project hierarchy. The script should contain a single call to the
-project() function with a Project instance or an init function as
-an argument.
-
-VcsRoots, BuildTypes, Templates, and subprojects can be
-registered inside the project using the vcsRoot(), buildType(),
-template(), and subProject() methods respectively.
-
-To debug settings scripts in command-line, run the
-
-    mvnDebug org.jetbrains.teamcity:teamcity-configs-maven-plugin:generate
-
-command and attach your debugger to the port 8000.
-
-To debug in IntelliJ Idea, open the 'Maven Projects' tool window (View
--> Tool Windows -> Maven Projects), find the generate task node
-(Plugins -> teamcity-configs -> teamcity-configs:generate), the
-'Debug' option is available in the context menu for the task.
-*/
-
-version = "2019.2"
-
-project {
-
-    vcsRoot(HttpsGithubComSimon987sist2refsHeadsMaster)
-
-    buildType(Build)
-}
-
-object Build : BuildType({
-    name = "Build"
-
-    artifactRules = """
-        sist2
-        sist2_scan
-    """.trimIndent()
-
-    vcs {
-        root(HttpsGithubComSimon987sist2refsHeadsMaster)
-    }
-
-    steps {
-        exec {
-            name = "Build"
-            path = "./ci/build.sh"
-            dockerImage = "simon987/general_ci"
-            dockerImagePlatform = ExecBuildStep.ImagePlatform.Linux
-            dockerPull = true
-        }
-    }
-
-    triggers {
-        vcs {
-        }
-    }
-})
-
-object HttpsGithubComSimon987sist2refsHeadsMaster : GitVcsRoot({
-    name = "https://github.com/simon987/sist2#refs/heads/master"
-    url = "https://github.com/simon987/sist2"
-})

CMakeLists.txt

@@ -1,141 +1,165 @@
 cmake_minimum_required(VERSION 3.7)
+
+project(sist2)
 set(CMAKE_C_STANDARD 11)
 
-project(sist2 C)
+option(SIST_DEBUG "Build a debug executable" on)
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules")
+option(SIST_FAST "Enable more optimisation flags" off)
+option(SIST_DEBUG_INFO "Turn on debug information in web interface" on)
+
+add_compile_definitions(
+        "SIST_PLATFORM=${SIST_PLATFORM}"
+)
+
+if (SIST_DEBUG)
+    add_compile_definitions(
+            "SIST_DEBUG=${SIST_DEBUG}"
+    )
+    set(VCPKG_BUILD_TYPE debug)
+else ()
+    set(VCPKG_BUILD_TYPE release)
+endif ()
+
+if (SIST_DEBUG_INFO)
+    add_compile_definitions(
+            "SIST_DEBUG_INFO=${SIST_DEBUG_INFO}"
+    )
+endif ()
+
+
+add_subdirectory(third-party/libscan)
+set(ARGPARSE_SHARED off)
+add_subdirectory(third-party/argparse)
 
 add_executable(
         sist2
+        # argparse
+        third-party/argparse/argparse.h third-party/argparse/argparse.c
+
         src/main.c
         src/sist.h
         src/io/walk.h src/io/walk.c
-        src/parsing/media.h src/parsing/media.c
-        src/parsing/pdf.h src/parsing/pdf.c
-        src/io/store.h src/io/store.c
         src/tpool.h src/tpool.c
         src/parsing/parse.h src/parsing/parse.c
+        src/parsing/magic_util.c src/parsing/magic_util.h
         src/io/serialize.h src/io/serialize.c
         src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
-        src/parsing/text.h src/parsing/text.c
         src/index/web.c src/index/web.h
         src/web/serve.c src/web/serve.h
-        src/web/auth_basic.h src/web/auth_basic.c
+        src/web/web_util.c src/web/web_util.h
         src/index/elastic.c src/index/elastic.h
         src/util.c src/util.h
-        src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
+        src/ctx.c src/ctx.h
-        src/parsing/arc.c src/parsing/arc.h
+        src/types.h
-        src/parsing/doc.c src/parsing/doc.h
         src/log.c src/log.h
-
-        # argparse
-        argparse/argparse.h argparse/argparse.c
-
-        # cJSON
-        cJSON/cJSON.h cJSON/cJSON.c
-
-        # LMDB
-        lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
-        lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
         src/cli.c src/cli.h
+        src/database/database.c src/database/database.h
+        src/parsing/fs_util.h
 
-        # utf8.h
+        src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
-        utf8.h/utf8.h
+
-)
+        src/database/database_stats.c
+        src/database/database_schema.c
+        src/database/database_fts.c
+        src/web/web_fts.c
+        src/database/database_embeddings.c)
+set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
+
+target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
+set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
 
 find_package(PkgConfig REQUIRED)
-set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
 
-find_package(Freetype REQUIRED)
+find_package(cJSON CONFIG REQUIRED)
+find_package(unofficial-mongoose CONFIG REQUIRED)
+find_package(CURL CONFIG REQUIRED)
+find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
+find_package(unofficial-sqlite3 CONFIG REQUIRED)
+find_package(OpenBLAS CONFIG REQUIRED)
 
-pkg_check_modules(GLIB REQUIRED glib-2.0)
-pkg_check_modules(GOBJECT REQUIRED gobject-2.0)
-pkg_check_modules(UUID REQUIRED uuid)
-
-add_definitions(${UUID_CFLAGS_OTHER})
-add_definitions(${GLIB_CFLAGS_OTHER})
-add_definitions(${GOBJECT_CFLAGS_OTHER})
-add_definitions(${FREETYPE_CFLAGS_OTHER})
-
-list(REMOVE_ITEM GLIB_LIBRARIES pcre)
-list(REMOVE_ITEM GOBJECT_LIBRARIES pcre)
-list(REMOVE_ITEM UUID_LIBRARIES pcre)
 
 target_include_directories(
         sist2 PUBLIC
-        ${GOBJECT_INCLUDE_DIRS}
+        ${CMAKE_SOURCE_DIR}/third-party/onion/src/
-        ${GLIB_INCLUDE_DIRS}
+        ${CMAKE_SOURCE_DIR}/third-party/utf8.h/
-        ${PROJECT_SOURCE_DIR}/lib/ffmpeg/
+        ${CMAKE_SOURCE_DIR}/third-party/libscan/
-        ${FREETYPE_INCLUDE_DIRS}
+        ${CMAKE_SOURCE_DIR}/
-        ${UUID_INCLUDE_DIRS}
-        ${PROJECT_SOURCE_DIR}/
-        ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
-        ${PROJECT_SOURCE_DIR}/lib/onion/src/
-        ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
-        ${PROJECT_SOURCE_DIR}/include/
-        /usr/include/libxml2/
-        ${PROJECT_SOURCE_DIR}/lib/tesseract/include/
-)
-target_link_directories(
-        sist2 PUBLIC
-        ${UUID_LIBRARY_DIRS}
 )
 
-target_compile_options(sist2
+target_compile_options(
+        sist2
         PRIVATE
-        -Ofast
-#        -march=native
         -fPIC
-        -fno-stack-protector
+)
-        -fomit-frame-pointer
-        )
 
-TARGET_LINK_LIBRARIES(
+if (SIST_DEBUG)
+    target_compile_options(
+            sist2
+            PRIVATE
+            -g
+            -fstack-protector
+            -fno-omit-frame-pointer
+            -fsanitize=address
+            -fno-inline
+            #            -O2
+    )
+    target_link_options(
+            sist2
+            PRIVATE
+            -fsanitize=address
+            -static-libasan
+    )
+    set_target_properties(
+            sist2
+            PROPERTIES
+            OUTPUT_NAME sist2_debug
+    )
+elseif (SIST_FAST)
+    target_compile_options(
+            sist2
+            PRIVATE
+
+            -Ofast
+            -march=native
+            -fno-stack-protector
+            -fomit-frame-pointer
+            -freciprocal-math
+    )
+else ()
+    target_compile_options(
+            sist2
+            PRIVATE
+
+            -Ofast
+#            -g
+            -fno-stack-protector
+            -fomit-frame-pointer
+            -w
+    )
+endif ()
+
+add_dependencies(
+        sist2
+        scan
+        argparse
+)
+
+target_link_libraries(
         sist2
 
-        ${GLIB_LIBRARIES}
+        m
-        ${GOBJECT_LIBRARIES}
+        z
-        ${UUID_LIBRARIES}
+        argparse
-
+        unofficial::mongoose::mongoose
-        # ffmpeg
+        CURL::libcurl
-        ${PROJECT_SOURCE_DIR}/lib/libavcodec.a
-        ${PROJECT_SOURCE_DIR}/lib/libavformat.a
-        ${PROJECT_SOURCE_DIR}/lib/libavutil.a
-        ${PROJECT_SOURCE_DIR}/lib/libswscale.a
-        ${PROJECT_SOURCE_DIR}/lib/libswresample.a
-
-        # mupdf
-        ${PROJECT_SOURCE_DIR}/lib/libmupdf.a
-        ${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
-
-        # onion
-        ${PROJECT_SOURCE_DIR}/lib/libonion_static.a
 
         pthread
 
-        m
+        scan
-        bz2
-        ${PROJECT_SOURCE_DIR}/lib/libmagic.a
-        ${PROJECT_SOURCE_DIR}/lib/libharfbuzz.a
-        ${PROJECT_SOURCE_DIR}/lib/libopenjp2.a
-        freetype
-        archive
 
-        xml2
+        ${MAGIC_LIB}
-        ${PROJECT_SOURCE_DIR}/lib/libopc/libmce.a
+        unofficial::sqlite3::sqlite3
-        ${PROJECT_SOURCE_DIR}/lib/libopc/libopc.a
+        OpenBLAS::OpenBLAS
-        ${PROJECT_SOURCE_DIR}/lib/libopc/libplib.a
-
-        ${PROJECT_SOURCE_DIR}/lib/libtesseract.a
-        ${PROJECT_SOURCE_DIR}/lib/liblept.a
-        ${PROJECT_SOURCE_DIR}/lib/libtiff.a
-        ${PROJECT_SOURCE_DIR}/lib/libpng16.a
-        stdc++
-
-        # curl
-        ${PROJECT_SOURCE_DIR}/lib/libcurl.a
-        ${PROJECT_SOURCE_DIR}/lib/libcrypto.a
-        ${PROJECT_SOURCE_DIR}/lib/libssl.a
-        dl
 )
 
 add_custom_target(

CMakeModules/FindFFmpeg.cmake

@@ -1,80 +0,0 @@
-# - Try to find ffmpeg libraries (libavcodec, libavformat and libavutil)
-# Once done this will define
-#
-# FFMPEG_FOUND - system has ffmpeg or libav
-# FFMPEG_INCLUDE_DIR - the ffmpeg include directory
-# FFMPEG_LIBRARIES - Link these to use ffmpeg
-# FFMPEG_LIBAVCODEC
-# FFMPEG_LIBAVFORMAT
-# FFMPEG_LIBAVUTIL
-#
-# Copyright (c) 2008 Andreas Schneider <mail@cynapses.org>
-# Modified for other libraries by Lasse Kärkkäinen <tronic>
-# Modified for Hedgewars by Stepik777
-#
-# Redistribution and use is allowed according to the terms of the New
-# BSD license.
-#
-
-if (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
-# in cache already
-set(FFMPEG_FOUND TRUE)
-else (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
-# use pkg-config to get the directories and then use these values
-# in the FIND_PATH() and FIND_LIBRARY() calls
-find_package(PkgConfig)
-if (PKG_CONFIG_FOUND)
-pkg_check_modules(_FFMPEG_AVCODEC libavcodec)
-pkg_check_modules(_FFMPEG_AVFORMAT libavformat)
-pkg_check_modules(_FFMPEG_AVUTIL libavutil)
-endif (PKG_CONFIG_FOUND)
-
-find_path(FFMPEG_AVCODEC_INCLUDE_DIR
-NAMES libavcodec/avcodec.h
-PATHS ${_FFMPEG_AVCODEC_INCLUDE_DIRS} /usr/include /usr/local/include /opt/local/include /sw/include
-PATH_SUFFIXES ffmpeg libav
-)
-
-find_library(FFMPEG_LIBAVCODEC
-NAMES avcodec
-PATHS ${_FFMPEG_AVCODEC_LIBRARY_DIRS} /usr/lib /usr/local/lib /opt/local/lib /sw/lib
-)
-
-find_library(FFMPEG_LIBAVFORMAT
-NAMES avformat
-PATHS ${_FFMPEG_AVFORMAT_LIBRARY_DIRS} /usr/lib /usr/local/lib /opt/local/lib /sw/lib
-)
-
-find_library(FFMPEG_LIBAVUTIL
-NAMES avutil
-PATHS ${_FFMPEG_AVUTIL_LIBRARY_DIRS} /usr/lib /usr/local/lib /opt/local/lib /sw/lib
-)
-
-if (FFMPEG_LIBAVCODEC AND FFMPEG_LIBAVFORMAT)
-set(FFMPEG_FOUND TRUE)
-endif()
-
-if (FFMPEG_FOUND)
-set(FFMPEG_INCLUDE_DIR ${FFMPEG_AVCODEC_INCLUDE_DIR})
-
-set(FFMPEG_LIBRARIES
-${FFMPEG_LIBAVCODEC}
-${FFMPEG_LIBAVFORMAT}
-${FFMPEG_LIBAVUTIL}
-)
-
-endif (FFMPEG_FOUND)
-
-if (FFMPEG_FOUND)
-if (NOT FFMPEG_FIND_QUIETLY)
-message(STATUS "Found FFMPEG or Libav: ${FFMPEG_LIBRARIES}, ${FFMPEG_INCLUDE_DIR}")
-endif (NOT FFMPEG_FIND_QUIETLY)
-else (FFMPEG_FOUND)
-if (FFMPEG_FIND_REQUIRED)
-message(FATAL_ERROR "Could not find libavcodec or libavformat or libavutil")
-endif (FFMPEG_FIND_REQUIRED)
-endif (FFMPEG_FOUND)
-
-endif (FFMPEG_LIBRARIES AND FFMPEG_INCLUDE_DIR)
-
-

CMakeModules/FindLibMagic.cmake

@@ -1,100 +0,0 @@
-
-#-------------------------------------------------------------------------------
-# Copyright (c) 2013-2013, Lars Baehren <lbaehren@gmail.com>
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#  * Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#-------------------------------------------------------------------------------
-
-# - Check for the presence of LIBMAGIC
-#
-# The following variables are set when LIBMAGIC is found:
-#  LIBMAGIC_FOUND      = Set to true, if all components of LIBMAGIC have been
-#                        found.
-#  LIBMAGIC_INCLUDES   = Include path for the header files of LIBMAGIC
-#  LIBMAGIC_LIBRARIES  = Link these to use LIBMAGIC
-#  LIBMAGIC_LFLAGS     = Linker flags (optional)
-
-if (NOT LIBMAGIC_FOUND)
-
-  if (NOT LIBMAGIC_ROOT_DIR)
-    set (LIBMAGIC_ROOT_DIR ${CMAKE_INSTALL_PREFIX})
-  endif (NOT LIBMAGIC_ROOT_DIR)
-
-  ##____________________________________________________________________________
-  ## Check for the header files
-
-  find_path (LIBMAGIC_FILE_H
-    NAMES file/file.h
-    HINTS ${LIBMAGIC_ROOT_DIR} ${CMAKE_INSTALL_PREFIX}
-    PATH_SUFFIXES include
-    )
-  if (LIBMAGIC_FILE_H)
-    list (APPEND LIBMAGIC_INCLUDES ${LIBMAGIC_FILE_H})
-  endif (LIBMAGIC_FILE_H)
-
-  find_path (LIBMAGIC_MAGIC_H
-    NAMES magic.h
-    HINTS ${LIBMAGIC_ROOT_DIR} ${CMAKE_INSTALL_PREFIX}
-    PATH_SUFFIXES include include/linux
-    )
-  if (LIBMAGIC_MAGIC_H)
-    list (APPEND LIBMAGIC_INCLUDES ${LIBMAGIC_MAGIC_H})
-  endif (LIBMAGIC_MAGIC_H)
-
-  list (REMOVE_DUPLICATES LIBMAGIC_INCLUDES)
-
-  ##____________________________________________________________________________
-  ## Check for the library
-
-  find_library (LIBMAGIC_LIBRARIES magic
-    HINTS ${LIBMAGIC_ROOT_DIR} ${CMAKE_INSTALL_PREFIX}
-    PATH_SUFFIXES lib
-    )
-
-  ##____________________________________________________________________________
-  ## Actions taken when all components have been found
-
-  #find_package_handle_standard_args (LIBMAGIC DEFAULT_MSG LIBMAGIC_LIBRARIES LIBMAGIC_INCLUDES)
-
-  if (LIBMAGIC_FOUND)
-    if (NOT LIBMAGIC_FIND_QUIETLY)
-      message (STATUS "Found components for LIBMAGIC")
-      message (STATUS "LIBMAGIC_ROOT_DIR  = ${LIBMAGIC_ROOT_DIR}")
-      message (STATUS "LIBMAGIC_INCLUDES  = ${LIBMAGIC_INCLUDES}")
-      message (STATUS "LIBMAGIC_LIBRARIES = ${LIBMAGIC_LIBRARIES}")
-    endif (NOT LIBMAGIC_FIND_QUIETLY)
-  else (LIBMAGIC_FOUND)
-    if (LIBMAGIC_FIND_REQUIRED)
-      message (FATAL_ERROR "Could not find LIBMAGIC!")
-    endif (LIBMAGIC_FIND_REQUIRED)
-  endif (LIBMAGIC_FOUND)
-
-  ##____________________________________________________________________________
-  ## Mark advanced variables
-
-  mark_as_advanced (
-    LIBMAGIC_ROOT_DIR
-    LIBMAGIC_INCLUDES
-    LIBMAGIC_LIBRARIES
-    )
-
-endif (NOT LIBMAGIC_FOUND)

CMakeModules/FindOpenSSL.cmake

@@ -1,478 +0,0 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
-macro(_OpenSSL_test_and_find_dependencies ssl_library crypto_library)
-  if((CMAKE_SYSTEM_NAME STREQUAL "Linux") AND
-     (("${ssl_library}" MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$") OR
-      ("${crypto_library}" MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$")))
-    set(_OpenSSL_has_dependencies TRUE)
-    find_package(Threads)
-  else()
-    set(_OpenSSL_has_dependencies FALSE)
-  endif()
-endmacro()
-
-function(_OpenSSL_add_dependencies libraries_var library)
-  if(CMAKE_THREAD_LIBS_INIT)
-    list(APPEND ${libraries_var} ${CMAKE_THREAD_LIBS_INIT})
-  endif()
-  list(APPEND ${libraries_var} ${CMAKE_DL_LIBS})
-  set(${libraries_var} ${${libraries_var}} PARENT_SCOPE)
-endfunction()
-
-function(_OpenSSL_target_add_dependencies target)
-  if(_OpenSSL_has_dependencies)
-    set_property( TARGET ${target} APPEND PROPERTY INTERFACE_LINK_LIBRARIES Threads::Threads )
-    set_property( TARGET ${target} APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS} )
-  endif()
-endfunction()
-
-if (UNIX)
-  find_package(PkgConfig QUIET)
-  pkg_check_modules(_OPENSSL QUIET openssl)
-endif ()
-
-# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES
-if(OPENSSL_USE_STATIC_LIBS)
-  set(_openssl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-  if(WIN32)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
-  else()
-    set(CMAKE_FIND_LIBRARY_SUFFIXES .a )
-  endif()
-endif()
-
-if (WIN32)
-  # http://www.slproweb.com/products/Win32OpenSSL.html
-  set(_OPENSSL_ROOT_HINTS
-    ${OPENSSL_ROOT_DIR}
-    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\OpenSSL (32-bit)_is1;Inno Setup: App Path]"
-    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\OpenSSL (64-bit)_is1;Inno Setup: App Path]"
-    ENV OPENSSL_ROOT_DIR
-    )
-  file(TO_CMAKE_PATH "$ENV{PROGRAMFILES}" _programfiles)
-  set(_OPENSSL_ROOT_PATHS
-    "${_programfiles}/OpenSSL"
-    "${_programfiles}/OpenSSL-Win32"
-    "${_programfiles}/OpenSSL-Win64"
-    "C:/OpenSSL/"
-    "C:/OpenSSL-Win32/"
-    "C:/OpenSSL-Win64/"
-    )
-  unset(_programfiles)
-else ()
-  set(_OPENSSL_ROOT_HINTS
-    ${OPENSSL_ROOT_DIR}
-    ENV OPENSSL_ROOT_DIR
-    )
-endif ()
-
-set(_OPENSSL_ROOT_HINTS_AND_PATHS
-    HINTS ${_OPENSSL_ROOT_HINTS}
-    PATHS ${_OPENSSL_ROOT_PATHS}
-    )
-
-find_path(OPENSSL_INCLUDE_DIR
-  NAMES
-    openssl/ssl.h
-  ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-  HINTS
-    ${_OPENSSL_INCLUDEDIR}
-  PATH_SUFFIXES
-    include
-)
-
-if(WIN32 AND NOT CYGWIN)
-  if(MSVC)
-    # /MD and /MDd are the standard values - if someone wants to use
-    # others, the libnames have to change here too
-    # use also ssl and ssleay32 in debug as fallback for openssl < 0.9.8b
-    # enable OPENSSL_MSVC_STATIC_RT to get the libs build /MT (Multithreaded no-DLL)
-    # In Visual C++ naming convention each of these four kinds of Windows libraries has it's standard suffix:
-    #   * MD for dynamic-release
-    #   * MDd for dynamic-debug
-    #   * MT for static-release
-    #   * MTd for static-debug
-
-    # Implementation details:
-    # We are using the libraries located in the VC subdir instead of the parent directory even though :
-    # libeay32MD.lib is identical to ../libeay32.lib, and
-    # ssleay32MD.lib is identical to ../ssleay32.lib
-    # enable OPENSSL_USE_STATIC_LIBS to use the static libs located in lib/VC/static
-
-    if (OPENSSL_MSVC_STATIC_RT)
-      set(_OPENSSL_MSVC_RT_MODE "MT")
-    else ()
-      set(_OPENSSL_MSVC_RT_MODE "MD")
-    endif ()
-
-    # Since OpenSSL 1.1, lib names are like libcrypto32MTd.lib and libssl32MTd.lib
-    if( "${CMAKE_SIZEOF_VOID_P}" STREQUAL "8" )
-        set(_OPENSSL_MSVC_ARCH_SUFFIX "64")
-    else()
-        set(_OPENSSL_MSVC_ARCH_SUFFIX "32")
-    endif()
-
-    if(OPENSSL_USE_STATIC_LIBS)
-      set(_OPENSSL_PATH_SUFFIXES
-        "lib/VC/static"
-        "VC/static"
-        "lib"
-        )
-    else()
-      set(_OPENSSL_PATH_SUFFIXES
-        "lib/VC"
-        "VC"
-        "lib"
-        )
-    endif ()
-
-    find_library(LIB_EAY_DEBUG
-      NAMES
-        libcrypto${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}d
-        libcrypto${_OPENSSL_MSVC_RT_MODE}d
-        libcryptod
-        libeay32${_OPENSSL_MSVC_RT_MODE}d
-        libeay32d
-        cryptod
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    find_library(LIB_EAY_RELEASE
-      NAMES
-        libcrypto${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}
-        libcrypto${_OPENSSL_MSVC_RT_MODE}
-        libcrypto
-        libeay32${_OPENSSL_MSVC_RT_MODE}
-        libeay32
-        crypto
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    find_library(SSL_EAY_DEBUG
-      NAMES
-        libssl${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}d
-        libssl${_OPENSSL_MSVC_RT_MODE}d
-        libssld
-        ssleay32${_OPENSSL_MSVC_RT_MODE}d
-        ssleay32d
-        ssld
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    find_library(SSL_EAY_RELEASE
-      NAMES
-        libssl${_OPENSSL_MSVC_ARCH_SUFFIX}${_OPENSSL_MSVC_RT_MODE}
-        libssl${_OPENSSL_MSVC_RT_MODE}
-        libssl
-        ssleay32${_OPENSSL_MSVC_RT_MODE}
-        ssleay32
-        ssl
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        ${_OPENSSL_PATH_SUFFIXES}
-    )
-
-    set(LIB_EAY_LIBRARY_DEBUG "${LIB_EAY_DEBUG}")
-    set(LIB_EAY_LIBRARY_RELEASE "${LIB_EAY_RELEASE}")
-    set(SSL_EAY_LIBRARY_DEBUG "${SSL_EAY_DEBUG}")
-    set(SSL_EAY_LIBRARY_RELEASE "${SSL_EAY_RELEASE}")
-
-    include(${CMAKE_CURRENT_LIST_DIR}/SelectLibraryConfigurations.cmake)
-    select_library_configurations(LIB_EAY)
-    select_library_configurations(SSL_EAY)
-
-    mark_as_advanced(LIB_EAY_LIBRARY_DEBUG LIB_EAY_LIBRARY_RELEASE
-                     SSL_EAY_LIBRARY_DEBUG SSL_EAY_LIBRARY_RELEASE)
-    set(OPENSSL_SSL_LIBRARY ${SSL_EAY_LIBRARY} )
-    set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY_LIBRARY} )
-  elseif(MINGW)
-    # same player, for MinGW
-    set(LIB_EAY_NAMES crypto libeay32)
-    set(SSL_EAY_NAMES ssl ssleay32)
-    find_library(LIB_EAY
-      NAMES
-        ${LIB_EAY_NAMES}
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        "lib/MinGW"
-        "lib"
-    )
-
-    find_library(SSL_EAY
-      NAMES
-        ${SSL_EAY_NAMES}
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      PATH_SUFFIXES
-        "lib/MinGW"
-        "lib"
-    )
-
-    mark_as_advanced(SSL_EAY LIB_EAY)
-    set(OPENSSL_SSL_LIBRARY ${SSL_EAY} )
-    set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY} )
-    unset(LIB_EAY_NAMES)
-    unset(SSL_EAY_NAMES)
-  else()
-    # Not sure what to pick for -say- intel, let's use the toplevel ones and hope someone report issues:
-    find_library(LIB_EAY
-      NAMES
-        libcrypto
-        libeay32
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      HINTS
-        ${_OPENSSL_LIBDIR}
-      PATH_SUFFIXES
-        lib
-    )
-
-    find_library(SSL_EAY
-      NAMES
-        libssl
-        ssleay32
-      NAMES_PER_DIR
-      ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-      HINTS
-        ${_OPENSSL_LIBDIR}
-      PATH_SUFFIXES
-        lib
-    )
-
-    mark_as_advanced(SSL_EAY LIB_EAY)
-    set(OPENSSL_SSL_LIBRARY ${SSL_EAY} )
-    set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY} )
-  endif()
-else()
-
-  find_library(OPENSSL_SSL_LIBRARY
-    NAMES
-      ssl
-      ssleay32
-      ssleay32MD
-    NAMES_PER_DIR
-    ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-    HINTS
-      ${_OPENSSL_LIBDIR}
-    PATH_SUFFIXES
-      lib
-  )
-
-  find_library(OPENSSL_CRYPTO_LIBRARY
-    NAMES
-      crypto
-    NAMES_PER_DIR
-    ${_OPENSSL_ROOT_HINTS_AND_PATHS}
-    HINTS
-      ${_OPENSSL_LIBDIR}
-    PATH_SUFFIXES
-      lib
-  )
-
-  mark_as_advanced(OPENSSL_CRYPTO_LIBRARY OPENSSL_SSL_LIBRARY)
-
-endif()
-
-# compat defines
-set(OPENSSL_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY})
-set(OPENSSL_CRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY})
-_OpenSSL_test_and_find_dependencies("${OPENSSL_SSL_LIBRARY}" "${OPENSSL_CRYPTO_LIBRARY}")
-if(_OpenSSL_has_dependencies)
-  _OpenSSL_add_dependencies( OPENSSL_SSL_LIBRARIES "${OPENSSL_SSL_LIBRARY}" )
-  _OpenSSL_add_dependencies( OPENSSL_CRYPTO_LIBRARIES "${OPENSSL_CRYPTO_LIBRARY}" )
-endif()
-
-function(from_hex HEX DEC)
-  string(TOUPPER "${HEX}" HEX)
-  set(_res 0)
-  string(LENGTH "${HEX}" _strlen)
-
-  while (_strlen GREATER 0)
-    math(EXPR _res "${_res} * 16")
-    string(SUBSTRING "${HEX}" 0 1 NIBBLE)
-    string(SUBSTRING "${HEX}" 1 -1 HEX)
-    if (NIBBLE STREQUAL "A")
-      math(EXPR _res "${_res} + 10")
-    elseif (NIBBLE STREQUAL "B")
-      math(EXPR _res "${_res} + 11")
-    elseif (NIBBLE STREQUAL "C")
-      math(EXPR _res "${_res} + 12")
-    elseif (NIBBLE STREQUAL "D")
-      math(EXPR _res "${_res} + 13")
-    elseif (NIBBLE STREQUAL "E")
-      math(EXPR _res "${_res} + 14")
-    elseif (NIBBLE STREQUAL "F")
-      math(EXPR _res "${_res} + 15")
-    else()
-      math(EXPR _res "${_res} + ${NIBBLE}")
-    endif()
-
-    string(LENGTH "${HEX}" _strlen)
-  endwhile()
-
-  set(${DEC} ${_res} PARENT_SCOPE)
-endfunction()
-
-if(OPENSSL_INCLUDE_DIR AND EXISTS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h")
-  file(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h" openssl_version_str
-       REGEX "^#[\t ]*define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])+.*")
-
-  if(openssl_version_str)
-    # The version number is encoded as 0xMNNFFPPS: major minor fix patch status
-    # The status gives if this is a developer or prerelease and is ignored here.
-    # Major, minor, and fix directly translate into the version numbers shown in
-    # the string. The patch field translates to the single character suffix that
-    # indicates the bug fix state, which 00 -> nothing, 01 -> a, 02 -> b and so
-    # on.
-
-    string(REGEX REPLACE "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F]).*$"
-           "\\1;\\2;\\3;\\4;\\5" OPENSSL_VERSION_LIST "${openssl_version_str}")
-    list(GET OPENSSL_VERSION_LIST 0 OPENSSL_VERSION_MAJOR)
-    list(GET OPENSSL_VERSION_LIST 1 OPENSSL_VERSION_MINOR)
-    from_hex("${OPENSSL_VERSION_MINOR}" OPENSSL_VERSION_MINOR)
-    list(GET OPENSSL_VERSION_LIST 2 OPENSSL_VERSION_FIX)
-    from_hex("${OPENSSL_VERSION_FIX}" OPENSSL_VERSION_FIX)
-    list(GET OPENSSL_VERSION_LIST 3 OPENSSL_VERSION_PATCH)
-
-    if (NOT OPENSSL_VERSION_PATCH STREQUAL "00")
-      from_hex("${OPENSSL_VERSION_PATCH}" _tmp)
-      # 96 is the ASCII code of 'a' minus 1
-      math(EXPR OPENSSL_VERSION_PATCH_ASCII "${_tmp} + 96")
-      unset(_tmp)
-      # Once anyone knows how OpenSSL would call the patch versions beyond 'z'
-      # this should be updated to handle that, too. This has not happened yet
-      # so it is simply ignored here for now.
-      string(ASCII "${OPENSSL_VERSION_PATCH_ASCII}" OPENSSL_VERSION_PATCH_STRING)
-    endif ()
-
-    set(OPENSSL_VERSION "${OPENSSL_VERSION_MAJOR}.${OPENSSL_VERSION_MINOR}.${OPENSSL_VERSION_FIX}${OPENSSL_VERSION_PATCH_STRING}")
-  endif ()
-endif ()
-
-set(OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARIES} )
-list(REMOVE_DUPLICATES OPENSSL_LIBRARIES)
-
-foreach(_comp IN LISTS OpenSSL_FIND_COMPONENTS)
-  if(_comp STREQUAL "Crypto")
-    if(EXISTS "${OPENSSL_INCLUDE_DIR}" AND
-        (EXISTS "${OPENSSL_CRYPTO_LIBRARY}" OR
-        EXISTS "${LIB_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${LIB_EAY_LIBRARY_RELEASE}")
-    )
-      set(OpenSSL_${_comp}_FOUND TRUE)
-    else()
-      set(OpenSSL_${_comp}_FOUND FALSE)
-    endif()
-  elseif(_comp STREQUAL "SSL")
-    if(EXISTS "${OPENSSL_INCLUDE_DIR}" AND
-        (EXISTS "${OPENSSL_SSL_LIBRARY}" OR
-        EXISTS "${SSL_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${SSL_EAY_LIBRARY_RELEASE}")
-    )
-      set(OpenSSL_${_comp}_FOUND TRUE)
-    else()
-      set(OpenSSL_${_comp}_FOUND FALSE)
-    endif()
-  else()
-    message(WARNING "${_comp} is not a valid OpenSSL component")
-    set(OpenSSL_${_comp}_FOUND FALSE)
-  endif()
-endforeach()
-unset(_comp)
-
-include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
-find_package_handle_standard_args(OpenSSL
-  REQUIRED_VARS
-    OPENSSL_CRYPTO_LIBRARY
-    OPENSSL_INCLUDE_DIR
-  VERSION_VAR
-    OPENSSL_VERSION
-  HANDLE_COMPONENTS
-  FAIL_MESSAGE
-    "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR"
-)
-
-mark_as_advanced(OPENSSL_INCLUDE_DIR OPENSSL_LIBRARIES)
-
-if(OPENSSL_FOUND)
-  if(NOT TARGET OpenSSL::Crypto AND
-      (EXISTS "${OPENSSL_CRYPTO_LIBRARY}" OR
-        EXISTS "${LIB_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${LIB_EAY_LIBRARY_RELEASE}")
-      )
-    add_library(OpenSSL::Crypto UNKNOWN IMPORTED)
-    set_target_properties(OpenSSL::Crypto PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}")
-    if(EXISTS "${OPENSSL_CRYPTO_LIBRARY}")
-      set_target_properties(OpenSSL::Crypto PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-        IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}")
-    endif()
-    if(EXISTS "${LIB_EAY_LIBRARY_RELEASE}")
-      set_property(TARGET OpenSSL::Crypto APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS RELEASE)
-      set_target_properties(OpenSSL::Crypto PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C"
-        IMPORTED_LOCATION_RELEASE "${LIB_EAY_LIBRARY_RELEASE}")
-    endif()
-    if(EXISTS "${LIB_EAY_LIBRARY_DEBUG}")
-      set_property(TARGET OpenSSL::Crypto APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS DEBUG)
-      set_target_properties(OpenSSL::Crypto PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C"
-        IMPORTED_LOCATION_DEBUG "${LIB_EAY_LIBRARY_DEBUG}")
-    endif()
-    _OpenSSL_target_add_dependencies(OpenSSL::Crypto)
-  endif()
-
-  if(NOT TARGET OpenSSL::SSL AND
-      (EXISTS "${OPENSSL_SSL_LIBRARY}" OR
-        EXISTS "${SSL_EAY_LIBRARY_DEBUG}" OR
-        EXISTS "${SSL_EAY_LIBRARY_RELEASE}")
-      )
-    add_library(OpenSSL::SSL UNKNOWN IMPORTED)
-    set_target_properties(OpenSSL::SSL PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}")
-    if(EXISTS "${OPENSSL_SSL_LIBRARY}")
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-        IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}")
-    endif()
-    if(EXISTS "${SSL_EAY_LIBRARY_RELEASE}")
-      set_property(TARGET OpenSSL::SSL APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS RELEASE)
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C"
-        IMPORTED_LOCATION_RELEASE "${SSL_EAY_LIBRARY_RELEASE}")
-    endif()
-    if(EXISTS "${SSL_EAY_LIBRARY_DEBUG}")
-      set_property(TARGET OpenSSL::SSL APPEND PROPERTY
-        IMPORTED_CONFIGURATIONS DEBUG)
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C"
-        IMPORTED_LOCATION_DEBUG "${SSL_EAY_LIBRARY_DEBUG}")
-    endif()
-    if(TARGET OpenSSL::Crypto)
-      set_target_properties(OpenSSL::SSL PROPERTIES
-        INTERFACE_LINK_LIBRARIES OpenSSL::Crypto)
-    endif()
-    _OpenSSL_target_add_dependencies(OpenSSL::SSL)
-  endif()
-endif()
-
-# Restore the original find library ordering
-if(OPENSSL_USE_STATIC_LIBS)
-  set(CMAKE_FIND_LIBRARY_SUFFIXES ${_openssl_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
-endif()

CMakeModules/FindPackageHandleStandardArgs.cmake

@@ -1,268 +0,0 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
-include(${CMAKE_CURRENT_LIST_DIR}/FindPackageMessage.cmake)
-
-# internal helper macro
-macro(_FPHSA_FAILURE_MESSAGE _msg)
-  set (__msg "${_msg}")
-  if (FPHSA_REASON_FAILURE_MESSAGE)
-    string(APPEND __msg "\n    Reason given by package: ${FPHSA_REASON_FAILURE_MESSAGE}\n")
-  endif()
-  if (${_NAME}_FIND_REQUIRED)
-    message(FATAL_ERROR "${__msg}")
-  else ()
-    if (NOT ${_NAME}_FIND_QUIETLY)
-      message(STATUS "${__msg}")
-    endif ()
-  endif ()
-endmacro()
-
-
-# internal helper macro to generate the failure message when used in CONFIG_MODE:
-macro(_FPHSA_HANDLE_FAILURE_CONFIG_MODE)
-  # <PackageName>_CONFIG is set, but FOUND is false, this means that some other of the REQUIRED_VARS was not found:
-  if(${_NAME}_CONFIG)
-    _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE}: missing:${MISSING_VARS} (found ${${_NAME}_CONFIG} ${VERSION_MSG})")
-  else()
-    # If _CONSIDERED_CONFIGS is set, the config-file has been found, but no suitable version.
-    # List them all in the error message:
-    if(${_NAME}_CONSIDERED_CONFIGS)
-      set(configsText "")
-      list(LENGTH ${_NAME}_CONSIDERED_CONFIGS configsCount)
-      math(EXPR configsCount "${configsCount} - 1")
-      foreach(currentConfigIndex RANGE ${configsCount})
-        list(GET ${_NAME}_CONSIDERED_CONFIGS ${currentConfigIndex} filename)
-        list(GET ${_NAME}_CONSIDERED_VERSIONS ${currentConfigIndex} version)
-        string(APPEND configsText "\n    ${filename} (version ${version})")
-      endforeach()
-      if (${_NAME}_NOT_FOUND_MESSAGE)
-        if (FPHSA_REASON_FAILURE_MESSAGE)
-          string(PREPEND FPHSA_REASON_FAILURE_MESSAGE "${${_NAME}_NOT_FOUND_MESSAGE}\n    ")
-        else()
-          set(FPHSA_REASON_FAILURE_MESSAGE "${${_NAME}_NOT_FOUND_MESSAGE}")
-        endif()
-      else()
-        string(APPEND configsText "\n")
-      endif()
-      _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE} ${VERSION_MSG}, checked the following files:${configsText}")
-
-    else()
-      # Simple case: No Config-file was found at all:
-      _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE}: found neither ${_NAME}Config.cmake nor ${_NAME_LOWER}-config.cmake ${VERSION_MSG}")
-    endif()
-  endif()
-endmacro()
-
-
-function(FIND_PACKAGE_HANDLE_STANDARD_ARGS _NAME _FIRST_ARG)
-
-# Set up the arguments for `cmake_parse_arguments`.
-  set(options  CONFIG_MODE  HANDLE_COMPONENTS)
-  set(oneValueArgs  FAIL_MESSAGE  REASON_FAILURE_MESSAGE VERSION_VAR  FOUND_VAR)
-  set(multiValueArgs REQUIRED_VARS)
-
-# Check whether we are in 'simple' or 'extended' mode:
-  set(_KEYWORDS_FOR_EXTENDED_MODE  ${options} ${oneValueArgs} ${multiValueArgs} )
-  list(FIND _KEYWORDS_FOR_EXTENDED_MODE "${_FIRST_ARG}" INDEX)
-
-  if(${INDEX} EQUAL -1)
-    set(FPHSA_FAIL_MESSAGE ${_FIRST_ARG})
-    set(FPHSA_REQUIRED_VARS ${ARGN})
-    set(FPHSA_VERSION_VAR)
-  else()
-    cmake_parse_arguments(FPHSA "${options}" "${oneValueArgs}" "${multiValueArgs}"  ${_FIRST_ARG} ${ARGN})
-
-    if(FPHSA_UNPARSED_ARGUMENTS)
-      message(FATAL_ERROR "Unknown keywords given to FIND_PACKAGE_HANDLE_STANDARD_ARGS(): \"${FPHSA_UNPARSED_ARGUMENTS}\"")
-    endif()
-
-    if(NOT FPHSA_FAIL_MESSAGE)
-      set(FPHSA_FAIL_MESSAGE  "DEFAULT_MSG")
-    endif()
-
-    # In config-mode, we rely on the variable <PackageName>_CONFIG, which is set by find_package()
-    # when it successfully found the config-file, including version checking:
-    if(FPHSA_CONFIG_MODE)
-      list(INSERT FPHSA_REQUIRED_VARS 0 ${_NAME}_CONFIG)
-      list(REMOVE_DUPLICATES FPHSA_REQUIRED_VARS)
-      set(FPHSA_VERSION_VAR ${_NAME}_VERSION)
-    endif()
-
-    if(NOT FPHSA_REQUIRED_VARS)
-      message(FATAL_ERROR "No REQUIRED_VARS specified for FIND_PACKAGE_HANDLE_STANDARD_ARGS()")
-    endif()
-  endif()
-
-# now that we collected all arguments, process them
-
-  if("x${FPHSA_FAIL_MESSAGE}" STREQUAL "xDEFAULT_MSG")
-    set(FPHSA_FAIL_MESSAGE "Could NOT find ${_NAME}")
-  endif()
-
-  list(GET FPHSA_REQUIRED_VARS 0 _FIRST_REQUIRED_VAR)
-
-  string(TOUPPER ${_NAME} _NAME_UPPER)
-  string(TOLOWER ${_NAME} _NAME_LOWER)
-
-  if(FPHSA_FOUND_VAR)
-    if(FPHSA_FOUND_VAR MATCHES "^${_NAME}_FOUND$"  OR  FPHSA_FOUND_VAR MATCHES "^${_NAME_UPPER}_FOUND$")
-      set(_FOUND_VAR ${FPHSA_FOUND_VAR})
-    else()
-      message(FATAL_ERROR "The argument for FOUND_VAR is \"${FPHSA_FOUND_VAR}\", but only \"${_NAME}_FOUND\" and \"${_NAME_UPPER}_FOUND\" are valid names.")
-    endif()
-  else()
-    set(_FOUND_VAR ${_NAME_UPPER}_FOUND)
-  endif()
-
-  # collect all variables which were not found, so they can be printed, so the
-  # user knows better what went wrong (#6375)
-  set(MISSING_VARS "")
-  set(DETAILS "")
-  # check if all passed variables are valid
-  set(FPHSA_FOUND_${_NAME} TRUE)
-  foreach(_CURRENT_VAR ${FPHSA_REQUIRED_VARS})
-    if(NOT ${_CURRENT_VAR})
-      set(FPHSA_FOUND_${_NAME} FALSE)
-      string(APPEND MISSING_VARS " ${_CURRENT_VAR}")
-    else()
-      string(APPEND DETAILS "[${${_CURRENT_VAR}}]")
-    endif()
-  endforeach()
-  if(FPHSA_FOUND_${_NAME})
-    set(${_NAME}_FOUND TRUE)
-    set(${_NAME_UPPER}_FOUND TRUE)
-  else()
-    set(${_NAME}_FOUND FALSE)
-    set(${_NAME_UPPER}_FOUND FALSE)
-  endif()
-
-  # component handling
-  unset(FOUND_COMPONENTS_MSG)
-  unset(MISSING_COMPONENTS_MSG)
-
-  if(FPHSA_HANDLE_COMPONENTS)
-    foreach(comp ${${_NAME}_FIND_COMPONENTS})
-      if(${_NAME}_${comp}_FOUND)
-
-        if(NOT DEFINED FOUND_COMPONENTS_MSG)
-          set(FOUND_COMPONENTS_MSG "found components:")
-        endif()
-        string(APPEND FOUND_COMPONENTS_MSG " ${comp}")
-
-      else()
-
-        if(NOT DEFINED MISSING_COMPONENTS_MSG)
-          set(MISSING_COMPONENTS_MSG "missing components:")
-        endif()
-        string(APPEND MISSING_COMPONENTS_MSG " ${comp}")
-
-        if(${_NAME}_FIND_REQUIRED_${comp})
-          set(${_NAME}_FOUND FALSE)
-          string(APPEND MISSING_VARS " ${comp}")
-        endif()
-
-      endif()
-    endforeach()
-    set(COMPONENT_MSG "${FOUND_COMPONENTS_MSG} ${MISSING_COMPONENTS_MSG}")
-    string(APPEND DETAILS "[c${COMPONENT_MSG}]")
-  endif()
-
-  # version handling:
-  set(VERSION_MSG "")
-  set(VERSION_OK TRUE)
-
-  # check with DEFINED here as the requested or found version may be "0"
-  if (DEFINED ${_NAME}_FIND_VERSION)
-    if(DEFINED ${FPHSA_VERSION_VAR})
-      set(_FOUND_VERSION ${${FPHSA_VERSION_VAR}})
-
-      if(${_NAME}_FIND_VERSION_EXACT)       # exact version required
-        # count the dots in the version string
-        string(REGEX REPLACE "[^.]" "" _VERSION_DOTS "${_FOUND_VERSION}")
-        # add one dot because there is one dot more than there are components
-        string(LENGTH "${_VERSION_DOTS}." _VERSION_DOTS)
-        if (_VERSION_DOTS GREATER ${_NAME}_FIND_VERSION_COUNT)
-          # Because of the C++ implementation of find_package() ${_NAME}_FIND_VERSION_COUNT
-          # is at most 4 here. Therefore a simple lookup table is used.
-          if (${_NAME}_FIND_VERSION_COUNT EQUAL 1)
-            set(_VERSION_REGEX "[^.]*")
-          elseif (${_NAME}_FIND_VERSION_COUNT EQUAL 2)
-            set(_VERSION_REGEX "[^.]*\\.[^.]*")
-          elseif (${_NAME}_FIND_VERSION_COUNT EQUAL 3)
-            set(_VERSION_REGEX "[^.]*\\.[^.]*\\.[^.]*")
-          else ()
-            set(_VERSION_REGEX "[^.]*\\.[^.]*\\.[^.]*\\.[^.]*")
-          endif ()
-          string(REGEX REPLACE "^(${_VERSION_REGEX})\\..*" "\\1" _VERSION_HEAD "${_FOUND_VERSION}")
-          unset(_VERSION_REGEX)
-          if (NOT ${_NAME}_FIND_VERSION VERSION_EQUAL _VERSION_HEAD)
-            set(VERSION_MSG "Found unsuitable version \"${_FOUND_VERSION}\", but required is exact version \"${${_NAME}_FIND_VERSION}\"")
-            set(VERSION_OK FALSE)
-          else ()
-            set(VERSION_MSG "(found suitable exact version \"${_FOUND_VERSION}\")")
-          endif ()
-          unset(_VERSION_HEAD)
-        else ()
-          if (NOT ${_NAME}_FIND_VERSION VERSION_EQUAL _FOUND_VERSION)
-            set(VERSION_MSG "Found unsuitable version \"${_FOUND_VERSION}\", but required is exact version \"${${_NAME}_FIND_VERSION}\"")
-            set(VERSION_OK FALSE)
-          else ()
-            set(VERSION_MSG "(found suitable exact version \"${_FOUND_VERSION}\")")
-          endif ()
-        endif ()
-        unset(_VERSION_DOTS)
-
-      else()     # minimum version specified:
-        if (${_NAME}_FIND_VERSION VERSION_GREATER _FOUND_VERSION)
-          set(VERSION_MSG "Found unsuitable version \"${_FOUND_VERSION}\", but required is at least \"${${_NAME}_FIND_VERSION}\"")
-          set(VERSION_OK FALSE)
-        else ()
-          set(VERSION_MSG "(found suitable version \"${_FOUND_VERSION}\", minimum required is \"${${_NAME}_FIND_VERSION}\")")
-        endif ()
-      endif()
-
-    else()
-
-      # if the package was not found, but a version was given, add that to the output:
-      if(${_NAME}_FIND_VERSION_EXACT)
-         set(VERSION_MSG "(Required is exact version \"${${_NAME}_FIND_VERSION}\")")
-      else()
-         set(VERSION_MSG "(Required is at least version \"${${_NAME}_FIND_VERSION}\")")
-      endif()
-
-    endif()
-  else ()
-    # Check with DEFINED as the found version may be 0.
-    if(DEFINED ${FPHSA_VERSION_VAR})
-      set(VERSION_MSG "(found version \"${${FPHSA_VERSION_VAR}}\")")
-    endif()
-  endif ()
-
-  if(VERSION_OK)
-    string(APPEND DETAILS "[v${${FPHSA_VERSION_VAR}}(${${_NAME}_FIND_VERSION})]")
-  else()
-    set(${_NAME}_FOUND FALSE)
-  endif()
-
-
-  # print the result:
-  if (${_NAME}_FOUND)
-    FIND_PACKAGE_MESSAGE(${_NAME} "Found ${_NAME}: ${${_FIRST_REQUIRED_VAR}} ${VERSION_MSG} ${COMPONENT_MSG}" "${DETAILS}")
-  else ()
-
-    if(FPHSA_CONFIG_MODE)
-      _FPHSA_HANDLE_FAILURE_CONFIG_MODE()
-    else()
-      if(NOT VERSION_OK)
-        _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE}: ${VERSION_MSG} (found ${${_FIRST_REQUIRED_VAR}})")
-      else()
-        _FPHSA_FAILURE_MESSAGE("${FPHSA_FAIL_MESSAGE} (missing:${MISSING_VARS}) ${VERSION_MSG}")
-      endif()
-    endif()
-
-  endif ()
-
-  set(${_NAME}_FOUND ${${_NAME}_FOUND} PARENT_SCOPE)
-  set(${_NAME_UPPER}_FOUND ${${_NAME}_FOUND} PARENT_SCOPE)
-endfunction()

CMakeModules/FindPackageMessage.cmake

@@ -1,48 +0,0 @@
-# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
-# file Copyright.txt or https://cmake.org/licensing for details.
-
-#[=======================================================================[.rst:
-FindPackageMessage
-------------------
-
-.. code-block:: cmake
-
-  find_package_message(<name> "message for user" "find result details")
-
-This function is intended to be used in FindXXX.cmake modules files.
-It will print a message once for each unique find result.  This is
-useful for telling the user where a package was found.  The first
-argument specifies the name (XXX) of the package.  The second argument
-specifies the message to display.  The third argument lists details
-about the find result so that if they change the message will be
-displayed again.  The macro also obeys the QUIET argument to the
-find_package command.
-
-Example:
-
-.. code-block:: cmake
-
-  if(X11_FOUND)
-    find_package_message(X11 "Found X11: ${X11_X11_LIB}"
-      "[${X11_X11_LIB}][${X11_INCLUDE_DIR}]")
-  else()
-   ...
-  endif()
-#]=======================================================================]
-
-function(find_package_message pkg msg details)
-  # Avoid printing a message repeatedly for the same find result.
-  if(NOT ${pkg}_FIND_QUIETLY)
-    string(REPLACE "\n" "" details "${details}")
-    set(DETAILS_VAR FIND_PACKAGE_MESSAGE_DETAILS_${pkg})
-    if(NOT "${details}" STREQUAL "${${DETAILS_VAR}}")
-      # The message has not yet been printed.
-      message(STATUS "${msg}")
-
-      # Save the find details in the cache to avoid printing the same
-      # message again.
-      set("${DETAILS_VAR}" "${details}"
-        CACHE INTERNAL "Details about finding ${pkg}")
-    endif()
-  endif()
-endfunction()

Docker/Dockerfile

@@ -1,22 +0,0 @@
-FROM ubuntu:19.10
-MAINTAINER simon987 <me@simon987.net>
-
-RUN apt update
-RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
- curl libtiff5 libpng16-16
-
-RUN mkdir -p /usr/share/tessdata && \
-    cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
-
-ADD sist2 /root/sist2
-
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-ENTRYPOINT ["/root/sist2"]

Docker/build.sh

@@ -1,15 +0,0 @@
-rm ./sist2
-cp ../sist2 .
-strip sist2
-
-version=$(./sist2 --version)
-
-echo "Version ${version}"
-docker build . -t simon987/sist2:${version} -t simon987/sist2:latest \
-  -t docker.pkg.github.com/simon987/sist2/sist2:latest -t docker.pkg.github.com/simon987/sist2/sist2:${version}
-docker push simon987/sist2:${version}
-docker push simon987/sist2:latest
-docker push docker.pkg.github.com/simon987/sist2/sist2:latest
-docker push docker.pkg.github.com/simon987/sist2/sist2:${version}
-
-docker run --rm -it simon987/sist2 -v

Dockerfile

@@ -0,0 +1,52 @@
+FROM sist2app/sist2-build as build
+
+WORKDIR /build/
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
+
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
+
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN cd build && make -j$(nproc)
+RUN strip build/sist2 || mv build/sist2_debug build/sist2
+
+FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]
+
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 libmagic1 python3  \
+    python3-pip git tesseract-ocr && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /usr/share/tessdata && \
+    cd /usr/share/tessdata/ && \
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+
+# sist2
+COPY --from=build /build/build/sist2 /root/sist2
+
+# sist2-admin
+WORKDIR /root/sist2-admin
+COPY sist2-admin/requirements.txt /root/sist2-admin/
+RUN ln /usr/bin/python3 /usr/bin/python
+RUN python -m pip install --no-cache -r /root/sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ /root/sist2-admin/

Dockerfile.arm64

@@ -0,0 +1,53 @@
+FROM sist2app/sist2-build-arm64 as build
+
+WORKDIR /build/
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
+
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
+
+WORKDIR /build/
+ADD . /build/
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN cd build && make -j$(nproc)
+RUN strip build/sist2 || mv build/sist2_debug build/sist2
+
+FROM --platform=linux/arm64/v8 ubuntu@sha256:537da24818633b45fcb65e5285a68c3ec1f3db25f5ae5476a7757bc8dfae92a3
+
+WORKDIR /root
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]
+
+RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-pip python3 git && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /usr/share/tessdata && \
+    cd /usr/share/tessdata/ && \
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+
+# sist2
+COPY --from=build /build/build/sist2 /root/sist2
+
+# sist2-admin
+COPY sist2-admin/requirements.txt sist2-admin/
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ sist2-admin/

README.md

@@ -1,6 +1,10 @@
-![GitHub](https://img.shields.io/github/license/simon987/sist2.svg)
+![GitHub](https://img.shields.io/github/license/sist2app/sist2.svg)
-[![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
+[![CodeFactor](https://www.codefactor.io/repository/github/sist2app/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/sist2app/sist2)
-[![Development snapshots](https://ci.simon987.net/app/rest/builds/buildType(Sist2_Build)/statusIcon)](https://files.simon987.net/artifacts/Sist2/Build/)
+[![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
+
+**Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
+
+**Community URL:** [Discord](https://discord.gg/2PEjDy3Rfs)
 
 # sist2
 
@@ -8,151 +12,218 @@ sist2 (Simple incremental search tool)
 
 *Warning: sist2 is in early development*
 
+![search panel](docs/sist2.gif)
+
 ## Features
 
 * Fast, low memory usage, multi-threaded
-* Portable (all its features are packaged in a single executable)
+* Manage & schedule scan jobs with simple web interface (Docker only)
-* Extracts text from common file types \*
+* Mobile-friendly Web interface
+* Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
-* Automatic tagging from file attributes via [user scripts](scripting/README.md)
+* Manual tagging from the UI and automatic tagging based on file attributes via [user scripts](docs/scripting.md)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
-
+* Stats page & disk utilisation visualization
+* Named-entity recognition (client-side) \*\*\*\*
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)    
+\*\*\*\* See [Named-Entity Recognition](#NER)
 
 ## Getting Started
 
-1. Have an [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) instance running
+### Using Docker Compose *(Windows/Linux/Mac)*
-1. 
-    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
-    1. *(or)* Download a [development snapshot](https://files.simon987.net/artifacts/Sist2/Build/) *(Not recommended!)*
-    1. *(or)* `docker pull simon987/sist2:latest`
-   
 
-\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)    
+```yaml
-\* *Mac users*: See [#1](https://github.com/simon987/sist2/issues/1)
+services:
-
+  elasticsearch:
-
+    image: elasticsearch:7.17.9
-## Example usage
+    restart: unless-stopped
-
+    volumes:
-
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
-![demo](demo.gif)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
-
+    environment:
-See help page `sist2 --help` for more details.
+      - "discovery.type=single-node"
-
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
-**Scan a directory**
+      - "PUID=1000"
-```bash
+      - "PGID=1000"
-sist2 scan ~/Documents -o ./orig_idx/
+  sist2-admin:
-sist2 scan --threads 4 --content-size 16384 /mnt/Pictures
+    image: sist2app/sist2:x64-linux
-sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
+    restart: unless-stopped
+    volumes:
+      - /data/sist2-admin-data/:/sist2-admin/
+      - /<path to index>/:/host
+    ports:
+      - 4090:4090
+      # NOTE: Don't expose this port publicly!
+      - 8080:8080
+    working_dir: /root/sist2-admin/
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py
 ```
 
-**Push index to Elasticsearch or file**
+Navigate to http://localhost:8080/ to configure sist2-admin.
-```bash
-sist2 index --force-reset ./my_idx
-sist2 index --print ./my_idx > raw_documents.ndjson
-```
 
-**Start web interface**
+### Using the executable file *(Linux/WSL only)*
-```bash
-sist2 web --bind 0.0.0.0 --port 4321 ./my_idx1 ./my_idx2 ./my_idx3
-```
 
-### Use sist2 with docker
+1. Choose search backend (See [comparison](#search-backends)):
+    * **Elasticsearch**: have an Elasticsearch (version >= 6.8.X, ideally >=7.14.0) instance running
+        1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
+        2. *(or)* Run using docker:
+            ```bash
+            docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
+            ```
+    * **SQLite**: No installation required
 
-**scan**
+2. Download the [latest sist2 release](https://github.com/sist2app/sist2/releases).
-```bash
+   Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
-docker run -it \
+3. See [usage guide](docs/USAGE.md) for command line usage.
-    -v /path/to/files/:/files \
-    -v $PWD/out/:/out \
-    simon987/sist2 scan -t 4 /files -o /out/my_idx1
-```
-**index**
-```bash
-docker run -it --network host\
-    -v $PWD/out/:/out \
-    simon987/sist2 index /out/my_idx1
-```
 
-**web**
+Example usage:
-```bash
-docker run --rm --network host -d --name sist2\
-    -v $PWD/out/my_idx:/idx \
-    -v $PWD/my/files:/files
-    simon987/sist2 web --bind 0.0.0.0 /idx
-docker stop sist2
-```
 
+1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
+2. Prepare search index:
+    * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
+    * **SQLite**: `sist2 sqlite-index --search-index ./search.sist2 ./documents.sist2`
+3. Start web interface: 
+   * **Elasticsearch**: `sist2 web ./documents.sist2`
+   * **SQLite**: `sist2 web --search-index ./search.sist2 ./documents.sist2`
 
 ## Format support
 
-File type | Library | Content | Thumbnail | Metadata
+| File type                                                                 | Library                                                                      | Content  | Thumbnail   | Metadata                                                                                                                               |
-:---|:---|:---|:---|:---
+|:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
-pdf,xps,cbz,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
+| pdf,xps,fb2,epub                                                          | MuPDF                                                                        | text+ocr | yes         | author, title                                                                                                                          |
-`audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags |
+| cbz,cbr                                                                   | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
-`video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist |
+| `audio/*`                                                                 | ffmpeg                                                                       | -        | yes         | ID3 tags                                                                                                                               |
-`image/*` | ffmpeg | - | yes, `jpeg` | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+| `video/*`                                                                 | ffmpeg                                                                       | -        | yes         | title, comment, artist                                                                                                                 |
-ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
+| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/sist2app/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
-`text/plain` | *(none)* | yes | no | - |
+| raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw                                                                       | no       | yes         | Common EXIF tags, GPS tags                                                                                                             |
-tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
+| ttf,ttc,cff,woff,fnt,otf                                                  | Freetype2                                                                    | -        | yes, `bmp`  | Name & style                                                                                                                           |
-docx, xlsx, pptx | libOPC | yes | no | no |
+| `text/plain`                                                              | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| html, xml                                                                 | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
+| docx, xlsx, pptx                                                          | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
+| doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
+| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
+| wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
+| json, jsonl, ndjson                                                       | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
 \* *See [Archive files](#archive-files)*
- 
+
 ### Archive files
-**sist2** will scan files stored into archive files (zip, tar, 7z...) as if
+
-they were directly in the file system. Recursive (archives inside archives)
+**sist2** will scan files stored into archive files (zip, tar, 7z...) as if they were directly in the file system.
+Recursive (archives inside archives)
 scan is also supported.
 
 **Limitations**:
-* Parsing media files with formats that require
-*seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) is not supported.
-* Archive files are scanned sequentially, by a single thread. On systems where
-**sist2** is not I/O bound, scans might be faster when larger archives are split
- into smaller parts.
 
-To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
+* Support for parsing media files with formats that require *seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.)
- 
+  is limitted (see `--mem-buffer` option)
- 
+* Archive files are scanned sequentially, by a single thread. On systems where
+  **sist2** is not I/O bound, scans might be faster when larger archives are split into smaller parts.
+
 ### OCR
 
-You can enable OCR support for pdf,xps,cbz,fb2,epub file types with the
+You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with the
-`--ocr <lang>` option. Download the language data files with your
+`--ocr-lang <lang>` option in combination with `--ocr-images` and/or `--ocr-ebooks`.
-package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
+Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
+directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
-The `simon987/sist2` github image comes with common languages 
+The `sist2app/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
+
+You can use the `+` separator to specify multiple languages. The language
+name must be identical to the `*.traineddata` file installed on your system
+(use `chi_sim` rather than `chi-sim`).
+
+Examples:
 
-Examples
 ```bash
-sist2 scan --ocr jpn ~/Books/Manga/
+sist2 scan --ocr-ebooks --ocr-lang jpn ~/Books/Manga/
-sist2 scan --ocr eng ~/Books/Textbooks/
+sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
+sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
+### Search backends
+
+sist2 v3.0.7+ supports SQLite search backend. The SQLite search backend has
+fewer features and generally comparable query performance for medium-size
+indices, but it uses much less memory and is easier to set up.
+
+|                                              |                       SQLite                        |                                                             Elasticsearch                                                             |
+|----------------------------------------------|:---------------------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
+| Requires separate search engine installation |                                                     |                                                                   ✓                                                                   |
+| Memory footprint                             |                        ~20MB                        |                                                                >500MB                                                                 |
+| Query syntax                                 |      [fts5](https://www.sqlite.org/fts5.html)       | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
+| Fuzzy search                                 |                                                     |                                                                   ✓                                                                   |
+| Media Types tree real-time updating          |                                                     |                                                                   ✓                                                                   |
+| Manual tagging                               |                          ✓                          |                                                                   ✓                                                                   |
+| User scripts                                 |                          ✓                          |                                                                   ✓                                                                   |
+| Media Type breakdown for search results      |                                                     |                                                                   ✓                                                                   |
+| Embeddings search                            |                      ✓ *O(n)*                       |                                                              ✓ *O(logn)*                                                              |
+
+### NER
+
+sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to
+**Configuration** > **Machine learning options** > **Model repositories**
+to enable it.
+
+The text processing is done in your browser, no data is sent to any third-party services.
+See [sist2app/sist2-ner-models](https://github.com/sist2app/sist2-ner-models) for more details.
+
+#### List of available repositories:
+
+| URL                                                                                                     | Maintainer                              | Purpose |
+|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
+| [sist2app/sist2-ner-models](https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json) | [sist2app](https://github.com/sist2app) | General |
+
+<details>
+  <summary>Screenshot</summary>
+
+![ner](docs/ner.png)
+
+</details>
 
 ## Build from source
 
-You can compile **sist2** by yourself if you don't want to use the pre-compiled
+You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
-binaries.
+
+### Using docker
+
+```bash
+git clone --recursive https://github.com/sist2app/sist2/
+cd sist2
+docker build . -t my-sist2-image
+# Copy sist2 executable from docker image
+docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
+```
+
+### Using a linux computer
 
 1. Install compile-time dependencies
 
-    *(Debian)*
+   ```bash
-    ```bash
+   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
-    apt install git cmake pkg-config libglib2.0-dev \
-        libssl-dev uuid-dev python3 libmagic-dev libfreetype6-dev \
-        libcurl-dev libbz2-dev yasm libharfbuzz-dev ragel \
-        libarchive-dev libtiff5 libpng16-16 libpango1.0-dev
    ```
 
-2. Build
+2. Install vcpkg using my fork: https://github.com/sist2app/vcpkg
+3. Install vcpkg dependencies
+
     ```bash
-    git clone --recurse-submodules https://github.com/simon987/sist2
+    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5,json1] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
-    ./scripts/get_static_libs.sh
+    ```
-    cmake .
+
+4. Build
+    ```bash
+    git clone --recursive https://github.com/sist2app/sist2/
+    (cd sist2-vue; npm install; npm run build)
+    (cd sist2-admin/frontend; npm install; npm run build)
+    cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

argparse

@@ -1 +0,0 @@
-Subproject commit fafc503d23d077bda40c29e8a20ea74707452721

cJSON

@@ -1 +0,0 @@
-Subproject commit 2d4ad841927590198ecfb8b27335a0cd97cf15c1

ci/build.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-./scripts/get_static_libs.sh
-
-cmake .
-make
-strip sist2

contrib/systemd/Makefile

@@ -0,0 +1,7 @@
+install:
+	install sist2-update-all.sh /usr/bin/sist2-update-all.sh
+	install sist2-update-files.sh /usr/bin/sist2-update-files.sh
+	install sist2-update-nextcloud.sh /usr/bin/sist2-update-nextcloud.sh
+	install sist2-update.service /etc/systemd/system/sist2-update.service
+	install sist2-update.timer /etc/systemd/system/sist2-update.timer
+	systemctl daemon-reload

contrib/systemd/README.md

@@ -0,0 +1,31 @@
+# Systemd integration example
+
+This example contains my (yatli) personal configuration for sist2 auto-updating.
+The following indices are involved in this configuration:
+
+| Index     | Path             | Description                                |
+|-----------|------------------|--------------------------------------------|
+| files     | /zpool/files     | Main file repository                       |
+| nextcloud | /zpool/nextcloud | Externally synchronized to a cloud account |
+
+The systemd integration achieves automatic sist2 scanning & indexing everyday at 3:00AM.
+
+### Tailoring the configuration for yourself
+
+`sist2-update-all.sh` calls update scripts for each sist2 index. Add or remove
+update scripts accordingly to suit your need. Each update script (e.g.
+`sist2-update-files.sh`) has important parameters laid down at the beginning so
+make sure to edit them to point to your files and index locations.
+
+### Installation
+
+```bash
+# install the services and scripts
+sudo make install
+# enable & start the timer
+sudo systemctl enable sist2-update.timer
+sudo systemctl start sist2-update.timer
+# verify that the timer has been enabled
+systemctl list-timers --all
+```
+

contrib/systemd/sist2-update-all.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+echo "Update index: Files"
+source ${__dir}/sist2-update-files.sh
+echo "Update index: Nextcloud"
+source ${__dir}/sist2-update-nextcloud.sh
+echo "Done. Restarting sist2."
+docker restart sist2-sist2-1

contrib/systemd/sist2-update-files.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+DATE=$(date +%Y_%m_%d)
+CONTENT=/zpool/files
+ORIG=/mnt/ssd/sist-index/files.idx
+NEW=/mnt/ssd/sist-index/files_$DATE.idx
+EXCLUDE='ZArchives|TorrentStore|TorrentDownload|624f0c59-1fef-44f6-95e9-7483296f2833|ubuntu-full-2021-12-07'
+NAME=Files
+#REWRITE_URL="http://localhost:33333/activate?collection=$NAME&path="
+REWRITE_URL=""
+
+sist2 scan \
+  --threads 14 \
+  --mem-throttle 32768 \
+  --thumbnail-quality 2 \
+  --name $NAME \
+  --ocr-lang=eng+chi_sim \
+  --ocr-ebooks \
+  --ocr-images \
+  --exclude=$EXCLUDE \
+  --rewrite-url=$REWRITE_URL \
+  --incremental=$ORIG \
+  --output=$NEW \
+  $CONTENT
+echo ">>> Scan complete"
+rm -rf $ORIG
+mv $NEW $ORIG 
+
+unset http_proxy
+unset https_proxy
+unset HTTP_PROXY
+unset HTTPS_PROXY
+sist2 index $ORIG --incremental-index
+echo ">>> Index complete"

contrib/systemd/sist2-update-nextcloud.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+DATE=$(date +%Y_%m_%d)
+CONTENT=/zpool/nextcloud/v-yadli
+ORIG=/mnt/ssd/sist-index/nextcloud.idx
+NEW=/mnt/ssd/sist-index/nextcloud_$DATE.idx
+EXCLUDE='Yatao|.*263418493\\/Image\\/.*'
+NAME=NextCloud
+# REWRITE_URL="http://localhost:33333/activate?collection=$NAME&path="
+REWRITE_URL=""
+
+sist2 scan \
+  --threads 14 \
+  --mem-throttle 32768 \
+  --thumbnail-quality 2 \
+  --name $NAME \
+  --ocr-lang=eng+chi_sim \
+  --ocr-ebooks \
+  --ocr-images \
+  --exclude=$EXCLUDE \
+  --rewrite-url=$REWRITE_URL \
+  --incremental=$ORIG \
+  --output=$NEW \
+  $CONTENT
+echo ">>> Scan complete"
+rm -rf $ORIG
+mv $NEW $ORIG 
+
+unset http_proxy
+unset https_proxy
+unset HTTP_PROXY
+unset HTTPS_PROXY
+sist2 index $ORIG --incremental-index

contrib/systemd/sist2-update.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=sist2-update
+
+[Service]
+User=yatli
+ExecStart=/bin/bash /usr/bin/sist2-update-all.sh

contrib/systemd/sist2-update.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=sist2-update
+
+[Timer]
+OnCalendar=*-*-* 3:00:00
+Persistent=true
+Unit=sist2-update.service
+
+[Install]
+WantedBy=timers.target

docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3"
+
+services:
+  elasticsearch:
+    image: elasticsearch:7.17.9
+    container_name: sist2-es
+    volumes:
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "PUID=1000"
+      - "PGID=1000"
+  sist2-admin:
+    build:
+      context: .
+    container_name: sist2-admin
+    volumes:
+      - /data/sist2-admin-data/:/sist2-admin/
+      - /<path to index>/:/host
+    ports:
+      - 4090:4090
+      # NOTE: Don't export this port publicly!
+      - 8080:8080
+    working_dir: /root/sist2-admin/
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py

docs/USAGE.md

@@ -0,0 +1,232 @@
+# Usage
+
+```
+Usage: sist2 scan [OPTION]... PATH
+   or: sist2 index [OPTION]... INDEX
+   or: sist2 sqlite-index [OPTION]... INDEX
+   or: sist2 web [OPTION]... INDEX...
+
+Lightning-fast file system indexer and search tool.
+
+    -h, --help                        show this help message and exit
+    -v, --version                     Print version and exit.
+    --verbose                         Turn on logging.
+    --very-verbose                    Turn on debug messages.
+    --json-logs                       Output logs in JSON format.
+
+Scan options
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
+    -q, --thumbnail_count-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
+    --thumbnail_count-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
+    --thumbnail_count-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
+    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
+    -o, --output=<str>                Output index file path. DEFAULT: index.sist2
+    --incremental                     If the output file path exists, only scan new or modified files.
+    --optimize-index                  Defragment index file after scan to reduce its file size.
+    --rewrite-url=<str>               Serve files from this url instead of from disk.
+    --name=<str>                      Index display name. DEFAULT: index
+    --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
+    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: don't scan, list: only save file names as text, shallow: don't scan archives inside archives. DEFAULT: recurse
+    --archive-passphrase=<str>        Passphrase for encrypted archive files
+    --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
+    --ocr-images                      Enable OCR'ing of image files.
+    --ocr-ebooks                      Enable OCR'ing of ebook files.
+    -e, --exclude=<str>               Files that match this regex will not be scanned.
+    --fast                            Only index file names & mime type.
+    --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
+    --read-subtitles                  Read subtitles from media files.
+    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata).
+    --checksums                       Calculate file checksums when scanning.
+    --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
+
+Index options
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
+    --es-url=<str>                    Elasticsearch url with port. DEFAULT: http://localhost:9200
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
+    -p, --print                       Print JSON documents to stdout instead of indexing to elasticsearch.
+    --incremental-index               Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
+    --script-file=<str>               Path to user script.
+    --mappings-file=<str>             Path to Elasticsearch mappings.
+    --settings-file=<str>             Path to Elasticsearch settings.
+    --async-script                    Execute user script asynchronously.
+    --batch-size=<int>                Index batch size. DEFAULT: 70
+    -f, --force-reset                 Reset Elasticsearch mappings and settings.
+
+sqlite-index options
+    --search-index=<str>              Path to search index. Will be created if it does not exist yet.
+
+Web options
+    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --search-index=<str>              Path to SQLite search index.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
+    --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
+    --auth=<str>                      Basic auth in user:password format
+    --auth0-audience=<str>            API audience/identifier
+    --auth0-domain=<str>              Application domain
+    --auth0-client-id=<str>           Application client ID
+    --auth0-public-key-file=<str>     Path to Auth0 public key file extracted from <domain>/pem
+    --tag-auth=<str>                  Basic auth in user:password format for tagging
+    --tagline=<str>                   Tagline in navbar
+    --dev                             Serve html & js files from disk (for development)
+    --lang=<str>                      Default UI language. Can be changed by the user
+
+Made by simon987 <me@simon987.net>. Released under GPL-3.0
+```
+
+#### Thumbnail database size estimation
+
+See chart below for rough estimate of thumbnail_count size vs. thumbnail_count size & quality arguments:
+
+For example, `--thumbnail_count-size=500`, `--thumbnail_count-quality=50` for a directory with 8 million images will create a thumbnail_count database 
+that is about `8000000 * 11.8kB = 94.4GB`.
+
+![thumbnail_size](thumbnail_size.png)
+
+### Scan examples
+
+Simple scan
+```bash
+sist2 scan ~/Documents
+
+sist2 scan \
+    --threads 4 --content-size 16000000 --thumbnail_count-quality 2 --archive shallow \
+    --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
+    ~/Documents -o ./documents.sist2
+```
+
+Incremental scan
+
+If the index file does not exist, `--incremental` has no effect.
+```bash
+sist scan ~/Documents -o ./documents.sist2
+sist scan ~/Documents -o ./documents.sist2 --incremental
+# or
+sist scan ~/Documents -o ./documents.sist2 --incremental
+sist scan ~/Documents -o ./documents.sist2 --incremental
+```
+
+### Index documents to Elasticsearch search backend
+
+```bash
+sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index.sist2
+sist2 index ./my_index.sist2
+```
+
+#### Index documents to SQLite search backend
+```bash
+# The search index will be created if it does not exist already
+sist2 sqlite-index ./index1.sist2 --search-index search.sist2
+sist2 sqlite-index ./index2.sist2 --search-index search.sist2
+```
+
+**Save index in JSON format**
+```bash
+sist2 index --print ./my_index.sist2 > my_index.ndjson
+```
+
+**Inspect contents of an index**
+```bash
+sist2 index --print ./my_index.sist2 | jq | less
+```
+
+## Web
+
+### Web examples
+
+**Single index (Elasticsearch backend)**
+```bash
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index.sist2
+```
+
+**Multiple indices (Elasticsearch backend)**
+```bash
+# Indices will be displayed in this order in the web interface
+sist2 web index1.sist2 index2.sist2 index3.sist2 index4.sist2
+```
+
+**SQLite search backend**
+```bash
+sist2 web --search-index search.sist2 index1.sist2
+```
+
+#### Auth0 authentication
+
+See [auth0.md](auth0.md)
+
+### rewrite_url
+
+When the `rewrite_url` field is not empty, the web module ignores the `root`
+field and will return a HTTP redirect to `<rewrite_url><path>/<name><extension>`
+instead of serving the file from disk. 
+Both the `root` and `rewrite_url` fields are safe to manually modify from the 
+`descriptor.json` file.
+
+# Elasticsearch
+
+Elasticsearch versions >=6.8.0, 7.X.X and 8.X.X are supported by sist2. 
+
+Using a version >=7.14.0 is recommended to enable the following features:
+
+- Bug fix for large documents (See #198)
+
+Using a version >=8.0.0 is recommended to enable the following features:
+
+- Approximate KNN search for Embeddings search (faster queries).
+
+When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
+If you don't care about the features above, you can ignore it or disable it in the configuration page.
+
+# Embeddings search
+
+Since v3.2.0, User scripts can be used to generate _embeddings_ (vector of float32 numbers) which are stored in the .sist2 index file 
+(see [scripting](scripting.md)). Embeddings can be used for:
+
+* Nearest-neighbor queries (e.g. "return the documents most similar to this one")
+* Semantic searches (e.g. "return the documents that are most closely related to the given topic")
+
+In theory, embeddings can be created for any type of documents (image, text, audio etc.).
+
+For example, the [clip](https://github.com/sist2app/sist2-script-clip) User Script, generates 512-d embeddings of images
+(videos are also supported using the thumbnails generated by sist2). When the user enters a query in the "Embeddings Search"
+textbox, the query's embedding is generated in their browser, leveraging the ONNX web runtime.
+
+<details>
+  <summary>Screenshots</summary>
+ 
+ ![embeddings-1](embeddings-1.png)
+ ![embeddings-2](embeddings-2.png)
+
+1. Embeddings search bar. You can select the model using the dropdown on the left.
+2. This icon appears for indices with embeddings search enabled.
+3. Documents with this icon have embeddings. Click on the icon to perform KNN search.
+</details>
+
+
+# Tagging
+
+### Manual tagging
+
+You can modify tags of individual documents directly from the 
+ `web` interface. Note that you can setup authentication for this feature
+ with the `--tag-auth` option (See [web options](#web-options))
+
+![manual_tag](manual_tag.png)
+
+Tags that are manually added are saved both in the 
+ index folder (in `/tags/`) and in Elasticsearch*. When re-`index`ing, 
+ they are read from the index and automatically applied.
+ 
+You can safely copy the `/tags/` database to another index.
+
+See [Automatic tagging](#automatic-tagging) for information about tag 
+ hierarchies and tag colors.
+
+\* *It can take a few seconds to take effect in new search queries.*
+
+
+### Automatic tagging
+
+See [scripting](scripting.md) documentation.

docs/auth0.md

@@ -0,0 +1,19 @@
+
+# Authentication with Auth0
+
+1. Create a new Auth0 application (Single page app)
+2. Create a new Auth0 API:
+   1. Choose `RS256` signing algorithm
+   2. Set identifier (audience) to `https://sist2`
+3. Download the Auth0 certificate from https://<domain>.auth0.com/pem (you can find the domain Applications->Basic information)
+4. Extract the public key from the certificate using `openssl x509 -pubkey -noout -in cert.pem > pubkey.txt`
+5. Start the sist2 web server
+
+Example options:
+```bash
+sist2 web \
+  --auth0-client-id XXX \
+  --auth0-audience https://sist2 \
+  --auth0-domain YYY.auth0.com \
+  --auth0-public-key-file /ZZZ/pubkey.txt
+```

docs/scripting.md

@@ -1,18 +1,47 @@
 ## User scripts
 
-*This document is under construction, more in-depth guide coming soon*
+User scripts are used to augment your sist2 index with additional metadata, neural network embeddings, tags etc.
+
+
+Since version 3.2.0, user scripts are written in Python, and are ran against the sist2 index file. User scripts do not 
+need a connection to the search backend.
+
+You can create a user script based on a template from the sist2-admin interface:
+
+![sist2-admin-scripts](sist2-admin-scripts.png)
+
+User scripts leverage the [sist2-python](https://github.com/simon987/sist2-python) library to interface with the
+index file*. You can find sist2-python documentation and examples
+here: [sist2-python.readthedocs.io](https://sist2-python.readthedocs.io/).
+
+If you are not using the sist2-admin interface, you can run user scripts manually from the command line:
+
+```
+pip install git+https://github.com/simon987/sist2-python.git
+
+python my_script.py /path/to/my_index.sist2
+```
+
+\* It is possible to manually update the index using raw SQL queries, but the database schema is not stable and
+can change at any time; it is recommended to use the more stable sist2-python wrapper instead.
+
+<hr>
+
+<details>
+  <summary>Legacy user scripts (sist2 version < 3.2.0)</summary>
 
 During the `index` step, you can use the `--script-file <script>` option to
 modify documents or add user tags. This option is mainly used to
 implement automatic tagging based on file attributes.
 
-The scripting language used 
+The scripting language used
-([Painless Scripting Language](https://www.elastic.co/guide/en/elasticsearch/painless/7.4/index.html)) 
+([Painless Scripting Language](https://www.elastic.co/guide/en/elasticsearch/painless/7.4/index.html))
 is very similar to Java, but you should be able to create user scripts
 without programming experience at all if you're somewhat familiar with
 regex.
 
 This is the base structure of the documents we're working with:
+
 ```json
 {
   "_id": "e171405c-fdb5-4feb-bb32-82637bc32084",
@@ -34,12 +63,13 @@ This is the base structure of the documents we're working with:
 **Example script**
 
 This script checks if the `genre` attribute exists, if it does
-it adds the `genre.<genre>` tag. 
+it adds the `genre.<genre>` tag.
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 if (ctx._source?.genre != null) {
-    tags.add("genre." + ctx._source.genre.toLowerCase())
+    tags.add("genre." + ctx._source.genre.toLowerCase());
 }
 ```
 
@@ -47,26 +77,34 @@ You can use `.` to create a hierarchical tag tree:
 
 ![scripting/genre_example](genre_example.png)
 
-
 To use regular expressions, you need to add this line in `/etc/elasticsearch/elasticsearch.yml`
+
 ```yaml
 script.painless.regex.enabled: true
 ```
+
 Or, if you're using docker add `-e "script.painless.regex.enabled=true"`
 
+**Tag color**
+
+You can specify the color for an individual tag by appending an
+hexadecimal color code (`#RRGGBBAA`) to the tag name.
+
 ### Examples
 
 If `(20XX)` is in the file name, add the `year.<year>` tag:
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 Matcher m = /[\(\.+](20[0-9]{2})[\)\.+]/.matcher(ctx._source.name);
 if (m.find()) {
-    tags.add("year." + m.group(1))
+    tags.add("year." + m.group(1));
 }
 ```
 
 Use default *Calibre* folder structure to infer author.
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -79,8 +117,9 @@ if (ctx._source.name.contains("-") && ctx._source.extension == "pdf") {
 }
 ```
 
-If the file matches a specific pattern `AAAA-000 fName1 lName1, <fName2 lName2>...`, add the `actress.<actress>` and 
+If the file matches a specific pattern `AAAA-000 fName1 lName1, <fName2 lName2>...`, add the `actress.<actress>` and
 `studio.<studio>` tag:
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -97,21 +136,47 @@ if (m.find()) {
 ```
 
 Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 if (ctx._source.path != "") {
-    String[] names = ctx._source.path.splitOnToken('/');
+    String[] names = ctx._source.path.splitOnToken('/');
     tags.add("studio." + names[names.length-1]);
 }
 ```
 
-Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
+Parse `EXIF:F Number` tag
+
 ```Java
-ArrayList tags = ctx._source.tag = new ArrayList();
+if (ctx._source?.exif_fnumber != null) {
-
+    String[] values = ctx._source.exif_fnumber.splitOnToken(' ');
-if (ctx._source.path != "") {
+    String aperture = String.valueOf(Float.parseFloat(values[0]) / Float.parseFloat(values[1]));
-    String[] names = ctx._source.path.splitOnToken('/');
+    if (aperture == "NaN") {
-    tags.add("studio." + names[names.length-1]);
+        aperture = "0,0";
+    }
+    tags.add("Aperture.f/" + aperture.replace(".", ","));
 }
 ```
+
+Display year and months from `EXIF:DateTime` tag
+
+```Java
+if (ctx._source?.exif_datetime != null) {
+    SimpleDateFormat parser = new SimpleDateFormat("yyyy:MM:dd HH:mm:ss");
+    Date date = parser.parse(ctx._source.exif_datetime);
+
+    SimpleDateFormat yp = new SimpleDateFormat("yyyy");
+    SimpleDateFormat mp = new SimpleDateFormat("MMMMMMMMM");
+
+    String year = yp.format(date);
+    String month = mp.format(date);
+
+    tags.add("Month." + month);
+    tags.add("Year." + year);
+}
+
+```
+
+</details>
+

include/mce/config.h

@@ -1,53 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
- notice, this list of conditions and the following disclaimer in 
- the documentation and/or other materials provided with the 
- distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
- may be used to endorse or promote products derived from this 
- software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-/**@file config/mce/config.h
- */
-#ifndef MCE_CONFIG_H
-#define MCE_CONFIG_H
-
-#include <libxml/xmlstring.h>
-#include <stdio.h>
-#include <plib/plib.h>
-#include <assert.h>
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define MCE_NAMESPACE_SUBSUMPTION_ENABLED 0
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif  
-
-#endif /* MCE_CONFIG_H */

include/mce/helper.h

@@ -1,189 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file mce/helper.h
-Helper functions needed by mce/textreader.h and mce/textwriter.h to implement MCE:
-- mceQNameLevelAdd(), mceQNameLevelLookup() and mceQNameLevelCleanup() maintain a set of mceQNameLevel_t tuples.
-- mceQNameLevelPush() and mceQNameLevelPopIfMatch() maintain a stack of mceQNameLevel_t tuples.
-- mceCtxInit(), mceCtxCleanup() and mceCtxUnderstandsNamespace() manage a context which holds all information needed to do MCE proprocessing.
- */
-#include <mce/config.h>
-
-#ifndef MCE_HELPER_H
-#define MCE_HELPER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-    /**
-      Tiple (ns, ln, level).
-    */
-    typedef struct MCE_QNAME_LEVEL {
-        xmlChar *ns;
-        xmlChar *ln;
-        puint32_t level;
-        puint32_t flag; // used by mceTextWriter
-    } mceQNameLevel_t;
-
-    /**
-     */
-    typedef enum MCE_SKIP_STATE_ENUM {
-        MCE_SKIP_STATE_IGNORE,
-        MCE_SKIP_STATE_ALTERNATE_CONTENT,
-        MCE_SKIP_STATE_CHOICE_MATCHED
-    } mceSkipState_t;
-
-    /**
-     Represents an intervall of levels which are "skipped" i.e. ignored.
-     */
-    typedef struct MCE_SKIP_ITEM {
-        puint32_t level_start;
-        puint32_t level_end;
-        mceSkipState_t state;
-    } mceSkipItem_t;
-
-    /**
-      Either represents a set of (ns, ln, level) triples.
-    */
-    typedef struct MCE_QNAME_LEVEL_SET {
-        mceQNameLevel_t *list_array;
-        puint32_t list_items;
-        puint32_t max_level;
-    } mceQNameLevelSet_t;
-
-    /**
-     The skip stack.
-     */
-    typedef struct MCE_SKIP_STACK {
-        mceSkipItem_t *stack_array;
-        puint32_t stack_items;
-    } mceSkipStack_t;
-
-
-    typedef enum MCE_ERROR_ENUM {
-        MCE_ERROR_NONE,
-        MCE_ERROR_XML,
-        MCE_ERROR_MUST_UNDERSTAND,
-        MCE_ERROR_VALIDATION,
-        MCE_ERROR_MEMORY
-    } mceError_t;
-
-    /**
-      Holds all information to do MCE preprocessing.
-    */
-    typedef struct MCE_CONTEXT {
-        mceQNameLevelSet_t ignorable_set;
-        mceQNameLevelSet_t understands_set;
-        mceQNameLevelSet_t processcontent_set;
-        mceQNameLevelSet_t suspended_set;
-#if (MCE_NAMESPACE_SUBSUMPTION_ENABLED)
-        mceQNameLevelSet_t subsume_namespace_set;
-        mceQNameLevelSet_t subsume_exclude_set;
-        mceQNameLevelSet_t subsume_prefix_set;
-#endif
-        mceSkipStack_t skip_stack;
-        mceError_t error;
-        pbool_t mce_disabled;        
-        puint32_t suspended_level;
-    } mceCtx_t;
-
-    /**
-      Add a new tiple (ns, ln, level) to the triple set \c qname_level_set.
-      The \c ns_sub string is optional and will not be touched.
-    */
-    pbool_t mceQNameLevelAdd(mceQNameLevelSet_t *qname_level_set, const xmlChar *ns, const xmlChar *ln, puint32_t level);
-
-    /**
-      Lookup a tiple (ns, ln, level) via \c ns and \c ln. If \c ignore_ln is PTRUE then the first tiple matching \c ns will be returned.
-    */
-    mceQNameLevel_t* mceQNameLevelLookup(mceQNameLevelSet_t *qname_level_set, const xmlChar *ns, const xmlChar *ln, pbool_t ignore_ln);
-
-    /**
-      Remove all triples (ns, ln, level) where the level greater or equal to \c level.
-    */
-    pbool_t mceQNameLevelCleanup(mceQNameLevelSet_t *qname_level_set, puint32_t level);
-
-    /**
-      Push a new skip intervall (level_start, level_end, state) on the stack \c skip_stack.
-    */
-    pbool_t mceSkipStackPush(mceSkipStack_t *skip_stack, puint32_t level_start, puint32_t level_end, mceSkipState_t state);
-
-    /**
-      Pop the intervall (ns, ln, level) from the stack \c qname_level_array.
-    */
-    void mceSkipStackPop(mceSkipStack_t *skip_stack);
-
-    /**
-     Returns top item or NULL.
-     */
-    mceSkipItem_t *mceSkipStackTop(mceSkipStack_t *skip_stack);
-
-    /**
-     Returns TRUE, if the \c level is in the top skip intervall.
-     */
-    pbool_t mceSkipStackSkip(mceSkipStack_t *skip_stack, puint32_t level);
-
-    /**
-      Initialize the mceCtx_t \c ctx.
-    */
-    pbool_t mceCtxInit(mceCtx_t *ctx);
-
-    /**
-      Cleanup, i.e. release all resourced from the mceCtx_t \c ctx.
-    */
-    pbool_t mceCtxCleanup(mceCtx_t *ctx);
-
-    /**
-      Register the namespace \ns in \c ctx.
-    */
-    pbool_t mceCtxUnderstandsNamespace(mceCtx_t *ctx, const xmlChar *ns);
-
-    /**
-     Register the namespace \ns in \c ctx.
-     */
-    pbool_t mceCtxSuspendProcessing(mceCtx_t *ctx, const xmlChar *ns, const xmlChar *ln);
-    
-
-
-#if (MCE_NAMESPACE_SUBSUMPTION_ENABLED)
-    /**
-    Subsume namespace \c ns_new with \c ns_old.
-     */
-    pbool_t mceCtxSubsumeNamespace(mceCtx_t *ctx, const xmlChar *prefix_new, const xmlChar *ns_new, const xmlChar *ns_old);
-#endif
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif /* MCE_HELPER_H */

include/mce/textreader.h

@@ -1,464 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file mce/textreader.h
- 
- */
-#ifndef MCE_TEXTREADER_H
-#define MCE_TEXTREADER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-    /**
-      A handle to an MCE-aware libxml2 xmlTextReader.
-    */
-    typedef struct MCE_TEXTREADER mceTextReader_t;
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-
-#include <mce/config.h>
-#include <opc/opc.h>
-#include <mce/helper.h>
-#include <libxml/xmlwriter.h>
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-    struct MCE_TEXTREADER {
-        xmlTextReaderPtr reader;
-        mceCtx_t mceCtx;
-    };
-
-    /**
-      Wrapper around an libxml2 xmlTextReaderRead function.
-      \see http://xmlsoft.org/html/libxml-xmlreader.html#xmlTextReaderRead
-    */
-    int mceTextReaderRead(mceTextReader_t *mceTextReader);
-
-    /**
-      Wrapper around a libxml2 xmlTextReaderNext function.
-      \see http://xmlsoft.org/html/libxml-xmlreader.html#xmlTextReaderNext
-    */
-    int mceTextReaderNext(mceTextReader_t *mceTextReader);
-
-    /** 
-      Creates an mceTextReader from an XmlTextReader. 
-      \code
-      mceTextReader reader;
-      mceTextReaderInit(&reader, xmlNewTextReaderFilename("sample.xml"));
-      // reader is ready to use.
-      mceTextReaderCleanup(&reader);
-      \endcode
-      \see http://xmlsoft.org/html/libxml-xmlreader.html#xmlNewTextReaderFilename
-    */
-    int mceTextReaderInit(mceTextReader_t *mceTextReader, xmlTextReaderPtr reader);
-
-    /**
-      Cleanup MCE reader, i.e. free all resources. Also calls xmlTextReaderClose and xmlFreeTextReader.
-      \see http://xmlsoft.org/html/libxml-xmlreader.html#xmlTextReaderClose
-      \see http://xmlsoft.org/html/libxml-xmlreader.html#xmlFreeTextReader
-    */
-    int mceTextReaderCleanup(mceTextReader_t *mceTextReader);
-
-    /** 
-      Reads all events \c mceTextReader and pipes them to \writer.
-      \code
-      mceTextReader reader;
-      mceTextReaderInit(&reader, xmlNewTextReaderFilename("sample.xml"));
-      mceTextReaderUnderstandsNamespace(&reader, _X("http://myextension"));
-      xmlTextWriterPtr writer=xmlNewTextWriterFilename("out.xml", 0);
-      mceTextReaderDump(&reader, writer, P_FALSE);
-      xmlFreeTextWriter(writer);
-      mceTextReaderCleanup(&reader);
-      \endcode
-      */
-    int mceTextReaderDump(mceTextReader_t *mceTextReader, xmlTextWriter *writer, pbool_t fragment);
-
-    /**
-      Registers an MCE namespace.
-      \see mceTextReaderDump()
-      */
-    int mceTextReaderUnderstandsNamespace(mceTextReader_t *mceTextReader, const xmlChar *ns);
-
-    /**
-     Disable MCE processing.
-     \return Returns old value.
-     */
-    pbool_t mceTextReaderDisableMCE(mceTextReader_t *mceTextReader, pbool_t flag);
-
-
-    /**
-     Signal an error to the MCE processor.
-     */
-    void mceRaiseError(xmlTextReader *reader, mceCtx_t *ctx, mceError_t error, const xmlChar *str, ...);
-
-    /**
-        Internal function which does the MCE postprocessing. E.g. mceTextReaderRead() is implemented as
-        \code
-        mceTextReaderPostprocess(mceTextReader->reader, &mceTextReader->mceCtx, xmlTextReaderRead(mceTextReader->reader))
-        \endcode
-        This function is exposed to make existing libxm2 xmlTextReader MCE aware.
-    */
-    int mceTextReaderPostprocess(xmlTextReader *reader, mceCtx_t *ctx, int ret);
-
-    /**
-     Get the error code.
-     */
-    mceError_t mceTextReaderGetError(mceTextReader_t *mceTextReader);
-
-/**
- Helper macro to declare a start/end document block in a declarative way:
- \code
-  mce_start_document(reader) {
-  } mce_end_document(reader);
-  \endcode
-  \hideinitializer
-*/
-#define mce_start_document(_reader_) \
-    if (NULL!=(_reader_)) {            \
-        mceTextReaderRead(_reader_); \
-        if (0)                     
-
-/**
-  \see mce_start_document.
-  \hideinitializer
-*/
-#define mce_end_document(_reader_)   \
-    } /* if (NULL!=reader) */        \
-
-
-/**
-  Container for mce_start_element and mce_start_attribute declarations.
-  \see mce_match_element
-  \see mce_match_attribute
-  \hideinitializer
-  */
-#define mce_start_choice(_reader_)  \
-    if (0)                          
-
-/**
-  \see mce_start_choice
-  \hideinitializer
-  */
-#define mce_end_choice(_reader_) 
-
-
-/**
-  Skips the attributes. 
-  \see mce_match_element.
-  \hideinitializer
-*/
-#define mce_skip_attributes(_reader_) \
-    mce_start_attributes(_reader_) {  \
-    } mce_end_attributes(_reader_);   
-
-
-/**
-  Skips the attributes. 
-  \see mce_match_attribute.
-  \hideinitializer
-*/
-#define mce_skip_children(_reader_) \
-    mce_start_children(_reader_) {  \
-    } mce_end_children(_reader_);   
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_start_children(_reader_)                  \
-if (!xmlTextReaderIsEmptyElement((_reader_)->reader)) { \
-    mceTextReaderRead(_reader_); do {                 \
-        if (0)                                        
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_end_children(_reader_)                                                      \
-        else {                                                                          \
-            if (XML_READER_TYPE_END_ELEMENT!=xmlTextReaderNodeType((_reader_)->reader)) { \
-                mceTextReaderNext(_reader_); /*skip unhandled element */                \
-            }                                                                           \
-        }                                                                               \
-    } while(XML_READER_TYPE_END_ELEMENT!=xmlTextReaderNodeType((_reader_)->reader) &&     \
-            XML_READER_TYPE_NONE!=xmlTextReaderNodeType((_reader_)->reader));             \
-} /* if (!xmlTextReaderIsEmptyElement(reader->reader)) */                               
-
-
-/**
-  Helper macro to match an element. Usefull for calling code in a seperate function:
-
-  \code
-  void handleElement(reader) {
-    mce_start_choice(reader) {
-        mce_start_element(reader, _X("ns"), _X("element")) {
-            
-        } mce_end_element(reader)
-    } mce_end_choice(reader);
-  }
-
-  void parse(reader) {
-    mce_start_document(reader) {
-      mce_start_element(reader, _X("ns"), _X("ln")) {
-        mce_skip_attributes(reader);
-        mce_start_children(reader) {
-           mce_match_element(reader, _X("ns"), _X("element")) {
-             handleElement(reader);
-           }
-        } mce_end_children(reader);
-      } mce_end_element();
-    } mce_end_document(reader);
-  }
-  \endcode
-  \hideinitializer
-*/
-#define mce_match_element(_reader_, ns, ln)                                                       \
-    } else if (XML_READER_TYPE_ELEMENT==xmlTextReaderNodeType((_reader_)->reader)                 \
-            && (NULL==ns || 0==xmlStrcmp(ns, xmlTextReaderConstNamespaceUri((_reader_)->reader))) \
-            && (NULL==ln || 0==xmlStrcmp(ln, xmlTextReaderConstLocalName((_reader_)->reader)))) { 
-
-
-/**
- Helper macro to declare a element block in a declarative way:
- \code
-  mce_start_element(reader) {
-    mce_start_attributes(reader) {
-      mce_start_attribute(reader, _X("ns"), _X("lnA")) {
-         // code for handling lnA.
-      } mce_end_attribute(reader);
-      mce_start_attribute(reader, _X("ns"), _X("lnB")) {
-         // code for handling lnB.
-      } mce_end_attribute(reader);
-    } mce_end_attributes(reader);
-    mce_start_children(reader) {
-        mce_start_element(reader, _X("ns"), _X("lnA")) {
-         // code for handling lnA.
-        } mce_end_element(reader);
-        mce_start_element(reader, _X("ns"), _X("lnB")) {
-         // code for handling lnB.
-        } mce_end_element(reader);
-        mce_start_text(reader) {
-         // code for handling text.
-        } mce_end_text(reader);
-    } mce_end_children(reader);
-  } mce_end_element(reader);
-  \endcode
-  \hideinitializer
-*/
-#define mce_start_element(_reader_, ns, ln) \
-    mce_match_element(_reader_, ns, ln)     
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_end_element(_reader_) \
-    mceTextReaderNext(_reader_)   
-
-/**
-  Matches #TEXT without consuming it.
-  \hideinitializer
-*/
-#define mce_match_text(_reader_)                                                                   \
-    } else if (XML_READER_TYPE_TEXT==xmlTextReaderNodeType((_reader_)->reader)                     \
-            || XML_READER_TYPE_SIGNIFICANT_WHITESPACE==xmlTextReaderNodeType((_reader_)->reader)) {
-
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_start_text(_reader_) \
-    mce_match_text(_reader_)      
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_end_text(_reader_) \
-    mceTextReaderNext(_reader_)
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_start_attributes(_reader_)                            \
-    if (1==xmlTextReaderMoveToFirstAttribute((_reader_)->reader)) { \
-        do {                                                      \
-            if (0)                                                
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_end_attributes(_reader_)                                    \
-            else { /* skipped attribute */ }                            \
-        } while(1==xmlTextReaderMoveToNextAttribute((_reader_)->reader)); \
-    xmlTextReaderMoveToElement((_reader_)->reader); }                     
-
-/**
-  Helper macro to match an attribute. Usefull for calling code in a seperate function:
-
-  \code
-  void handleA(reader) {
-    mce_start_choice(reader) {
-        mce_start_attribute(reader, _X("ns"), _X("attr")) {
-
-        } mce_end_attribute(reader);
-    } mce_end_choice(reader);
-  }
-
-  void parse(reader) {
-    mce_start_document(reader) {
-      mce_start_element(reader, _X("ns"), _X("ln")) {
-        mce_start_attributes(reader) {
-           mce_match_attribute(reader, _X("ns"), _X("attr")) {
-             handleA(reader);
-           }
-        } mce_end_attributes(reader);
-        mce_skip_children(reader);
-      } mce_end_element();
-    } mce_end_document(reader);
-  }
-  \endcode
-  \hideinitializer
-*/
-#define mce_match_attribute(_reader_, ns, ln)                                                   \
-    } else if ((NULL==ns || 0==xmlStrcmp(ns, xmlTextReaderConstNamespaceUri((_reader_)->reader))) \
-            && (NULL==ln || 0==xmlStrcmp(ln, xmlTextReaderConstLocalName((_reader_)->reader)))) { 
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_start_attribute(_reader_, ns, ln) \
-    mce_match_attribute(_reader_, ns, ln) 
-
-/**
-  \see mce_start_element.
-  \hideinitializer
-*/
-#define mce_end_attribute(_reader_)
-
-
-/**
-  Error handling for MCE parsers.
-  \code
-   mce_start_element(&reader, NULL, _X("Default")) {
-       const xmlChar *ext=NULL;
-       const xmlChar *type=NULL;
-       mce_start_attributes(&reader) {
-           mce_start_attribute(&reader, NULL, _X("Extension")) {
-               ext=xmlTextReaderConstValue(reader.reader);
-           } mce_end_attribute(&reader);
-           mce_start_attribute(&reader, NULL, _X("ContentType")) {
-               type=xmlTextReaderConstValue(reader.reader);
-           } mce_end_attribute(&reader);
-       } mce_end_attributes(&reader);
-       mce_error_guard_start(&reader) {
-           mce_error(&reader, NULL==ext || ext[0]==0, MCE_ERROR_VALIDATION, "Missing @Extension attribute!");
-           mce_error(&reader, NULL==type || type[0]==0, MCE_ERROR_VALIDATION, "Missing @ContentType attribute!");
-           opcContainerType *ct=insertType(c, type, OPC_TRUE);
-           mce_error(&reader, NULL==ct, MCE_ERROR_MEMORY, NULL);
-           opcContainerExtension *ce=opcContainerInsertExtension(c, ext, OPC_TRUE);
-           mce_error(&reader, NULL==ce, MCE_ERROR_MEMORY, NULL);
-           mce_errorf(&reader, NULL!=ce->type && 0!=xmlStrcmp(ce->type, type), MCE_ERROR_VALIDATION, "Extension \"%s\" is mapped to type \"%s\" as well as \"%s\"", ext, type, ce->type);
-           ce->type=ct->type;
-       } mce_error_guard_end(&reader);
-       mce_skip_children(&reader);
-   } mce_end_element(&reader);
-  \endcode
-  \hideinitializer
-*/
-#define mce_error_guard_start(_reader_) if (MCE_ERROR_NONE==(_reader_)->mceCtx.error) do {
-
-/**
-  \see mce_error_guard_start
-  \hideinitializer
-*/
-#define mce_error_guard_end(_reader_)  } while(0)
-
-/**
-  Signal an error if guard if false.
-  \hideinitializer
-*/
-#define mce_error(_reader_, guard, err, msg) if (guard) { (_reader_)->mceCtx.error=(err); fprintf(stderr, (NULL!=msg?msg:#err));  continue; }
-
-/**
-  Signal an error if guard if false.
-  \hideinitializer
-*/
-#if defined(__GNUC__)
-#define mce_errorf(_reader_, guard, err, msg, ...) if (guard) { mceRaiseError((_reader_)->reader, &(_reader_)->mceCtx, err, _X((NULL!=msg?msg:#err)), ##__VA_ARGS__ );  continue; }
-#else
-#define mce_errorf(_reader_, guard, err, msg, ...) if (guard) { mceRaiseError((_reader_)->reader, &(_reader_)->mceCtx, err, _X((NULL!=msg?msg:#err)), __VA_ARGS__ );  continue; }
-#endif
-
-/**
-  Only issues the error when in "strict mode".
-  \hideinitializer
-*/
-#define mce_error_strict mce_error
-
-/**
-  \see mce_error_strict
-  \hideinitializer
-*/
-#define mce_error_strictf mce_errorf
-
-
-/**
-  Marker for a MCE defintion.
-  \hideinitializer
-*/
-#define mce_def
-
-/**
-  Marker for a MCE reference.
-  \hideinitializer
-*/
-#define mce_ref(r) (r)
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif /* MCE_TEXTREADER_H */

include/mce/textwriter.h

@@ -1,176 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file mce/textwriter.h
-
-*/
-#include <mce/config.h>
-#include <libxml/xmlwriter.h>
-#include <mce/helper.h>
-
-#ifndef MCE_TEXTWRITER_H
-#define MCE_TEXTWRITER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-/**
-  Default flags for an MCE namespace declaration.
-  */
-#define MCE_DEFAULT 0x0
-
-/**
-  Flags MCE namespace declaration "ignorable".
-  */
-#define MCE_IGNORABLE 0x1
-
-/**
-  Flags MCE namespace declaration "must understand".
-  */
-#define MCE_MUSTUNDERSTAND 0x2
-
-    /**
-      The MCE text writer context.
-      */
-    typedef struct MCE_TEXTWRITER_STRUCT mceTextWriter;
-
-    /**
-      Create a new MCE text writer.
-      \see http://xmlsoft.org/html/libxml-xmlIO.html#xmlOutputBufferCreateIO
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlNewTextWriter
-      */
-    mceTextWriter *mceTextWriterCreateIO(xmlOutputWriteCallback iowrite, xmlOutputCloseCallback  ioclose, void *ioctx, xmlCharEncodingHandlerPtr encoder);
-
-    /**
-      Helper which create a new MCE text writer for a FILE handle.
-      */
-    mceTextWriter *mceNewTextWriterFile(FILE *file);
-
-    /**
-      Free all resources for \w.
-      */
-    int mceTextWriterFree(mceTextWriter *w);
-
-    /**
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlTextWriterStartDocument
-      */
-    int mceTextWriterStartDocument(mceTextWriter *w);
-
-    /**
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlTextWriterEndDocument
-      */
-    int mceTextWriterEndDocument(mceTextWriter *w);
-
-    /**
-      Start a new XML element. If ns==NULL then there is no namespace and ""==ns means the default namespace.
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlTextWriterStartElement
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlTextWriterStartElementNS
-      */
-    int mceTextWriterStartElement(mceTextWriter *w, const xmlChar *ns, const xmlChar *ln);
-
-    /**
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlTextWriterEndElement
-      */
-    int mceTextWriterEndElement(mceTextWriter *w, const xmlChar *ns, const xmlChar *ln);
-
-    /**
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlTextWriterWriteString
-      */
-    int mceTextWriterWriteString(mceTextWriter *w, const xmlChar *content);
-
-    /**
-      Register a namespace. Must be called before mceTextWriterStartElement.
-      \see MCE_DEFAULT
-      \see MCE_IGNORABLE
-      \see MCE_MUSTUNDERSTAND
-      */
-    const xmlChar *mceTextWriterRegisterNamespace(mceTextWriter *w, const xmlChar *ns, const xmlChar *prefix, int flags);
-
-    /**
-      Register qname (ns, ln) as a "process content" element wrt. MCE. Must be called before mceTextWriterStartElement.
-      */
-    int mceTextWriterProcessContent(mceTextWriter *w, const xmlChar *ns, const xmlChar *ln);
-
-    /**
-      Writes a formatted attribute.
-      \see http://xmlsoft.org/html/libxml-xmlwriter.html#xmlTextWriterWriteFormatAttribute
-      */
-    int mceTextWriterAttributeF(mceTextWriter *w, const xmlChar *ns, const xmlChar *ln, const char *value, ...);
-
-    /**
-      Starts an MCE alternate content section.
-      */
-    int mceTextWriterStartAlternateContent(mceTextWriter *w);
-
-    /**
-      Ends an MCE alternate content section.
-      */
-    int mceTextWriterEndAlternateContent(mceTextWriter *w);
-
-    /**
-      Start an MCE choice.
-      */
-    int mceTextWriterStartChoice(mceTextWriter *w, const xmlChar *ns);
-
-    /**
-      Ends an MCE choice.
-      */
-    int mceTextWriterEndChoice(mceTextWriter *w);
-
-    /**
-      Start an MCE fallback.
-      */
-    int mceTextWriterStartFallback(mceTextWriter *w);
-
-    /**
-      Ends an MCE fallback.
-      */
-    int mceTextWriterEndFallback(mceTextWriter *w);
-
-
-    /**
-      Returns the underlying xmlTextWriter.
-      */
-    xmlTextWriterPtr mceTextWriterIntern(mceTextWriter *w);
-
-    /**
-      Helper which create a new xmlTextWriterPtr for a FILE handle.
-      */
-    xmlTextWriterPtr xmlNewTextWriterFile(FILE *file);
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-
-#endif /* MCE_TEXTWRITER_H */

include/opc/config.h

@@ -1,189 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
- notice, this list of conditions and the following disclaimer in 
- the documentation and/or other materials provided with the 
- distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
- may be used to endorse or promote products derived from this 
- software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-/**@file config/opc/config.h
- */
-#ifndef OPC_CONFIG_H
-#define OPC_CONFIG_H
-
-#include <libxml/xmlstring.h>
-#include <plib/plib.h>
-#include <assert.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif   
-
-
-/**
-  Assert expression e is true. Will be removed entirely in release mode.
-  \hideinitializer
- */
-#define OPC_ASSERT(e) assert(e)
-
-/**
-  Assert expression e is true. Expression will be executed in release mode too.
-  \hideinitializer
- */
-#ifdef NDEBUG
-#define OPC_ENSURE(e) (void)(e)
-#else
-#define OPC_ENSURE(e) assert(e)
-#endif
-
-
-/**
-  Constant for boolean true.
-  \hideinitializer
- */
-#define OPC_TRUE (0==0)
-
-/**
-  Constant for boolean false.
-  \hideinitializer
- */
-#define OPC_FALSE (0==1)
-
-    /** 
-      Boolean type.
-      \hideinitializer
-      */
-    typedef pbool_t opc_bool_t;
-
-    /** 
-      Type which represents an offset in e.g. a file.
-      \hideinitializer
-      */
-    typedef pofs_t opc_ofs_t;
-
-    /** 
-      8-bit unsigned integer.
-      \hideinitializer
-      */
-    typedef puint8_t opc_uint8_t;
-
-    /** 
-      16-bit unsigned integer.
-      \hideinitializer
-      */
-    typedef puint16_t opc_uint16_t;
-
-    /** 
-      32-bit unsigned integer.
-      \hideinitializer
-      */
-    typedef puint32_t opc_uint32_t;
-
-    /** 
-      64-bit unsigned integer.
-      \hideinitializer
-      */
-    typedef puint64_t opc_uint64_t;
-
-    /** 
-      8-bit signed integer.
-      \hideinitializer
-      */
-    typedef pint8_t opc_int8_t;
-
-    /** 
-      16-bit signed integer.
-      \hideinitializer
-      */
-    typedef pint16_t opc_int16_t;
-
-    /** 
-      32-bit signed integer.
-      \hideinitializer
-      */
-    typedef pint32_t opc_int32_t;
-
-    /** 
-      64-bit signed integer.
-      \hideinitializer
-      */
-    typedef pint64_t opc_int64_t;
-
-/**
-  Default size fo the deflate buffer used by zlib.
-  */
-#define OPC_DEFLATE_BUFFER_SIZE 4096
-
-/**
-  Max system path len.
-  */
-#define OPC_MAX_PATH 512
-
-    /**
-      Error codes for the OPC module.
-      */
-    typedef enum OPC_ERROR_ENUM {
-        OPC_ERROR_NONE,
-        OPC_ERROR_STREAM,
-        OPC_ERROR_SEEK, // can't seek
-        OPC_ERROR_UNSUPPORTED_DATA_DESCRIPTOR,
-        OPC_ERROR_UNSUPPORTED_COMPRESSION,
-        OPC_ERROR_DEFLATE,
-        OPC_ERROR_HEADER,
-        OPC_ERROR_MEMORY,
-        OPC_ERROR_XML, 
-        OPC_ERROR_USER // user triggered an abort
-    } opc_error_t;
-    
-    /**
-      Compression options for OPC streams.
-      */
-    typedef enum OPC_COMPRESSIONOPTION_ENUM {
-        OPC_COMPRESSIONOPTION_NONE,
-        OPC_COMPRESSIONOPTION_NORMAL,
-        OPC_COMPRESSIONOPTION_MAXIMUM,
-        OPC_COMPRESSIONOPTION_FAST,
-        OPC_COMPRESSIONOPTION_SUPERFAST
-    } opcCompressionOption_t;
-
-
-/**
-  Helper for debug logs.
-  \hideinitializer
-  */
-#define opc_logf printf
-
-/**
-  Abstraction for memset(m, 0, s).
-  \hideinitializer
- */
-#define opc_bzero_mem(m,s) memset(m, 0, s)
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif  
-
-#endif /* OPC_CONFIG_H */

include/opc/container.h

@@ -1,300 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/container.h
-
- The container.h module has the fundamental methods for dealing with ZIP-based OPC container. 
- 
- OPC container can be opened in READ-ONLY mode, WRITE-ONLY mode, READ/WRITE mode, TEMPLATE mode and TRANSITION mode. 
- The most notable mode is the READ/WRITE mode, which gives you concurrent stream-based READ and WRITE access to a 
- single ZIP-based OPC container. This is achieved without the use of temporary files by taking advantage of the 
- OPC specific “interleave” mode. \see http://standards.iso.org/ittf/PubliclyAvailableStandards/c051459_ISOIEC_29500-2_2008(E).zip
- 
- The TEMPLATE mode allows very fast customized "cloning" of ZIP-based OPC container by using "RAW access" to the ZIP streams. 
- The TRANSITION mode is a special version of the TEMPLATE mode, which allows transition-based READ/WRITE access to the 
- ZIP-based OPC container using a temporary file.
- 
- */
-#include <opc/config.h>
-#include <opc/file.h>
-
-#ifndef OPC_CONTAINER_H
-#define OPC_CONTAINER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-    /**
-     Handle to an OPC container created by \ref opcContainerOpen.
-     \see opcContainerOpen.
-     */
-    typedef struct OPC_CONTAINER_STRUCT opcContainer;
-    
-    /**
-     Modes for opcContainerOpen();
-     \see opcContainerOpen
-     */
-    typedef enum {
-        /**
-         Opens the OPC container denoted by \a fileName in READ-ONLY mode. The \a destName parameter must be \a NULL.
-         \hideinitializer
-         */
-        OPC_OPEN_READ_ONLY=0, 
-        /**
-         Opens the OPC container denoted by \a fileName in WRITE-ONLY mode. The \a destName parameter must be \a NULL.
-         \hideinitializer
-         */
-        OPC_OPEN_WRITE_ONLY=1,
-        /**
-         Opens the OPC container denoted by \a fileName in READ/WRITE mode. The \a destName parameter must be \a NULL.
-         \hideinitializer
-         */
-        OPC_OPEN_READ_WRITE=2,
-        /**
-         This mode will open the container denoted by \a fileName in READ-ONLY mode and the container denoted by 
-         \a destName in write-only mode. Any modifications will be written to the container denoted by \a destName 
-         and the unmodified streams from \a fileName will be written to \a destName on closing.
-         \warning Currently not implemented.
-         \hideinitializer
-         */
-        OPC_OPEN_TEMPLATE=3,
-        /**
-         Like the OPC_OPEN_TEMPLATE mode, but the \a destName will be renamed to the \a fileName on closing. If \a destName 
-         is \a NULL, then the name of the temporary file will be generated automatically.
-         \warning Currently not implemented.
-         \hideinitializer
-         */
-        OPC_OPEN_TRANSITION=4
-    } opcContainerOpenMode; 
-    
-    /** Modes for opcContainerClose.
-     \see opcContainerClose.
-     */
-    typedef enum {
-        /**
-         Close the OPC container without any further postprocessing.
-         \hideinitializer
-         */
-        OPC_CLOSE_NOW = 0,
-        /**
-         Close the OPC container and trim the file by removing unused fragments like e.g. 
-         deleted parts.
-         \hideinitializer
-         */
-        OPC_CLOSE_TRIM = 1,
-        /**
-         Close the OPC container like in \a OPC_CLOSE_TRIM mode, but additionally remove any 
-         "interleaved" parts by reordering them.
-         \warning Currently not implemented. Same semantic as OPC_CLOSE_TRIM.       
-         \hideinitializer
-         */
-        OPC_CLOSE_DEFRAG = 2
-    } opcContainerCloseMode;
-    
-    /**
-     Opens a ZIP-based OPC container.
-     @param[in] fileName. For more details see \ref opcContainerOpenMode.
-     @param[in] mode. For more details see \ref opcContainerOpenMode.
-     @param[in] userContext. Will not be modified by libopc. Can be used to e.g. store the "this" pointer for C++ bindings.
-     @param[in] destName. For more details see \ref opcContainerOpenMode.
-     @return \a NULL if failed. 
-     \see opcContainerOpenMode
-     \see opcContainerDump
-     */
-    opcContainer* opcContainerOpen(const xmlChar *fileName, 
-                                   opcContainerOpenMode mode, 
-                                   void *userContext, 
-                                   const xmlChar *destName);
-
-    /**
-     Opens a ZIP-based OPC container from memory.
-     @param[in] data. 
-     @param[in] data_len.
-     @param[in] userContext. Will not be modified by libopc. Can be used to e.g. store the "this" pointer for C++ bindings.
-     @param[in] mode. For more details see \ref opcContainerOpenMode.
-     @return \a NULL if failed. 
-     */
-    opcContainer* opcContainerOpenMem(const opc_uint8_t *data, opc_uint32_t data_len,
-                                      opcContainerOpenMode mode, 
-                                      void *userContext);
-
-    /**
-     Opens a ZIP-based OPC container from memory.
-     @param[in] ioread. 
-     @param[in] iowrite. 
-     @param[in] ioclose. 
-     @param[in] ioseek. 
-     @param[in] iotrim. 
-     @param[in] ioflush. 
-     @param[in] iocontext. 
-     @param[in] file_size. 
-     @param[in] userContext. Will not be modified by libopc. Can be used to e.g. store the "this" pointer for C++ bindings.
-     @param[in] mode. For more details see \ref opcContainerOpenMode.
-     @return \a NULL if failed. 
-     */
-    opcContainer* opcContainerOpenIO(opcFileReadCallback *ioread,
-                                     opcFileWriteCallback *iowrite,
-                                     opcFileCloseCallback *ioclose,
-                                     opcFileSeekCallback *ioseek,
-                                     opcFileTrimCallback *iotrim,
-                                     opcFileFlushCallback *ioflush,
-                                     void *iocontext,
-                                     pofs_t file_size,
-                                     opcContainerOpenMode mode, 
-                                     void *userContext);
-    
-    /**
-     Close an OPC container.
-     @param[in] c. \ref opcContainer openered by \ref opcContainerOpen.
-     @param[in] mode. For more information see \ref opcContainerCloseMode.
-     @return Non-zero if successful.
-     \see opcContainerOpen
-     \see opcContainerCloseMode
-     */
-    opc_error_t opcContainerClose(opcContainer *c, opcContainerCloseMode mode);
-    
-    /**
-     Returns the unmodified user context passed to \ref opcContainerOpen.
-     \see opcContainerOpen
-     */
-    void *opcContainerGetUserContext(opcContainer *c);
-    
-    /**
-     List all types, relations and parts of the container \a c to \a out.
-     \par Sample:
-     \include opc_dump.c
-     */
-    opc_error_t opcContainerDump(opcContainer *c, FILE *out);
-    
-    /**
-     Exports the OPC container to "Flat OPC" (http://blogs.msdn.com/b/ericwhite/archive/2008/09/29/the-flat-opc-format.aspx).
-     The flat versions of an OPC file are very important when dealing with e.g XSL(T)-based or Javascript-based transformations.
-     \see opcContainerFlatImport.
-     \todo Implementation needed.
-     */
-    int opcContainerFlatExport(opcContainer *c, const xmlChar *fileName);
-    
-    /**
-     Imports the flat version of an OPC container. 
-     \see opcContainerFlatExport.
-     \todo Implementation needed.
-     */
-    int opcContainerFlatImport(opcContainer *c, const xmlChar *fileName);
-    
-    /**
-     Iterate all types.
-     \code
-     for(xmlChar *type=opcContentTypeFirst(c);
-         NULL!=type;
-         type=opcContentTypeNext(c, type)) {
-        printf("%s\n", type);
-     }
-     \endcode
-    */
-    const xmlChar *opcContentTypeFirst(opcContainer *container);
-    
-    /**
-     \see opcContentTypeNext()
-    */
-    const xmlChar *opcContentTypeNext(opcContainer *container, const xmlChar *type);
-
-    /**
-     Iterate extensions.
-     \code
-     for(const xmlChar *ext=opcExtensionFirst(c);
-         NULL!=ext;
-         ext=opcExtensionNext(ext)) {
-        printf("%s\n", ext);
-     }
-     \endcode
-    */
-    const xmlChar *opcExtensionFirst(opcContainer *container);
-    
-    /**
-     \see opcExtensionFirst()
-     */
-    const xmlChar *opcExtensionNext(opcContainer *container, const xmlChar *ext);
-    
-    /**
-     Get registered type for extension.
-     \see opcExtensionRegister()
-     */
-    const xmlChar *opcExtensionGetType(opcContainer *container, const xmlChar *ext);
-
-    /**
-     Register a mime-type and and extension.
-     \see opcExtensionGetType()
-     */
-    const xmlChar *opcExtensionRegister(opcContainer *container, const xmlChar *ext, const xmlChar *type);
-
-
-    /**
-     Iterator through all relation types of the container:
-     \code
-     for(xmlChar *type=opcRelationTypeFirst(c);
-         NULL!=type;
-         type=opcRelationTypeNext(c, type)) {
-        printf("%s\n", type);
-     }
-     \endcode
-     */
-    const xmlChar *opcRelationTypeFirst(opcContainer *container);
-
-    /**
-     \see opcRelationTypeFirst()
-    */
-    const xmlChar *opcRelationTypeNext(opcContainer *container, const xmlChar *type);
-
-
-    /**
-     Iterator through all relation types of the container:
-     \code
-     for(xmlChar *target=opcExternalTargetFirst(c);
-         NULL!=target;
-         type=opcExternalTargetNext(c, target)) {
-        printf("%s\n", target);
-     }
-     \endcode
-     */
-    const xmlChar *opcExternalTargetFirst(opcContainer *container);
-
-    /**
-     \see opcExternalTargetFirst()
-    */
-    const xmlChar *opcExternalTargetNext(opcContainer *container, const xmlChar *target);
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_CONTAINER_H */

include/opc/file.h

@@ -1,200 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/file.h
- The opc module contains the file library functions.
-*/
-#include <opc/config.h>
-
-#ifndef OPC_FILE_H
-#define OPC_FILE_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-/**
-  Flag for READ access.
-  \hideinitializer
-*/
-#define OPC_FILE_READ  (1<<0)
-
-/**
-  Flag for WRITE access.
-  \hideinitializer
-*/
-#define OPC_FILE_WRITE (1<<1)
-
-/**
-  Flag indicates that file will be truncated when opened.
-  \hideinitializer
-*/
-#define OPC_FILE_TRUNC  (1<<2)
-
-
-    /** 
-      Abstraction for see modes.
-      */
-    typedef enum OPC_FILESEEKMODE_ENUM {
-        opcFileSeekSet = SEEK_SET,
-        opcFileSeekCur = SEEK_CUR,
-        opcFileSeekEnd = SEEK_END
-    } opcFileSeekMode;
-
-     /**
-      Callback to read a file. E.g. for a FILE * context this can be implemented as
-      \code
-      static int opcFileRead(void *iocontext, char *buffer, int len) {
-          return fread(buffer, sizeof(char), len, (FILE*)iocontext);
-      }
-      \endcode
-      */
-    typedef int opcFileReadCallback(void *iocontext, char *buffer, int len);
-
-     /**
-      Callback to write a file. E.g. for a FILE * context this can be implemented as
-      \code
-      static int opcFileWrite(void *iocontext, const char *buffer, int len) {
-          return fwrite(buffer, sizeof(char), len, (FILE*)iocontext);
-      }
-      \endcode
-      */
-    typedef int opcFileWriteCallback(void *iocontext, const char *buffer, int len);
-
-     /**
-      Callback to close a file. E.g. for a FILE * context this can be implemented as
-      \code
-      static int opcFileClose(void *iocontext) {
-          return fclose((FILE*)iocontext);
-      }
-      \endcode
-      */
-    typedef int opcFileCloseCallback(void *iocontext);
-
-     /**
-      Callback to seek a file. E.g. for a FILE * context this can be implemented as
-      \code
-      static opc_ofs_t opcFileSeek(void *iocontext, opc_ofs_t ofs) {
-          int ret=fseek((FILE*)iocontext, ofs, SEEK_SET);
-          if (ret>=0) {
-              return ftell((FILE*)iocontext);
-          } else {
-              return ret;
-          }
-      }
-      \endcode
-      */
-    typedef opc_ofs_t opcFileSeekCallback(void *iocontext, opc_ofs_t ofs);
-
-     /**
-      Callback to trim a file. E.g. for a FILE * context this can be implemented as
-      \code
-      static int opcFileTrim(void *iocontext, opc_ofs_t new_size) {
-      #ifdef WIN32
-          return _chsize(fileno((FILE*)iocontext), new_size);
-      #else
-          return ftruncate(fileno((FILE*)iocontext), new_size);
-      #endif
-      }
-      \endcode
-      */
-    typedef int opcFileTrimCallback(void *iocontext, opc_ofs_t new_size);
-
-     /**
-      Callback to flush a file. E.g. for a FILE * context this can be implemented as
-      \code
-      static int opcFileFlush(void *iocontext) {
-          return fflush((FILE*)iocontext);
-      }
-      \endcode
-      */
-    typedef int opcFileFlushCallback(void *iocontext);
-
-    /**
-      Represents a state of a file, i.e. file position (buf_pos) and error status (err).
-      */
-    typedef struct OPC_FILERAWSTATE_STRUCT {
-        opc_error_t err;
-        opc_ofs_t   buf_pos; // current pos in file
-    } opcFileRawState;
-
-    /**
-     File IO context.
-     */
-    typedef struct OPC_IO_STRUCT {
-        opcFileReadCallback *_ioread;
-        opcFileWriteCallback *_iowrite;
-        opcFileCloseCallback *_ioclose;
-        opcFileSeekCallback *_ioseek;
-        opcFileTrimCallback *_iotrim;
-        opcFileFlushCallback *_ioflush;
-        void *iocontext;
-        int flags;
-        opcFileRawState state;
-        opc_ofs_t file_size;
-    } opcIO_t;
-
-    /**
-      Initialize an IO context.
-      */
-    opc_error_t opcFileInitIO(opcIO_t *io,
-                              opcFileReadCallback *ioread,
-                              opcFileWriteCallback *iowrite,
-                              opcFileCloseCallback *ioclose,
-                              opcFileSeekCallback *ioseek,
-                              opcFileTrimCallback *iotrim,
-                              opcFileFlushCallback *ioflush,
-                              void *iocontext,
-                              pofs_t file_size,
-                              int flags);
-
-    /**
-      Initialize an IO context for a file.
-      */
-    opc_error_t opcFileInitIOFile(opcIO_t *io, const xmlChar *filename, int flags);
-
-    /**
-      Initialize an IO for memory.
-      \warning Currently supports READ-ONLY file access.
-      */
-    opc_error_t opcFileInitIOMemory(opcIO_t *io, const opc_uint8_t *data, opc_uint32_t data_len, int flags);
-
-    /**
-      Cleanup an IO context, i.e. release all system resources.
-      */
-    opc_error_t opcFileCleanupIO(opcIO_t *io);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_FILE_H */

include/opc/helper.h

@@ -1,60 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/helper.h
-  Contains helper functions for the opc module.
-*/
-#include <opc/config.h>
-
-#ifndef OPC_HELPER_H
-#define OPC_HELPER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-
-    /**
-      Constructs a segment name.
-      */
-    opc_uint16_t opcHelperAssembleSegmentName(char *out, opc_uint16_t out_size, const xmlChar *name, opc_uint32_t segment_number, opc_uint32_t next_segment_id, opc_bool_t rels_segment, opc_uint16_t *out_max);
-
-    /**
-      Splits a filename into the segment informations.
-      */
-    opc_error_t opcHelperSplitFilename(opc_uint8_t *filename, opc_uint32_t filename_length, opc_uint32_t *segment_number, opc_bool_t *last_segment, opc_bool_t *rel_segment);
-
-#endif /* OPC_HELPER_H */

include/opc/inputstream.h

@@ -1,74 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/inputstream.h
- 
- */
-#include <opc/config.h>
-
-#ifndef OPC_INPUTSTREAM_H
-#define OPC_INPUTSTREAM_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-    /**
-      Internal type which represents a binary input stream.
-      */
-    typedef struct OPC_CONTAINER_INPUTSTREAM_STRUCT opcContainerInputStream;
-
-    /**
-      Opens the part \c name of the \c container for reading.
-      */
-    opcContainerInputStream* opcContainerOpenInputStream(opcContainer *container, const xmlChar *name);
-
-    /**
-     Reads maximal \c buffer_len bytes from the input \c stream to \c buffer. 
-     \return The number of byes read or "0" in case of an error or end-of-stream.
-     */
-    opc_uint32_t opcContainerReadInputStream(opcContainerInputStream* stream, opc_uint8_t *buffer, opc_uint32_t buffer_len);
-
-    /**
-      Closes the input stream and releases all system resources.
-      */
-    opc_error_t opcContainerCloseInputStream(opcContainerInputStream* stream);
-
-    /**
-      Returns the type of compression used for the stream.
-      */
-    opcCompressionOption_t opcContainerGetInputStreamCompressionOption(opcContainerInputStream* stream);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_INPUTSTREAM_H */

include/opc/opc.h

@@ -1,73 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/opc.h
- The opc module contains the basic library functions.
-*/
-#include <opc/config.h>
-#include <opc/container.h>
-#include <opc/part.h>
-#include <opc/relation.h>
-#include <opc/inputstream.h>
-#include <opc/outputstream.h>
-#include <opc/zip.h>
-#include <opc/xmlreader.h>
-#include <opc/xmlwriter.h>
-#include <opc/properties.h>
-
-#ifndef OPC_OPC_H
-#define OPC_OPC_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-    
-    /**
-     * Initialize libopc.
-     * Sample:
-     * \include opc_helloworld.c
-     * @return Non-zero if successful.
-     */
-    opc_error_t opcInitLibrary();
-
-    /**
-     * Free libopc. Clean up all resources.
-     * @return Non-zero if successful.
-     * \see opcInitLibrary.
-     */
-    opc_error_t opcFreeLibrary();
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_OPC_H */

include/opc/outputstream.h

@@ -1,71 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/outputstream.h
- 
- */
-#include <opc/config.h>
-
-#ifndef OPC_OUTPUTSTREAM_H
-#define OPC_OUTPUTSTREAM_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-    /**
-      Internal type which represents a binary output stream.
-      */
-    typedef struct OPC_CONTAINER_OUTPUTSTREAM_STRUCT opcContainerOutputStream;
-
-    /** 
-      Open the part \c name or writing in \c container with compression \c compression_option.
-      \note Make sure the part exists! 
-      \see opcPartCreate.
-      */
-    opcContainerOutputStream* opcContainerCreateOutputStream(opcContainer *container, const xmlChar *name, opcCompressionOption_t compression_option);
-
-    /**
-      Write \c buffer_len bytes from \c buffer to \c stream. 
-      \return Returns the number of bytes written.
-      */
-    opc_uint32_t opcContainerWriteOutputStream(opcContainerOutputStream* stream, const opc_uint8_t *buffer, opc_uint32_t buffer_len);
-
-    /**
-      Close the \c stream and free all associated resources.
-      */
-    opc_error_t opcContainerCloseOutputStream(opcContainerOutputStream* stream);
-        
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_OUTPUTSTREAM_H */

include/opc/part.h

@@ -1,118 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/part.h
- 
- */
-#include <opc/config.h>
-
-#ifndef OPC_PART_H
-#define OPC_PART_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-    /**
-     Handle to an OPC part created by \ref opcPartOpen.
-     \see opcPartOpen.
-     */
-    typedef xmlChar* opcPart;
-
-/**
-  Represents an invalid (resp. NULL) part.
-  In releations OPC_PART_INVALID also represents the root part.
-  \hideinitializer
-  */
-#define OPC_PART_INVALID NULL
-
-    /**
-     Find a part in a \ container by \c absolutePath and/or \c type.
-     Currently no flags are supported.
-     */
-    opcPart opcPartFind(opcContainer *container, 
-                        const xmlChar *absolutePath, 
-                        const xmlChar *type,
-                        int flags);
-
-    /**
-     Creates a part in a \ container with \c absolutePath and \c type.
-     Currently no flags are supported.
-     */
-    opcPart opcPartCreate(opcContainer *container, 
-                          const xmlChar *absolutePath, 
-                          const xmlChar *type,
-                          int flags);
-
-    /**
-      Returns the type of the container.
-      The string is interned and must not be freed.
-      */
-    const xmlChar *opcPartGetType(opcContainer *c, opcPart part);
-
-    /**
-      Returns the type of the container. 
-      If \c override_only then the return value will be NULL for parts not having an override type.
-      The string is interned and must not be freed.
-      */
-    const xmlChar *opcPartGetTypeEx(opcContainer *c, opcPart part, opc_bool_t override_only);
-
-    /**
-     Deleted that part \c absolutePath in the \c container.
-     */
-    opc_error_t opcPartDelete(opcContainer *container, const xmlChar *absolutePath);
-
-    /**
-      Get the first part.
-      \code
-      for(opcPart part=opcPartGetFirst(c);OPC_PART_INVALID!=part;part=opcPartGetNext(c, part)) {
-        printf("%s; \n", part, opcPartGetType(c, part));
-      }
-      \endcode 
-      */
-    opcPart opcPartGetFirst(opcContainer *container);
-
-    /**
-     Get the next part.
-     \see opcPartGetFirst
-      */
-    opcPart opcPartGetNext(opcContainer *container, opcPart part);
-
-    /**
-      Returns the size in bytes of the \c part.
-      */
-    opc_ofs_t opcPartGetSize(opcContainer *c, opcPart part);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_PART_H */

include/opc/properties.h

@@ -1,121 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/properties.h
- 
- */
-#include <opc/config.h>
-#include <opc/container.h>
-
-#ifndef OPC_PROPERTIES_H
-#define OPC_PROPERTIES_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-    /** 
-      Represents a simple Dublin Core type.
-      */
-    typedef struct OPC_DC_SIMPLE_TYPE {
-        xmlChar *str;
-        xmlChar *lang;
-    } opcDCSimpleType_t;
-
-    /** 
-      Represents the core properties of an OPC container.
-      */
-    typedef struct OPC_PROPERTIES_STRUCT {
-        xmlChar *category;                /* xsd:string     */
-        xmlChar *contentStatus;           /* xsd:string     */
-        xmlChar *created;                 /* dc:date        */
-        opcDCSimpleType_t creator;        /* dc:any         */
-        opcDCSimpleType_t description;    /* dc:any         */
-        opcDCSimpleType_t identifier;     /* dc:any         */
-        opcDCSimpleType_t *keyword_array; /* cp:CT_Keywords */
-        opc_uint32_t keyword_items;
-        opcDCSimpleType_t language;       /* dc:any         */
-        xmlChar *lastModifiedBy;          /* xsd:string     */
-        xmlChar *lastPrinted;             /* xsd:dateTime   */
-        xmlChar *modified;                /* dc:date        */
-        xmlChar *revision;                /* xsd:string     */
-        opcDCSimpleType_t subject;        /* dc:any         */
-        opcDCSimpleType_t title;          /* dc:any         */
-        xmlChar *version;                 /* xsd:string     */
-    } opcProperties_t;
-
-    /**
-      Initialize the core properties \c cp.
-      \see opcCorePropertiesSetString
-      */
-    opc_error_t opcCorePropertiesInit(opcProperties_t *cp);
-
-    /**
-      Cleanup the core properties \c cp, i.e. release all resources.
-      \see opcCorePropertiesSetString
-      */
-    opc_error_t opcCorePropertiesCleanup(opcProperties_t *cp);
-
-    /**
-      Rease the core properties \c cp from the container \c.
-      */
-    opc_error_t opcCorePropertiesRead(opcProperties_t *cp, opcContainer *c);
-
-
-    /**
-      Write/Update the core properties \c cp in the container \c.
-      */
-    opc_error_t opcCorePropertiesWrite(opcProperties_t *cp, opcContainer *c);
-
-    /**
-      Update a string in the core properties the right way.
-      \code
-      opcProperties_t cp;
-      opcCorePropertiesInit(&cp);
-      opcCorePropertiesSetString(&cp.revision, "1");
-      opcCorePropertiesSetStringLang(&cp.creator, "Florian Reuter", NULL);
-      opcCorePropertiesCleanup(&cp);
-      \endcode
-      */
-    opc_error_t opcCorePropertiesSetString(xmlChar **prop, const xmlChar *str);
-
-    /** 
-      Update a core properties the right way.
-      \see opcCorePropertiesSetString
-      */
-    opc_error_t opcCorePropertiesSetStringLang(opcDCSimpleType_t *prop, const xmlChar *str, const xmlChar *lang);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_PROPERTIES_H */

include/opc/relation.h

@@ -1,140 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/relation.h
- 
- */
-#include <opc/config.h>
-
-#ifndef OPC_RELATION_H
-#define OPC_RELATION_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-    /**
-     Indentifier for an OPC relation.
-     */
-    typedef opc_uint32_t opcRelation;
-
-/**
-  Constant which represents an invalid relation.
-*/
-#define OPC_RELATION_INVALID (-1)
-
-    /**
-      Find a relation originating from \c part in \c container with \c relationId and/or \c mimeType.
-      If \c part is OPC_PART_INVALID then part represents the root part.
-      @param[in] relationId The relationId (e.g. "rId1") or NULL.
-      @param[in] mimeType The mimeType or NULL.
-      */
-    opcRelation opcRelationFind(opcContainer *container, opcPart part, const xmlChar *relationId, const xmlChar *mimeType);
-
-    /**
-      Deleted the relation from the container.
-      \see opcRelationFind.
-      */
-    opc_error_t opcRelationDelete(opcContainer *container, opcPart part, const xmlChar *relationId, const xmlChar *mimeType);
-
-    /**
-      Returns the first relation.
-      The following code will dump all relations:
-      \code
-        for(opcPart part=opcPartGetFirst(c);OPC_PART_INVALID!=part;part=opcPartGetNext(c, part)) {
-           for(opcRelation rel=opcRelationFirst(part, c);
-               OPC_PART_INVALID!=rel;
-               rel=opcRelationNext(c, rel)) {
-               opcPart internal_target=opcRelationGetInternalTarget(c, part, rel);
-               const xmlChar *external_target=opcRelationGetExternalTarget(c, part, rel);
-               const xmlChar *target=(NULL!=internal_target?internal_target:external_target);
-               const xmlChar *prefix=NULL;
-               opc_uint32_t counter=-1;
-               const xmlChar *type=NULL;
-               opcRelationGetInformation(c, part, rel, &prefix, &counter, &type);        
-               if (-1==counter) { // no counter after prefix
-                  printf("%s;%s;%s;%s\n", part, prefix, target, type);
-               } else {
-                  printf("%s;%s%i;%s;%s\n", part, prefix, counter, target, type);
-               }
-           }
-        }
-      \endcode
-      */
-    opcRelation opcRelationFirst(opcContainer *container, opcPart part);
-
-    /**
-      \see opcRelationFirst
-      */
-    opcRelation opcRelationNext(opcContainer *container, opcPart part, opcRelation relation);
-    
-    /**
-      Returns the internal target.
-      \note To test for an external target use opcRelationGetExternalTarget.
-      \see opcRelationGetExternalTarget
-      */
-    opcPart opcRelationGetInternalTarget(opcContainer *container, opcPart part, opcRelation relation);
-
-    /**
-      Returns the external target or NULL if it is an internal target.
-      The string is interned. Must not be freed.
-      \see opcRelationGetExternalTarget
-      */
-    const xmlChar *opcRelationGetExternalTarget(opcContainer *container, opcPart part, opcRelation relation);
-
-    /**
-      Returns the relations type.
-      The string is interned. Must not be freed.
-      */
-    const xmlChar *opcRelationGetType(opcContainer *container, opcPart part, opcRelation relation);
-
-    /** 
-      Get information about a relation.
-      \see opcRelationFirst
-      */
-    void opcRelationGetInformation(opcContainer *container, opcPart part, opcRelation relation, const xmlChar **prefix, opc_uint32_t *counter, const xmlChar **type);
-
-    /** 
-      Add a relation to \c container from \c src part to \c dest part with id \c rid and type \c type.
-      */
-    opc_uint32_t opcRelationAdd(opcContainer *container, opcPart src, const xmlChar *rid, opcPart dest, const xmlChar *type);
-
-    /** 
-      Add an external relation to \c container from \c src part to \c target URL with id \c rid and type \c type.
-      */
-    opc_uint32_t opcRelationAddExternal(opcContainer *container, opcPart src, const xmlChar *rid, const xmlChar *target, const xmlChar *type);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-
-#endif /* OPC_RELATION_H */

include/opc/xmlreader.h

@@ -1,69 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/xmlreader.h
- 
- */
-
-#ifndef OPC_XMLREADER_H
-#define OPC_XMLREADER_H
-
-#include <opc/config.h>
-#include <libxml/xmlreader.h>
-#include <mce/textreader.h>
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-    /** 
-      Open an MCE reader for \c partName. Parameters \c URL, \c encoding and \c options will be passed unmodified to 
-      http://xmlsoft.org/html/libxml-xmlreader.html#xmlReaderForIO and they can we NULL, NULL, 0.
-      \note Make sure the part exists.
-      \see opcPartFind
-      */
-    opc_error_t opcXmlReaderOpen(opcContainer *container, mceTextReader_t *mceTextReader, const xmlChar *partName, const char * URL, const char * encoding, int options);
-
-    /**
-      Returns an libxml DOM document. Parameters \c URL, \c encoding and \c options will be passed unmodified to 
-      http://xmlsoft.org/html/libxml-parser.html#xmlReadIO and they can we NULL, NULL, 0.
-      \note Make sure the part exists.
-      \see opcPartFind
-      */
-    xmlDocPtr opcXmlReaderReadDoc(opcContainer *container, const xmlChar *partName, const char * URL, const char * encoding, int options);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_XMLREADER_H */

include/opc/xmlwriter.h

@@ -1,57 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/xmlwriter.h
- 
- */
-#include <opc/config.h>
-#include <mce/textwriter.h>
-
-#ifndef OPC_XMLWRITER_H
-#define OPC_XMLWRITER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-    /**
-      Create an MCE text writer for \c part in \c container with compression \c compression_option.
-      \note Make sure the part exists.
-      \see opcPartFind
-      */
-    mceTextWriter *mceTextWriterOpen(opcContainer *c, opcPart part, opcCompressionOption_t compression_option);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_XMLWRITER_H */

include/opc/zip.h

@@ -1,255 +0,0 @@
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-/** @file opc/zip.h
-  The ZIP file backend of an OPC container.
- */
-#include <opc/config.h>
-#include <opc/file.h>
-#include <opc/container.h>
-
-#ifndef OPC_ZIP_H
-#define OPC_ZIP_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif    
-
-    /**
-     Default growth hint of an OPC stream.
-     */
-    #define OPC_DEFAULT_GROWTH_HINT 512
-
-    /**
-     Handle to a ZIP archive.
-     \see internal.h
-     */
-    typedef struct OPC_ZIP_STRUCT opcZip;
-
-    /**
-      Handle to a raw ZIP input stream.
-     \see internal.h
-      */
-    typedef struct OPC_ZIPINPUTSTREAM_STRUCT opcZipInputStream;
-
-    /**
-      Handle to a raw ZIP output stream.
-     \see internal.h
-      */
-    typedef struct OPC_ZIPOUTPUTSTREAM_STRUCT opcZipOutputStream;
-
-    /**
-     Holds all information of a ZIP segment.
-     */
-    typedef struct OPC_ZIP_SEGMENT_INFO_STRUCT {
-        xmlChar name[OPC_MAX_PATH]; 
-        opc_uint32_t name_len;
-        opc_uint32_t segment_number;
-        opc_bool_t   last_segment;
-        opc_bool_t   rels_segment;
-        opc_uint32_t header_size;
-        opc_uint32_t min_header_size;
-        opc_uint32_t trailing_bytes;
-        opc_uint32_t compressed_size;
-        opc_uint32_t uncompressed_size;
-        opc_uint16_t bit_flag;
-        opc_uint32_t data_crc;
-        opc_uint16_t compression_method;
-        opc_ofs_t    stream_ofs;
-        opc_uint16_t growth_hint;
-    } opcZipSegmentInfo_t;
-
-    /**
-      \see opcZipLoader
-      */
-    typedef int opcZipLoaderOpenCallback(void *iocontext);
-    /**
-      \see opcZipLoader
-      */
-    typedef int opcZipLoaderSkipCallback(void *iocontext);
-    /**
-      \see opcZipLoader
-      */
-    typedef int opcZipLoaderReadCallback(void *iocontext, char *buffer, int len);
-    /**
-      \see opcZipLoader
-      */
-    typedef int opcZipLoaderCloseCallback(void *iocontext);
-
-    /**
-      \see opcZipLoader
-      */
-    typedef opc_error_t (opcZipLoaderSegmentCallback_t)(void *iocontext, void *userctx, opcZipSegmentInfo_t *info, opcZipLoaderOpenCallback *open, opcZipLoaderReadCallback *read, opcZipLoaderCloseCallback *close, opcZipLoaderSkipCallback *skip);
-
-    /**
-      Walks every segment in a ZIP archive and calls the \c segmentCallback callback method.
-      The implementer \c segmentCallback method must then eiher use the passed \c open, \c read and \c close methods
-      to read the stream or the passed \c skip methods to skip the stream.
-      This method can be used to e.g. read ZIP file in stream mode.
-      */
-    opc_error_t opcZipLoader(opcIO_t *io, void *userctx, opcZipLoaderSegmentCallback_t *segmentCallback);
-
-    /**
-      \see opcZipClose
-     */
-    typedef opc_error_t (opcZipSegmentReleaseCallback)(opcZip *zip, opc_uint32_t segment_id);
-
-    /** 
-     Closes the ZIP archive \c zip and will call \c releaseCallback for every segment to give the implementer a chance
-     to free user resources.
-     */
-    void opcZipClose(opcZip *zip, opcZipSegmentReleaseCallback* releaseCallback);
-
-    /**
-      Creates an empty ZIP archive with the given \c io.
-      */
-    opcZip *opcZipCreate(opcIO_t *io);
-
-    /**
-      Commits all buffers and writes the ZIP archives local header directories.
-      if \c trim is true then padding bytes will be removed, i.e. the ZIP file size fill be minimalized.
-     */
-    opc_error_t opcZipCommit(opcZip *zip, opc_bool_t trim);
-
-    /**
-      Garbage collection on the passed \c zip archive. This will e.g. make deleted files available as free space.
-      */
-    opc_error_t opcZipGC(opcZip *zip);
-
-    /**
-      Load segment information into \c info.
-      If \c rels_segment is -1 then load the info for part with name \c partName.
-      Otherwise load the segment information for the ".rels." segment of \c partName.
-      \return Returns the segment_id.
-      */
-    opc_uint32_t opcZipLoadSegment(opcZip *zip, const xmlChar *partName, opc_bool_t rels_segment, opcZipSegmentInfo_t *info);
-
-    /**
-      Create a segment with the given parameters.
-      \return Returns the segment_id.
-      */
-    opc_uint32_t opcZipCreateSegment(opcZip *zip, 
-                                     const xmlChar *partName, 
-                                     opc_bool_t relsSegment, 
-                                     opc_uint32_t segment_size, 
-                                     opc_uint32_t growth_hint,
-                                     opc_uint16_t compression_method,
-                                     opc_uint16_t bit_flag);
-
-    /**
-      Creates an input stream for the segment with \c segment_id.
-      \see opcZipLoadSegment
-      \see opcZipCreateSegment
-      */
-    opcZipInputStream *opcZipOpenInputStream(opcZip *zip, opc_uint32_t segment_id);
-
-    /**
-     Free all resources of the input stream.
-     */
-    opc_error_t opcZipCloseInputStream(opcZip *zip, opcZipInputStream *stream);
-
-    /**
-     Read maximal \c buf_len bytes from the input stream into \buf. 
-     \return Returns the number of bytes read.
-     */
-    opc_uint32_t opcZipReadInputStream(opcZip *zip, opcZipInputStream *stream, opc_uint8_t *buf, opc_uint32_t buf_len);
-
-
-    /**
-      Creates an output stream for the segment with \c segment_id.
-      If \c *segment_id is -1 then a new segment will be created. 
-      Otherwise the segment with \c *segment_id will be overwritten.
-     */
-    opcZipOutputStream *opcZipCreateOutputStream(opcZip *zip, 
-                                             opc_uint32_t *segment_id, 
-                                             const xmlChar *partName, 
-                                             opc_bool_t relsSegment, 
-                                             opc_uint32_t segment_size, 
-                                             opc_uint32_t growth_hint,
-                                             opc_uint16_t compression_method,
-                                             opc_uint16_t bit_flag);
-
-    /**
-      Opens an existing ouput stream for reading.
-      The \c *segment_id will be set to -1 and reset on opcZipCloseOutputStream.
-      \see opcZipCloseOutputStream
-     */
-    opcZipOutputStream *opcZipOpenOutputStream(opcZip *zip, opc_uint32_t *segment_id);
-
-    /** 
-      Will close the stream and free all resources. Additionally the new segment id will be stored in \c *segment_id.
-      \see opcZipOpenOutputStream
-      */
-    opc_error_t opcZipCloseOutputStream(opcZip *zip, opcZipOutputStream *stream, opc_uint32_t *segment_id);
-
-    /**
-     Write \c buf_len bytes to \c buf. 
-     \return Returns the number of bytes written.
-     */
-    opc_uint32_t opcZipWriteOutputStream(opcZip *zip, opcZipOutputStream *stream, const opc_uint8_t *buf, opc_uint32_t buf_len);
-
-    /**
-     Returns the first segment id or -1.
-     Use the following code to iterarte through all segments.
-     \code 
-     for(opc_uint32_t segment_id=opcZipGetFirstSegmentId(zip);
-         -1!=segment_id;
-         segment_id=opcZipGetNextSegmentId(zip, segment_id) {
-        ...
-     }
-     \endcode
-     \see opcZipGetNextSegmentId
-     */
-    opc_uint32_t opcZipGetFirstSegmentId(opcZip *zip);
-
-    /**
-     Returns the next segment id or -1.
-     \see opcZipGetFirstSegmentId
-     */
-    opc_uint32_t opcZipGetNextSegmentId(opcZip *zip, opc_uint32_t segment_id);
-
-    /**
-     Returns info about the given segment id.
-     */
-    opc_error_t opcZipGetSegmentInfo(opcZip *zip, opc_uint32_t segment_id, const xmlChar **name, opc_bool_t *rels_segment, opc_uint32_t *crc);
-
-    /**
-     Marks a given segments as deleted.
-     \see opcZipGC
-     */
-    opc_bool_t opcZipSegmentDelete(opcZip *zip, opc_uint32_t *first_segment, opc_uint32_t *last_segment, opcZipSegmentReleaseCallback* releaseCallback);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif    
-        
-#endif /* OPC_ZIP_H */

include/plib/plib.h

@@ -1,168 +0,0 @@
-/* include/plib/plib.h.  Generated from plib.h by configure.  */
-/*
- Copyright (c) 2010, Florian Reuter
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without 
- modification, are permitted provided that the following conditions 
- are met:
- 
- * Redistributions of source code must retain the above copyright 
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright 
-   notice, this list of conditions and the following disclaimer in 
-   the documentation and/or other materials provided with the 
-   distribution.
- * Neither the name of Florian Reuter nor the names of its contributors 
-   may be used to endorse or promote products derived from this 
-   software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
- STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
- OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-*/
-#ifndef _PLIB_PLIB_H_
-#define _PLIB_PLIB_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define HAVE_STDINT_H 1
-#define HAVE_STDDEF_H 1
-#define HAVE_STDIO_H 1
-#define HAVE_STRING_H 1
-#define HAVE_LIMITS_H 1
-#define HAVE_STDLIB_H 1
-/* #undef HAVE_IO_H */
-#define HAVE_UNISTD_H 1
-#define HAVE_SYS_TYPES_H 1
-#define IS_CONFIGURED 1
-
-#if !defined(IS_CONFIGURED)
-#if defined(WIN32)
-#define HAVE_STRING_H 1
-#define HAVE_STDINT_H 1
-#define HAVE_LIMITS_H 1
-#define HAVE_STDDEF_H 1
-#define HAVE_STDIO_H 1
-#define HAVE_STDLIB_H 1
-#define HAVE_IO_H
-#define snprintf _snprintf
-#else
-#error "configure not executed and we are not on a win32 machine? please run configure or define WIN32 is you are on a WIN32 platform."
-#endif
-#endif
-
-#ifdef HAVE_STDDEF_H
-#include <stddef.h>
-typedef size_t pofs_t; // maximum file offset for eg. read write ops
-#else
-#error "system types can not be determined"
-#endif
-
-#ifdef HAVE_STDIO_H
-#include <stdio.h>
-#else
-#error "system io can not be determined"
-#endif
-
-#ifdef HAVE_STDINT_H
-#include <stdint.h>
-
-typedef int8_t pint8_t;
-typedef uint8_t puint8_t;
-
-typedef int16_t pint16_t;
-typedef uint16_t puint16_t;
-
-typedef int32_t pint32_t;
-typedef uint32_t puint32_t;
-
-typedef int64_t pint64_t;
-typedef uint64_t puint64_t;
-
-typedef int pbool_t;
-
-typedef size_t psize_t;
-
-// INTN_MAX, INTN_MIN, UINTN_MAX
-#else
-#error "system types can not be determined"
-#endif
-
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-
-#ifdef HAVE_LIMITS_H
-#include <limits.h>
-#define PUINT8_MAX UCHAR_MAX 
-#define PINT32_MAX INT_MAX 
-#define PINT32_MIN INT_MIN 
-#define PUINT32_MAX UINT_MAX 
-#define PUINT32_MIN 0 
-#define PUINT16_MAX	USHRT_MAX 
-#define PUINT16_MIN 0 
-#else
-#error "limits can not be determined"
-#endif
-
-#ifdef HAVE_STDLIB_H
-#include <stdlib.h>
-#endif
-
-#ifdef HAVE_IO_H
-#include <io.h>
-#endif
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-
-/**
- Converts an ASCII string to a xmlChar string. This only works for ASCII strings.
- */
-#ifndef _X
-#define _X(s) BAD_CAST(s) 
-#endif
-
-
-/**
- Converts an xmlChar string to an ASCII string. This only works for ASCII charsets.
- */
-#ifndef _X2C
-#define _X2C(s) ((char*)(s))
-#endif
-
-
-#define PASSERT(e) assert(e)
-#ifdef NDEBUG
-#define PENSURE(e) (void)(e)
-#else
-#define PENSURE(e) assert(e)
-#endif
-#define PTRUE (0==0)
-#define PFALSE (0==1)
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-
-#endif /* _PLIB_PLIB_H_ */

lib/bzip2-1.0.6

@@ -1 +0,0 @@
-Subproject commit 288acf97a15d558f96c24c89f578b724d6e06b0c

lib/ffmpeg

@@ -1 +0,0 @@
-Subproject commit 8887991a3109f94b7d019a11a86e6cd900105258

lib/harfbuzz

@@ -1 +0,0 @@
-Subproject commit b28c282585afd3bff844e84eae7f29e1a1267aef

lib/leptonica

@@ -1 +0,0 @@
-Subproject commit cc03be70fded1ad6a8cedad656456386a1bd08e8

lib/libmagic

@@ -1 +0,0 @@
-Subproject commit 1249b5cd02c3b6fb9b917d16c76bc76c862932b6

lib/libpng

@@ -1 +0,0 @@
-Subproject commit 301f7a14295a3bdfaf406dbb5004d0784dc137ea

lib/libtiff

@@ -1 +0,0 @@
-Subproject commit 3db0ff91bc6db20fc4cb035be366a9bbb4e701cf

lib/mupdf

@@ -1 +0,0 @@
-Subproject commit c50ac19e412fe7830dfef5d1cbc71e9da3405c96

lib/onion

@@ -1 +0,0 @@
-Subproject commit 73329b61eb82d65b827e92ab46b2db7da85163c7

lib/openjpeg

@@ -1 +0,0 @@
-Subproject commit ac3737372a00b8778b528094dd5bd58a74f67d42

lib/tesseract

@@ -1 +0,0 @@
-Subproject commit f268e6615e619a7140de78d9db3f8240fa8c68bd

lmdb

@@ -1 +0,0 @@
-Subproject commit 5c012bbe033f9bbb273078b07dded59f080d348d

schema/mappings.json

@@ -4,10 +4,28 @@
       "type": "keyword",
       "doc_values": true
     },
+    "checksum": {
+      "type": "keyword",
+      "index": false
+    },
+    "_depth": {
+      "type": "integer"
+    },
     "path": {
       "type": "text",
       "analyzer": "path_analyzer",
-      "copy_to": "suggest-path"
+      "copy_to": "suggest-path",
+      "fielddata": true,
+      "fields": {
+        "nGram": {
+          "type": "text",
+          "analyzer": "my_nGram"
+        },
+        "text": {
+          "type": "text",
+          "analyzer": "content_analyzer"
+        }
+      }
     },
     "suggest-path": {
       "type": "completion",
@@ -16,6 +34,14 @@
     "mime": {
       "type": "keyword"
     },
+    "parent": {
+      "type": "keyword",
+      "index": false
+    },
+    "thumbnail": {
+      "type": "integer",
+      "index": false
+    },
     "videoc": {
       "type": "keyword",
       "index": false
@@ -25,7 +51,7 @@
       "index": false
     },
     "duration": {
-      "type": "float",
+      "type": "integer",
       "index": false
     },
     "width": {
@@ -36,8 +62,13 @@
       "type": "integer",
       "index": false
     },
+    "pages": {
+      "type": "integer",
+      "index": false
+    },
     "mtime": {
-      "type": "integer"
+      "type": "date",
+      "format": "epoch_second"
     },
     "size": {
       "type": "long"
@@ -48,6 +79,7 @@
     "name": {
       "analyzer": "content_analyzer",
       "type": "text",
+      "fielddata": true,
       "fields": {
         "nGram": {
           "type": "text",
@@ -79,10 +111,10 @@
       "analyzer": "my_nGram",
       "type": "text"
     },
-    "_keyword.*":  {
+    "_keyword.*": {
       "type": "keyword"
     },
-    "_text.*":  {
+    "_text.*": {
       "analyzer": "content_analyzer",
       "type": "text",
       "fields": {
@@ -108,7 +140,14 @@
       }
     },
     "tag": {
-      "type": "keyword"
+      "type": "text",
+      "fielddata": true,
+      "analyzer": "tag_analyzer",
+      "copy_to": "suggest-tag"
+    },
+    "suggest-tag": {
+      "type": "completion",
+      "analyzer": "case_insensitive_kw_analyzer"
     },
     "exif_make": {
       "type": "text"
@@ -133,6 +172,76 @@
     },
     "exif_user_comment": {
       "type": "text"
+    },
+    "exif_gps_longitude_ref": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_longitude_dms": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_longitude_dec": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_ref": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_dms": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_dec": {
+      "type": "keyword",
+      "index": false
+    },
+    "author": {
+      "type": "text"
+    },
+    "modified_by": {
+      "type": "text"
+    },
+    "emb.384.*": {
+      "type": "dense_vector",
+      "dims": 384
+    },
+    "emb.idx_384.*": {
+      "type": "dense_vector",
+      "dims": 384,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.idx_512.clip": {
+      "type": "dense_vector",
+      "dims": 512,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.512.*": {
+      "type": "dense_vector",
+      "dims": 512
+    },
+    "emb.idx_768.*": {
+      "type": "dense_vector",
+      "dims": 768,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.768.*": {
+      "type": "dense_vector",
+      "dims": 768
+    },
+    "emb.idx_1024.*": {
+      "type": "dense_vector",
+      "dims": 1024,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.1024.*": {
+      "type": "dense_vector",
+      "dims": 1024
     }
   }
 }

schema/pipeline.json

@@ -1,9 +1,9 @@
 {
-  "description": "Copy _id to _tie",
+  "description": "Copy _id to _tie, save path depth",
   "processors": [
     {
       "script": {
-        "source": "ctx._tie = ctx._id;"
+        "source": "ctx._tie = ctx._id; ctx._depth = ctx.path.length() == 0 ? 0 : 1 + ctx.path.length() - ctx.path.replace(\"/\", \"\").length();"
       }
     }
   ]

schema/settings.json

@@ -1,15 +1,22 @@
 {
   "index": {
     "refresh_interval": "30s",
-    "codec": "best_compression"
+    "codec": "best_compression",
+    "number_of_replicas": 0,
+    "highlight.max_analyzed_offset": 1000000
   },
   "analysis": {
     "tokenizer": {
       "path_tokenizer": {
-        "type": "path_hierarchy"
+        "type": "path_hierarchy",
+        "delimiter": "/"
+      },
+      "tag_tokenizer": {
+        "type": "path_hierarchy",
+        "delimiter": "."
       },
       "my_nGram_tokenizer": {
-        "type": "nGram",
+        "type": "ngram",
         "min_gram": 3,
         "max_gram": 3
       }
@@ -21,6 +28,12 @@
           "lowercase"
         ]
       },
+      "tag_analyzer": {
+        "tokenizer": "tag_tokenizer",
+        "filter": [
+          "lowercase"
+        ]
+      },
       "case_insensitive_kw_analyzer": {
         "tokenizer": "keyword",
         "filter": [

schema/settings_legacy.json

@@ -0,0 +1,58 @@
+{
+  "index": {
+    "refresh_interval": "30s",
+    "codec": "best_compression",
+    "number_of_replicas": 0
+  },
+  "analysis": {
+    "tokenizer": {
+      "path_tokenizer": {
+        "type": "path_hierarchy",
+        "delimiter": "/"
+      },
+      "tag_tokenizer": {
+        "type": "path_hierarchy",
+        "delimiter": "."
+      },
+      "my_nGram_tokenizer": {
+        "type": "nGram",
+        "min_gram": 3,
+        "max_gram": 3
+      }
+    },
+    "analyzer": {
+      "path_analyzer": {
+        "tokenizer": "path_tokenizer",
+        "filter": [
+          "lowercase"
+        ]
+      },
+      "tag_analyzer": {
+        "tokenizer": "tag_tokenizer",
+        "filter": [
+          "lowercase"
+        ]
+      },
+      "case_insensitive_kw_analyzer": {
+        "tokenizer": "keyword",
+        "filter": [
+          "lowercase"
+        ]
+      },
+      "my_nGram": {
+        "tokenizer": "my_nGram_tokenizer",
+        "filter": [
+          "lowercase",
+          "asciifolding"
+        ]
+      },
+      "content_analyzer": {
+        "tokenizer": "standard",
+        "filter": [
+          "lowercase",
+          "asciifolding"
+        ]
+      }
+    }
+  }
+}

scripts/before_build.sh

@@ -1,17 +1,13 @@
 #!/usr/bin/env bash
 
-rm -rf index.sist2/
+(
+  cd ..
+  rm -rf index.sist2
 
-rm web/js/bundle.js 2> /dev/null
+  python3 scripts/mime.py > src/parsing/mime_generated.c
-cat `ls web/js/*.min.js` > web/js/bundle.js
+  python3 scripts/serve_static.py > src/web/static_generated.c
-cat web/js/{util,dom,search}.js >> web/js/bundle.js
+  python3 scripts/index_static.py > src/index/static_generated.c
+  python3 scripts/magic_static.py > src/magic_generated.c
 
-rm web/css/bundle*.css 2> /dev/null
+  printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h
-cat web/css/*.min.css > web/css/bundle.css
+)
-cat web/css/light.css >> web/css/bundle.css
-cat web/css/*.min.css > web/css/bundle_dark.css
-cat web/css/dark.css >> web/css/bundle_dark.css
-
-python3 scripts/mime.py > src/parsing/mime_generated.c
-python3 scripts/serve_static.py > src/web/static_generated.c
-python3 scripts/index_static.py > src/index/static_generated.c

scripts/build.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+VCPKG_ROOT="/vcpkg"
+
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
+mkdir build
+(
+  cd build
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j $(nproc)
+  strip sist2
+  ./sist2 -v > VERSION
+)
+mv build/sist2 sist2-x64-linux
+
+(
+  cd build
+  rm -rf CMakeFiles CMakeCache.txt
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j  $(nproc)
+)
+mv build/sist2_debug sist2-x64-linux-debug

scripts/build_arm64.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+VCPKG_ROOT="/vcpkg"
+
+git submodule update --init --recursive
+
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
+mkdir build
+(
+  cd build
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j $(nproc)
+  strip sist2
+)
+mv build/sist2 sist2-arm64-linux
+
+rm -rf CMakeFiles CMakeCache.txt
+(
+  cd build
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j $(nproc)
+)
+mv build/sist2_debug sist2-arm64-linux-debug

scripts/get_static_libs.sh

@@ -1,128 +0,0 @@
-#!/usr/bin/env bash
-
-THREADS=$(nproc)
-
-cd lib
-
-cd mupdf
-CFLAGS=-fPIC make USE_SYSTEM_HARFBUZZ=yes USE_SYSTEM_OPENJPEG=yes HAVE_X11=no HAVE_GLUT=no -j $THREADS
-cd ..
-
-mv mupdf/build/release/libmupdf.a .
-mv mupdf/build/release/libmupdf-third.a .
-
-# openjp2
-cd openjpeg
-cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-O3 -march=native -DNDEBUG -fPIC"
-make -j $THREADS
-cd ..
-mv openjpeg/bin/libopenjp2.a .
-
-# harfbuzz
-cd harfbuzz
-./autogen.sh
-CFLAGS=-fPIC ./configure --disable-shared --enable-static
-make -j $THREADS
-cd ..
-mv harfbuzz/src/.libs/libharfbuzz.a .
-
-# ffmpeg
-cd ffmpeg
-./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay \
- --disable-ffprobe --disable-doc\
- --disable-manpages --disable-postproc --disable-avfilter \
- --disable-alsa --disable-lzma --disable-xlib --disable-debug\
- --disable-vdpau --disable-vaapi --disable-sdl2 --disable-network\
- --extra-cflags=-fPIC
-make -j $THREADS
-cd ..
-
-mv ffmpeg/libavcodec/libavcodec.a .
-mv ffmpeg/libavformat/libavformat.a .
-mv ffmpeg/libavutil/libavutil.a .
-mv ffmpeg/libswresample/libswresample.a .
-mv ffmpeg/libswscale/libswscale.a .
-
-# onion
-cd onion
-mkdir build 2> /dev/null
-cd build
-cmake -DONION_USE_SSL=false -DONION_USE_PAM=false -DONION_USE_PNG=false -DONION_USE_JPEG=false \
--DONION_USE_JPEG=false -DONION_USE_XML2=false -DONION_USE_SYSTEMD=false -DONION_USE_SQLITE3=false \
--DONION_USE_REDIS=false -DONION_USE_GC=false -DONION_USE_TESTS=false -DONION_EXAMPLES=false \
--DONION_USE_BINDINGS_CPP=false ..
-make -j $THREADS
-cd ../..
-
-mv onion/build/src/onion/libonion_static.a .
-
-#bzip2
-cd bzip2-1.0.6
-make -j $THREADS
-cd ..
-mv bzip2-1.0.6/libbz2.a .
-
-# magic
-cd libmagic
-./autogen.sh
-./configure --enable-static --disable-shared
-make -j $THREADS
-cd ..
-mv libmagic/src/.libs/libmagic.a .
-
-# tesseract
-cd tesseract
-mkdir build
-cd build
-cmake -DSTATIC=on -DBUILD_TRAINING_TOOLS=off -DBUILD_TESTS=off -DCMAKE_BUILD_TYPE=Release \
-  -DCMAKE_CXX_FLAGS="-fPIC" -DAUTO_OPTIMIZE=off ..
-make -j $THREADS
-cd ../..
-mv tesseract/build/libtesseract.a .
-
-# leptonica
-cd leptonica
-./autogen.sh
-CFLAGS="-fPIC" ./configure --without-zlib --without-jpeg --without-giflib \
-  --without-giflib --without-libwebp --without-libwebpmux --without-libopenjpeg \
-  --enable-static --disable-shared
-make -j $THREADS
-cd ..
-mv leptonica/src/.libs/liblept.a .
-
-# tiff
-cd libtiff
-./autogen.sh
-CFLAGS="-fPIC" CXXFLAGS="-fPIC" CXX_FLAGS="-fPIC" ./configure --enable-static --disable-shared --disable-lzw --disable-jpeg --disable-webp \
-  --disable-lzma --disable-zstd --disable-jbig
-make -j $THREADS
-cd ..
-mv libtiff/libtiff/.libs/libtiff.a .
-
-# png
-cd libpng
-CFLAGS="-fPIC" ./configure --enable-static --disable-shared
-make -j $THREADS
-cd ..
-mv libpng/.libs/libpng16.a .
-
-# openssl...
-git clone --depth 1 -b OpenSSL_1_1_0-stable https://github.com/openssl/openssl
-cd openssl
-./config --prefix=$(pwd)/../ssl
-make depend
-make -j $THREADS
-make install
-cd ..
-mv ./openssl/libcrypto.a ./openssl/libssl.a .
-
-# curl
-wget -nc https://curl.haxx.se/download/curl-7.68.0.tar.gz
-tar -xzf curl-7.68.0.tar.gz
-cd curl-7.68.0
-./configure --disable-ldap --disable-ldaps --without-librtmp --disable-rtsp --disable-crypto-auth \
-  --disable-smtp --without-libidn2 --without-nghttp2 --without-brotli --enable-static --disable-shared \
-  --without-libpsl --with-ssl=$(pwd)/../ssl
-make -j $THREADS
-cd ..
-mv curl-7.68.0/lib/.libs/libcurl.a .

scripts/index_static.py

@@ -3,6 +3,7 @@ import json
 files = [
     "schema/mappings.json",
     "schema/settings.json",
+    "schema/settings_legacy.json",
     "schema/pipeline.json",
 ]
 
@@ -14,4 +15,5 @@ def clean(filepath):
 for file in files:
     with open(file, "r") as f:
         data = json.dumps(json.load(f), separators=(",", ":")).encode()
+    data += b'\0'
     print("char %s[%d] = {%s};" % (clean(file), len(data), ",".join(str(int(b)) for b in data)))

scripts/magic_static.py

@@ -0,0 +1,16 @@
+
+MAGIC_PATHS = [
+    "/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+    "/work/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+    "/usr/lib/file/magic.mgc"
+]
+
+for path in MAGIC_PATHS:
+    try:
+        with open(path, "rb") as f:
+            data = f.read()
+        break
+    except:
+        continue
+
+print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))

scripts/mime.csv

@@ -1,15 +1,20 @@
+application/x-matlab-data,mat
 application/arj, arj
 application/base64, mme
 application/binhex, hqx
 application/book, boo|book
+application/CDFV2-corrupt,
 application/CDFV2, sdv
 application/clariscad, ccad
 application/commonground, dp
+application/csv,
+application/dicom, dcm
 application/drafting, drw
+application/epub+zip, epub
 application/freeloader, frl
 application/futuresplash, spl
 application/groupwise, vew
-application/gzip, gz
+application/gzip, gz|tgz
 application/hta, hta
 application/i-deas, unv
 application/iges, iges|igs
@@ -17,17 +22,19 @@ application/inf, inf
 application/java-archive, jar
 application/java, class
 application/javascript,
-application/x-archive, a
 application/json, json
+application/ndjson, jsonl|ndjson
 application/marc, mrc
 application/mbedlet, mbd
 application/mime, aps
 application/mspowerpoint, ppz
 application/msword, doc|dot|w6w|wiz|word
 application/netmc, mcp
-application/octet-stream, bin|dump|gpg
+application/octet-stream, bin|dump|gpg|pack|idx
 application/oda, oda
+application/ogg, ogv
 application/pdf, pdf
+application/pgp-keys,
 application/pgp-signature, pgp
 application/pkcs7-signature, p7s
 application/pkix-cert, cer|crt
@@ -43,6 +50,10 @@ application/vda, vda
 application/vnd.fdf, fdf
 application/vnd.font-fontforge-sfd, sfd
 application/vnd.hp-hpgl, hgl|hpg|hpgl
+application/vnd.iccprofile, icm
+application/vnd.iccprofile, icm
+application/vnd.lotus-1-2-3,
+application/vnd.ms-cab-compressed, cab
 application/vnd.ms-excel, xlb|xlc|xll|xlm|xls|xlw
 application/vnd.ms-fontobject, eot
 application/vnd.ms-opentype, otf
@@ -54,45 +65,73 @@ application/vnd.ms-project, mpp
 application/vnd.oasis.opendocument.base, odb
 application/vnd.oasis.opendocument.formula, odf
 application/vnd.oasis.opendocument.graphics, odg
+application/vnd.oasis.opendocument.presentation, odp
+application/vnd.oasis.opendocument.spreadsheet, ods
 application/vnd.oasis.opendocument.text, odt
+application/vnd.openxmlformats-officedocument.presentationml.presentation, pptx
+application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, xlsx
+application/vnd.openxmlformats-officedocument.wordprocessingml.document, docx
+application/vnd.symbian.install,
+application/vnd.tcpdump.pcap, pcap
 application/vnd.wap.wmlc, wmlc
 application/vnd.wap.wmlscriptc, wmlsc
 application/vnd.xara, web
 application/vocaltec-media-desc, vmd
 application/vocaltec-media-file, vmf
-application/wordperfect6.0, w60
+application/warc, warc
-application/wordperfect6.1, w61
+application/winhelp, hlp
-application/wordperfect, wp|wp5|wp6|wpd
+application/wordperfect, wp|wp5|wp6|wpd|w60|w61
 application/x-123, wk1
+application/x-7z-compressed, 7z
 application/x-aim, aim
+application/x-apple-diskimage,
+application/x-arc,
+application/x-archive, a
+application/x-atari-7800-rom, a78
 application/x-authorware-bin, aab
 application/x-authorware-map, aam
 application/x-authorware-seg, aas
+application/x-avira-qua,
 application/x-bcpio, bcpio
 application/x-bittorrent, torrent
 application/x-bsh, bsh
 application/x-bytecode.python, pyc
 application/x-bzip2, boz|bz2
 application/x-bzip, bz
+application/x-cbr, cbr
+application/x-cbz, cbz
 application/x-cdlink, vcd
 application/x-chat, cha|chat
+application/x-chrome-extension,
 application/x-cocoa, cco
 application/x-conference, nsc
+application/x-coredump,
 application/x-cpio, cpio
 application/x-dbf, dbf
 application/x-dbt,
+application/x-debian-package, deb
 application/x-deepv, deepv
-application/x-director, dcr|dir|dxr
+application/x-director, dir|dxr
+application/x-dmp, dmp
+application/x-dosdriver,
 application/x-dosexec, dll
 application/x-dvi, dvi
 application/x-elc, elc
+application/x-empty,
 application/x-envoy, env|evy
 application/x-esrehber, es
 application/x-excel, xla|xld|xlk|xlt|xlv
 application/x-executable, exe
+application/x-font-gdos,
+application/x-font-pf2, pf2
+application/x-font-pfm, pfm
 application/x-font-sfn,
 application/x-font-ttf, ttf|ttc
+application/x-fptapplication/x-dbt,
 application/x-freelance, pre
+application/x-gamecube-rom,
+application/x-gdbm,
+application/x-gettext-translation,
 application/x-git,
 application/x-gsp, gsp
 application/x-gss, gss
@@ -102,46 +141,67 @@ application/x-hdf, hdf
 application/x-helpfile, help
 application/x-httpd-imap, imap
 application/x-ima, ima
+application/x-innosetup,
 application/x-internett-signup, ins
 application/x-inventor, iv
 application/x-ip2, ip
 application/x-java-applet,
 application/x-java-commerce, jcm
 application/x-java-image,
+application/x-java-jmod, jmod
 application/x-java-keystore,
+application/x-kdelnk,
 application/x-koan, skd|skm|skp|skt
 application/x-latex, latex|ltx
 application/x-livescreen, ivy
 application/x-lotus, wq1
+application/x-lz4+json, jsonlz4
+application/x-lz4, lz4
+application/x-lzh-compressed,
 application/x-lzh, lzh
+application/x-lzip, lz
+application/x-lzma, lzma
+application/x-lzop, lzo
 application/x-lzx, lzx
 application/x-mach-binary, jnilib|dylib
 application/x-mach-executable,
 application/x-magic-cap-package-1.0, mc$
 application/x-mathcad, mcd
+application/x-maxis-dbpf,
 application/x-meme, mm
 application/x-midi, midi
 application/x-mif, mif
 application/x-mix-transfer, nix
 application/xml, opf
+application/x-mobipocket-ebook, mobi
+application/vnd.amazon.mobi8-ebook, azw|azw3
+application/x-msaccess, accdb
+application/x-ms-compress-szdd, fon
 application/x-ms-pdb, pdb
+application/x-ms-reader, lit
+application/x-n64-rom, z64
 application/x-navi-animation, ani
 application/x-navidoc, nvd
 application/x-navimap, map
 application/x-navistyle, stl
+application/x-nes-rom, nes
 application/x-netcdf, cdf|nc
 application/x-newton-compatible-pkg, pkg
+application/x-nintendo-ds-rom,
 application/x-object, o
 application/x-omcdatamaker, omcd
 application/x-omc, omc
 application/x-omcregerator, omcr
 application/x-pagemaker, pm4|pm5
 application/x-pcl, pcl
+application/x-pgp-keyring,
 application/x-pixclscript, plx
 application/x-pkcs7-certreqresp, p7r
 application/x-pkcs7-signature, p7a
 application/x-project, mpc|mpt|mpv|mpx
 application/x-qpro, wb1
+application/x-rar, rar
+application/x-rpm, rpm
 application/x-sdp, sdp
 application/x-sea, sea
 application/x-seelogo, sl
@@ -149,12 +209,17 @@ application/x-setupscript,
 application/x-sharedlib, so
 application/x-shar, shar
 application/x-shockwave-flash, swf
+application/x-snappy-framed,
 application/x-sprite, spr|sprite
 application/x-sqlite3,
+application/x-stargallery-thm,
+application/x-stuffit, sit
 application/x-sv4cpio, sv4cpio
 application/x-sv4crc, sv4crc
 application/x-tar, tar
 application/x-tbook, sbk|tbk
+application/x-terminfo,
+application/x-terminfo2,
 application/x-texinfo, texi|texinfo
 application/x-tex-tfm, tfm
 application/x-ustar, ustar
@@ -163,24 +228,33 @@ application/x-vnd.audioexplosion.mzz, mzz
 application/x-vnd.ls-xpix, xpix
 application/x-vrml, vrml
 application/x-wais-source, src|wsrc
+application/x-wine-extension-ini,
 application/x-wintalk, wtk
 application/x-world, svr
 application/x-wri, wri
 application/x-x509-ca-cert, der
 application/x-xz, xz
+application/x-zip,
+application/x-zstd, zst
 application/zip, zip
+application/zlib, z
+!audio/basic, au
 audio/it, it
 audio/make, funk|my|pfunk
 audio/midi, kar
 audio/mid, rmi
-audio/mpeg, m2a|mpa
+audio/mp4, m4b
+audio/mpeg, m2a|mpa|mpga
 audio/ogg, ogg
 audio/s3m, s3m
 audio/tsp-audio, tsi
 audio/tsplayer, tsp
 audio/vnd.qcelp, qcp
 audio/voxware, vox
+audio/x-aiff, aiff|aif
+audio/x-flac, flac
 audio/x-gsm, gsd|gsm
+audio/x-hx-aac-adts,
 audio/x-jam, jam
 audio/x-liveaudio, lam
 audio/x-m4a, m4a
@@ -194,17 +268,24 @@ audio/x-nspaudio, lma
 audio/x-pn-realaudio, ram|rm|rmm|rmp
 audio/x-psid, sid
 audio/x-realaudio, ra
+audio/x-s3m,
 audio/x-twinvq-plugin, vqe|vql
 audio/x-twinvq, vqf
 audio/x-voc, voc
 audio/x-wav, wav
+!audio/x-xbox360-executable, xex
+!audio/x-xbox-executable, xbe
 font/otf,
 font/sfnt,
+font/woff2, woff2
+font/woff, woff
+image/bmp,
 image/cmu-raster, rast
 image/fif, fif
 image/florian, flo|turbot
 image/g3fax, g3
 image/gif, gif
+image/heic, heic
 image/ief, ief|iefs
 image/jpeg, jfif|jfif-tbnl|jpe|jpeg|jpg
 image/jutvision, jut
@@ -213,6 +294,9 @@ image/pict, pic|pict
 image/png, png|x-png
 !image/svg, svg
 !image/svg+xml,
+image/tiff,
+!image/vnd.adobe.photoshop, psd
+!image/vnd.djvu, djvu
 image/vnd.fpx, fpx
 image/vnd.microsoft.icon,
 image/vnd.rn-realflash, rf
@@ -220,9 +304,15 @@ image/vnd.rn-realpix, rp
 image/vnd.wap.wbmp, wbmp
 image/vnd.xiff, xif
 image/webp, webp
+image/wmf,
+image/x-3ds, 3ds
+image/x-award-bioslogo,
 image/x-cmu-raster, ras
+image/x-cur, tga
 image/x-dwg, dwg|dxf|svf
 image/x-eps,
+image/x-exr, exr
+image/x-gem,
 image/x-icns,
 !image/x-icon, ico
 image/x-jg, art
@@ -236,32 +326,33 @@ image/x-portable-graymap, pgm
 image/x-portable-pixmap, ppm
 image/x-quicktime, qif|qti|qtif
 image/x-rgb, rgb
+image/x-tga,
 image/x-tiff, tif|tiff
-image/tiff,
+image/x-win-bitmap,
 !image/x-xcf, xcf
 !image/x-xpixmap, xpm
+image/x-xwindowdump, xwd
+message/news,
 message/rfc822, mht|mhtml|mime
 model/vnd.dwf, dwf
+model/vnd.gdl, gdl
+model/vnd.gs.gdl, gdsl
 model/vrml, wrz
 model/x-pov, pov
 text/asp, asp
 text/css, css
-text/x-sass, sass
-text/x-scss, scss
 text/html, acgi|htm|html|htmls|htx|shtml
 text/javascript, js
 text/mcf, mcf
 text/pascal, pas
-text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml
+text/PGP,
+text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+text/x-script.python, pyx
+text/csv,
+application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
 text/scriplet, wsc
-text/x-awk, awk
-!video/x-jng, jng
-video/x-mng, mng
-image/x-cur, tga
-image/x-xwindowdump, xwd
-!image/vnd.adobe.photoshop, psd
 text/tab-separated-values, tsv
 text/troff, man|me|ms|roff|t|tr
 text/uri-list, uji|unis|uri|uris
@@ -273,6 +364,7 @@ text/webviewhtml, htt
 text/x-Algol68,
 text/x-asm, asm|s
 text/x-audiosoft-intra, aip
+text/x-awk, awk
 text/x-bcpl,
 text/x-c, c|cc|h
 text/x-c++, cpp|cxx|c++
@@ -287,23 +379,31 @@ text/x-makefile, am|mak
 text/xml, xml|pom|iml|plist
 text/x-m, m
 text/x-msdos-batch, bat
+text/x-ms-regedit, reg
+text/x-objective-c,
 text/x-pascal, p
 text/x-perl, pl
 text/x-php, php
-text/x-python, py
+text/x-po, po
+text/x-python, py|pyi
 text/x-ruby, rb
+text/x-sass, sass
+text/x-scss, scss
 text/x-server-parsed-html, ssi
 text/x-setext, etx
 text/x-sgml, sgm|sgml
 text/x-shellscript, sh
 text/x-speech, talk
+text/x-tcl,
 text/x-tex, tex
 text/x-uil, uil
 text/x-uuencode, uue
 text/x-vcalendar, vcs
+text/x-vcard, vcf
 video/animaflex, afl
 video/avi, avi
 video/avs-video, avs
+video/MP2T,
 video/mp4, mp4
 video/mpeg, m1v|m2v|mpe|mpeg|mpg
 video/quicktime, moov|mov|qt
@@ -318,101 +418,35 @@ video/x-atomic3d-feature, fmf
 video/x-dl, dl
 video/x-dv, dif|dv
 video/x-fli, fli
+video/x-flv, flv
 video/x-isvideo, isu
+!video/x-jng, jng
+video/x-m4v, m4v
+video/x-matroska, mkv
+video/x-mng, mng
 video/x-motion-jpeg, mjpg
-video/x-ms-asf, asf|asx
+video/x-ms-asf, asf|asx|wmv
+video/x-msvideo, divx
 video/x-qtc, qtc
 video/x-sgi-movie, movie|mv
-application/x-7z-compressed, 7z
-application/vnd.openxmlformats-officedocument.wordprocessingml.document, docx
-text/x-po, po
-application/x-rpm, rpm
-application/x-debian-package, deb
-application/vnd.iccprofile, icm
-application/dicom, dcm
-image/x-exr, exr
-application/vnd.iccprofile, icm
-video/x-matroska, mkv
-application/x-empty,
-model/vnd.gdl, gdl
-model/vnd.gs.gdl, gdsl
-font/woff, woff
-font/woff2, woff2
-application/epub+zip, epub
-application/x-mobipocket-ebook, mobi
-audio/x-flac, flac
-application/x-rar, rar
-video/x-msvideo, divx
-video/x-flv, flv
-application/x-kdelnk,
-text/x-tcl,
-application/ogg, ogv
-application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, xlsx
-application/vnd.ms-cab-compressed, cab
-audio/mp4, m4b
-!image/vnd.djvu, djvu
-application/x-ms-reader, lit
-application/CDFV2-corrupt,
-text/x-vcard, vcf
-application/x-innosetup,
-application/winhelp, hlp
-image/x-tga,
-application/x-wine-extension-ini,
-application/x-cbz, cbz
-application/x-cbr, cbr
-application/x-ms-compress-szdd, fon
-application/x-atari-7800-rom, a78
-application/x-nes-rom, nes
-application/x-font-pfm, pfm
-application/x-gettext-translation,
-image/wmf,
-application/pgp-keys,
-image/x-3ds, 3ds
-application/x-lz4, lz4
-application/vnd.openxmlformats-officedocument.presentationml.presentation, pptx
-application/vnd.oasis.opendocument.presentation, odp
-application/x-msaccess, accdb
-application/vnd.oasis.opendocument.spreadsheet, ods
-audio/x-aiff, aiff|aif
-text/x-ms-regedit, reg
-application/x-gamecube-rom,
-application/x-nintendo-ds-rom,
-text/x-objective-c,
-application/x-font-gdos,
-application/x-apple-diskimage,
-application/x-zstd, zst
-video/x-m4v, m4v
-message/news,
-application/vnd.symbian.install,
-application/x-lzh-compressed,
-application/x-dosdriver,
-application/vnd.tcpdump.pcap, pcap
 x-epoc/x-sisx-app,
-application/x-avira-qua,
+application/x-zstd-dictionary,
-video/MP2T,
+application/vnd.ms-outlook, msg
-application/x-snappy-framed,
+image/x-olympus-orf, orf
-application/x-lz4+json, jsonlz4
+image/x-nikon-nef, nef
-application/x-dmp, dmp
+image/x-fuji-raf, raf
-application/zlib, z
+image/x-panasonic-raw, rw2|raw
-application/x-pgp-keyring,
+image/x-adobe-dng, dng
-application/x-gdbm,
+image/x-canon-cr2, cr2
-application/x-font-pf2, pf2
+image/x-canon-crw, crw
-application/x-zip,
+image/x-dcraw,
-application/x-coredump,
+image/x-kodak-dcr, dcr
-application/x-java-jmod, jmod
+image/x-kodak-k25, k25
-application/x-terminfo,
+image/x-kodak-kdc, kdc
-application/x-terminfo2,
+image/x-minolta-mrw, mrw
-application/x-arc,
+image/x-pentax-pef, pef
-application/vnd.lotus-1-2-3,
+image/x-sigma-x3f, xf3
-image/x-win-bitmap,
+image/x-sony-arw, arw
-application/x-maxis-dbpf,
+image/x-sony-sr2, sr2
-text/PGP,
+image/x-sony-srf, srf
-audio/x-hx-aac-adts,
+image/x-epson-erf, erf
-application/x-chrome-extension,
-image/heic, heic
-image/x-gem,
-application/x-lzma, lzma
-application/warc, warc
-application/x-lz4, lz4
-application/x-lzip, lz
-application/x-lzop, lzo

scripts/mime.py

@@ -1,8 +1,12 @@
+import zlib
+
 mimes = {}
 noparse = set()
 ext_in_hash = set()
+mime_ids = {}
 
 major_mime = {
+    "sist2": 0,
     "model": 1,
     "example": 2,
     "message": 3,
@@ -18,7 +22,6 @@ major_mime = {
 
 pdf = (
     "application/pdf",
-    "application/x-cbz",
     "application/epub+zip",
     "application/vnd.ms-xpsdocument",
 )
@@ -62,10 +65,47 @@ doc = (
     "application/vnd.openxmlformats-officedocument.presentationml.presentation"
 )
 
+mobi = (
+    "application/x-mobipocket-ebook",
+    "application/vnd.amazon.mobi8-ebook"
+)
+
+markup = (
+    "text/xml",
+    "text/html",
+    "text/x-sgml"
+)
+
+raw = (
+    "image/x-olympus-orf",
+    "image/x-nikon-nef",
+    "image/x-fuji-raf",
+    "image/x-panasonic-raw",
+    "image/x-adobe-dng",
+    "image/x-canon-cr2",
+    "image/x-canon-crw",
+    "image/x-dcraw",
+    "image/x-kodak-dcr",
+    "image/x-kodak-k25",
+    "image/x-kodak-kdc",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-sigma-x3f",
+    "image/x-sony-arw",
+    "image/x-sony-sr2",
+    "image/x-sony-srf",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-epson-erf",
+)
+
 cnt = 1
 
 
 def mime_id(mime):
+    if mime in mime_ids:
+        return mime_ids[mime]
+
     global cnt
     major = mime.split("/")[0]
     mime_id = str((major_mime[major] << 16) + cnt)
@@ -82,8 +122,16 @@ def mime_id(mime):
         mime_id += " | 0x08000000"
     elif mime in doc:
         mime_id += " | 0x04000000"
+    elif mime in mobi:
+        mime_id += " | 0x02000000"
+    elif mime in markup:
+        mime_id += " | 0x01000000"
+    elif mime in raw:
+        mime_id += " | 0x00800000"
     elif mime == "application/x-empty":
+        cnt -= 1
         return "1"
+    mime_ids[mime] = mime_id
     return mime_id
 
 
@@ -91,24 +139,40 @@ def clean(t):
     return t.replace("/", "_").replace(".", "_").replace("+", "_").replace("-", "_")
 
 
-with open("mime.csv") as f:
+def crc(s):
+    return zlib.crc32(s.encode()) & 0xffffffff
+
+
+with open("scripts/mime.csv") as f:
     for l in f:
         mime, ext_list = l.split(",")
         if l.startswith("!"):
             mime = mime[1:]
             noparse.add(mime)
-        ext = [x.strip() for x in ext_list.split("|")]
+        ext = [x.strip() for x in ext_list.split("|") if x.strip() != ""]
         mimes[mime] = ext
 
+    seen_crc = set()
+    for ext in mimes.values():
+        for e in ext:
+            if crc(e) in seen_crc:
+                raise Exception("CRC32 collision")
+            seen_crc.add(crc(e))
+
+    seen_crc = set()
+    for mime in mimes.keys():
+        if crc(mime) in seen_crc:
+            raise Exception("CRC32 collision")
+        seen_crc.add(crc(mime))
+
     print("// **Generated by mime.py**")
     print("#ifndef MIME_GENERATED_C")
     print("#define MIME_GENERATED_C")
-    print("#include <glib-2.0/glib.h>\n")
     print("#include <stdlib.h>\n")
     # Enum
     print("enum mime {")
     for mime, ext in sorted(mimes.items()):
-        print("    " + clean(mime) + "=" + mime_id(mime) + ",")
+        print(f"{clean(mime)}={mime_id(mime)},")
     print("};")
 
     # Enum -> string
@@ -119,20 +183,28 @@ with open("mime.csv") as f:
     print("default: return NULL;}}")
 
     # Ext -> Enum
-    print("GHashTable *mime_get_ext_table() {"
+    print("unsigned int mime_extension_lookup(unsigned long extension_crc32) {"
-          "GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);")
+          "switch (extension_crc32) {")
     for mime, ext in mimes.items():
-        for e in [e for e in ext if e]:
+        if len(ext) > 0:
-            print("g_hash_table_insert(ext_table, \"" + e + "\", (gpointer)" + clean(mime) + ");")
+            for e in ext:
-            if e in ext_in_hash:
+                print(f"case {crc(e)}:", end="")
-                raise Exception("extension already in hash: " + e)
+            print(f"return {clean(mime)};")
-            ext_in_hash.add(e)
+    print("default: return 0;}}")
-    print("return ext_table;}")
 
     # string -> Enum
-    print("GHashTable *mime_get_mime_table() {"
+    print("unsigned int mime_name_lookup(unsigned long mime_crc32) {"
-          "GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);")
+          "switch (mime_crc32) {")
-    for mime, ext in mimes.items():
+    for mime in mimes.keys():
-        print("g_hash_table_insert(mime_table, \"" + mime + "\", (gpointer)" + clean(mime) + ");")
+        print(f"case {crc(mime)}: return {clean(mime)};")
-    print("return mime_table;}")
+
+    print("default: return 0;}}")
+
+    # mime list
+
+    mime_list = ",".join(mime_id(x) for x in mimes.keys()) + ",0"
+
+    print(f"unsigned int mime_ids[] = {{{mime_list}}};")
+    print("unsigned int* get_mime_ids() { return mime_ids; }")
+
     print("#endif")

scripts/serve_static.py

@@ -1,10 +1,10 @@
 files = [
-    "web/css/bundle.css",
+    "sist2-vue/src/assets/favicon.ico",
-    "web/css/bundle_dark.css",
+    "sist2-vue/dist/css/chunk-vendors.css",
-    "web/js/bundle.js",
+    "sist2-vue/dist/css/index.css",
-    "web/img/sprite-skin-flat.png",
+    "sist2-vue/dist/js/chunk-vendors.js",
-    "web/img/sprite-skin-flat-dark.png",
+    "sist2-vue/dist/js/index.js",
-    "web/search.html",
+    "sist2-vue/dist/index.html",
 ]
 
 
@@ -13,6 +13,10 @@ def clean(filepath):
 
 
 for file in files:
-    with open(file, "rb") as f:
+    try:
-        data = f.read()
+        with open(file, "rb") as f:
+            data = f.read()
+    except:
+        data = bytes([])
+
     print("char %s[%d] = {%s};" % (clean(file), len(data), ",".join(str(int(b)) for b in data)))

scripts/sqlite_extension.c

@@ -0,0 +1,84 @@
+#include <sqlite3ext.h>
+#include <string.h>
+#include <stdlib.h>
+
+SQLITE_EXTENSION_INIT1
+
+static int sep_rfind(const char *str) {
+    for (int i = (int) strlen(str); i >= 0; i--) {
+        if (str[i] == '/') {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *value = (const char *) sqlite3_value_text(argv[0]);
+
+    int stop = sep_rfind(value);
+    if (stop == -1) {
+        sqlite3_result_null(ctx);
+        return;
+    }
+    char parent[4096 * 3];
+    strncpy(parent, value, stop);
+
+    sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
+}
+
+void random_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    char state_buf[32] = {0,};
+    struct random_data buf;
+    int result;
+
+    long seed = sqlite3_value_int64(argv[0]);
+
+    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+
+    random_r(&buf, &result);
+
+    sqlite3_result_int(ctx, result);
+}
+
+
+int sqlite3_extension_init(
+        sqlite3 *db,
+        char **pzErrMsg,
+        const sqlite3_api_routines *pApi
+) {
+    SQLITE_EXTENSION_INIT2(pApi);
+
+
+    sqlite3_create_function(
+            db,
+            "path_parent",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            path_parent_func,
+            NULL,
+            NULL
+    );
+
+    sqlite3_create_function(
+            db,
+            "random_seeded",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            random_func,
+            NULL,
+            NULL
+    );
+
+    return SQLITE_OK;
+}

scripts/sqlite_extension_compile.sh

@@ -0,0 +1 @@
+gcc -I/mnt/work/vcpkg/installed/x64-linux/include -g -fPIC -shared sqlite_extension.c -o sist2funcs.so

scripts/start_dev_es.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es3"\
+       	-p 9200:9200 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.17.9

scripts/start_dev_es_6.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es-6"\
+       	-p 9202:9200 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:6.8.0

scripts/start_dev_es_8.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es3"\
+       	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0

scripts/test_in_docker.sh

@@ -0,0 +1,7 @@
+docker build . -t tmp
+
+docker run --rm -it\
+  -v $(pwd):/host \
+  tmp \
+  scan --ocr-lang eng --ocr-ebooks -t6 --incremental --very-verbose \
+  -o /host/docker.sist2 /host/third-party/libscan/libscan-test-files/test_files/

sist2-admin/frontend/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ]
+}

sist2-admin/frontend/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "sist2-admin-vue",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve",
+    "build": "vue-cli-service build",
+    "watch": "vue-cli-service build --watch"
+  },
+  "dependencies": {
+    "axios": "^1.6.0",
+    "bootstrap-vue": "^2.21.2",
+    "core-js": "^3.6.5",
+    "moment": "^2.29.3",
+    "socket.io-client": "^4.5.1",
+    "vue": "^2.6.14",
+    "vue-i18n": "^8.24.4",
+    "vue-router": "^3.5.4",
+    "vuex": "^3.4.0"
+  },
+  "devDependencies": {
+    "@vue/cli-plugin-babel": "~5.0.8",
+    "@vue/cli-plugin-router": "~5.0.8",
+    "@vue/cli-plugin-vuex": "~5.0.8",
+    "@vue/cli-service": "~5.0.8",
+    "babel-eslint": "^10.1.0",
+    "bootstrap": "^4.5.2",
+    "vue-template-compiler": "^2.6.11"
+  },
+  "eslintConfig": {
+    "root": true,
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "plugin:vue/essential",
+      "eslint:recommended"
+    ],
+    "parserOptions": {
+      "parser": "babel-eslint"
+    },
+    "rules": {}
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead"
+  ]
+}