Fixes #38

README.md

@@ -75,7 +75,7 @@ pdf,xps,cbz,cbr,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
 `text/plain` | *(none)* | yes | no | - |
 tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
-docx, xlsx, pptx | libOPC | yes | no | no |
+docx, xlsx, pptx | *(none)* | yes | no | no |
 
 \* *See [Archive files](#archive-files)*
  

ci/build.sh

@@ -2,6 +2,11 @@
 
 ./scripts/get_static_libs.sh
 
-cmake .
+rm -rf CMakeFiles CmakeCache.txt
+cmake -DSIST_DEBUG=off .
 make
 strip sist2
+
+rm -rf CMakeFiles CmakeCache.txt
+cmake -DSIST_DEBUG=on .
+make

mime.csv

@@ -2,10 +2,14 @@ application/arj, arj
 application/base64, mme
 application/binhex, hqx
 application/book, boo|book
+application/CDFV2-corrupt,
 application/CDFV2, sdv
 application/clariscad, ccad
 application/commonground, dp
+application/csv,
+application/dicom, dcm
 application/drafting, drw
+application/epub+zip, epub
 application/freeloader, frl
 application/futuresplash, spl
 application/groupwise, vew
@@ -17,7 +21,6 @@ application/inf, inf
 application/java-archive, jar
 application/java, class
 application/javascript,
-application/x-archive, a
 application/json, json
 application/marc, mrc
 application/mbedlet, mbd
@@ -27,7 +30,9 @@ application/msword, doc|dot|w6w|wiz|word
 application/netmc, mcp
 application/octet-stream, bin|dump|gpg
 application/oda, oda
+application/ogg, ogv
 application/pdf, pdf
+application/pgp-keys,
 application/pgp-signature, pgp
 application/pkcs7-signature, p7s
 application/pkix-cert, cer|crt
@@ -43,6 +48,10 @@ application/vda, vda
 application/vnd.fdf, fdf
 application/vnd.font-fontforge-sfd, sfd
 application/vnd.hp-hpgl, hgl|hpg|hpgl
+application/vnd.iccprofile, icm
+application/vnd.iccprofile, icm
+application/vnd.lotus-1-2-3,
+application/vnd.ms-cab-compressed, cab
 application/vnd.ms-excel, xlb|xlc|xll|xlm|xls|xlw
 application/vnd.ms-fontobject, eot
 application/vnd.ms-opentype, otf
@@ -54,45 +63,75 @@ application/vnd.ms-project, mpp
 application/vnd.oasis.opendocument.base, odb
 application/vnd.oasis.opendocument.formula, odf
 application/vnd.oasis.opendocument.graphics, odg
+application/vnd.oasis.opendocument.presentation, odp
+application/vnd.oasis.opendocument.spreadsheet, ods
 application/vnd.oasis.opendocument.text, odt
+application/vnd.openxmlformats-officedocument.presentationml.presentation, pptx
+application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, xlsx
+application/vnd.openxmlformats-officedocument.wordprocessingml.document, docx
+application/vnd.symbian.install,
+application/vnd.tcpdump.pcap, pcap
 application/vnd.wap.wmlc, wmlc
 application/vnd.wap.wmlscriptc, wmlsc
 application/vnd.xara, web
 application/vocaltec-media-desc, vmd
 application/vocaltec-media-file, vmf
+application/warc, warc
+application/winhelp, hlp
 application/wordperfect6.0, w60
 application/wordperfect6.1, w61
 application/wordperfect, wp|wp5|wp6|wpd
 application/x-123, wk1
+application/x-7z-compressed, 7z
 application/x-aim, aim
+application/x-apple-diskimage,
+application/x-arc,
+application/x-archive, a
+application/x-atari-7800-rom, a78
 application/x-authorware-bin, aab
 application/x-authorware-map, aam
 application/x-authorware-seg, aas
+application/x-avira-qua,
 application/x-bcpio, bcpio
 application/x-bittorrent, torrent
 application/x-bsh, bsh
 application/x-bytecode.python, pyc
 application/x-bzip2, boz|bz2
 application/x-bzip, bz
+application/x-cbr, cbr
+application/x-cbz, cbz
 application/x-cdlink, vcd
 application/x-chat, cha|chat
+application/x-chrome-extension,
 application/x-cocoa, cco
 application/x-conference, nsc
+application/x-coredump,
 application/x-cpio, cpio
 application/x-dbf, dbf
 application/x-dbt,
+application/x-debian-package, deb
 application/x-deepv, deepv
 application/x-director, dcr|dir|dxr
+application/x-dmp, dmp
+application/x-dosdriver,
 application/x-dosexec, dll
 application/x-dvi, dvi
 application/x-elc, elc
+application/x-empty,
 application/x-envoy, env|evy
 application/x-esrehber, es
 application/x-excel, xla|xld|xlk|xlt|xlv
 application/x-executable, exe
+application/x-font-gdos,
+application/x-font-pf2, pf2
+application/x-font-pfm, pfm
 application/x-font-sfn,
 application/x-font-ttf, ttf|ttc
+application/x-fptapplication/x-dbt,
 application/x-freelance, pre
+application/x-gamecube-rom,
+application/x-gdbm,
+application/x-gettext-translation,
 application/x-git,
 application/x-gsp, gsp
 application/x-gss, gss
@@ -102,46 +141,67 @@ application/x-hdf, hdf
 application/x-helpfile, help
 application/x-httpd-imap, imap
 application/x-ima, ima
+application/x-innosetup,
 application/x-internett-signup, ins
 application/x-inventor, iv
 application/x-ip2, ip
 application/x-java-applet,
 application/x-java-commerce, jcm
 application/x-java-image,
+application/x-java-jmod, jmod
 application/x-java-keystore,
+application/x-kdelnk,
 application/x-koan, skd|skm|skp|skt
 application/x-latex, latex|ltx
 application/x-livescreen, ivy
 application/x-lotus, wq1
+application/x-lz4+json, jsonlz4
+application/x-lz4, lz4
+application/x-lz4, lz4
+application/x-lzh-compressed,
 application/x-lzh, lzh
+application/x-lzip, lz
+application/x-lzma, lzma
+application/x-lzop, lzo
 application/x-lzx, lzx
 application/x-mach-binary, jnilib|dylib
 application/x-mach-executable,
 application/x-magic-cap-package-1.0, mc$
 application/x-mathcad, mcd
+application/x-maxis-dbpf,
 application/x-meme, mm
 application/x-midi, midi
 application/x-mif, mif
 application/x-mix-transfer, nix
 application/xml, opf
+application/x-mobipocket-ebook, mobi
+application/x-msaccess, accdb
+application/x-ms-compress-szdd, fon
 application/x-ms-pdb, pdb
+application/x-ms-reader, lit
+application/x-n64-rom, z64
 application/x-navi-animation, ani
 application/x-navidoc, nvd
 application/x-navimap, map
 application/x-navistyle, stl
+application/x-nes-rom, nes
 application/x-netcdf, cdf|nc
 application/x-newton-compatible-pkg, pkg
+application/x-nintendo-ds-rom,
 application/x-object, o
 application/x-omcdatamaker, omcd
 application/x-omc, omc
 application/x-omcregerator, omcr
 application/x-pagemaker, pm4|pm5
 application/x-pcl, pcl
+application/x-pgp-keyring,
 application/x-pixclscript, plx
 application/x-pkcs7-certreqresp, p7r
 application/x-pkcs7-signature, p7a
 application/x-project, mpc|mpt|mpv|mpx
 application/x-qpro, wb1
+application/x-rar, rar
+application/x-rpm, rpm
 application/x-sdp, sdp
 application/x-sea, sea
 application/x-seelogo, sl
@@ -149,12 +209,17 @@ application/x-setupscript,
 application/x-sharedlib, so
 application/x-shar, shar
 application/x-shockwave-flash, swf
+application/x-snappy-framed,
 application/x-sprite, spr|sprite
 application/x-sqlite3,
+application/x-stargallery-thm,
+application/x-stuffit, sit
 application/x-sv4cpio, sv4cpio
 application/x-sv4crc, sv4crc
 application/x-tar, tar
 application/x-tbook, sbk|tbk
+application/x-terminfo,
+application/x-terminfo2,
 application/x-texinfo, texi|texinfo
 application/x-tex-tfm, tfm
 application/x-ustar, ustar
@@ -163,16 +228,22 @@ application/x-vnd.audioexplosion.mzz, mzz
 application/x-vnd.ls-xpix, xpix
 application/x-vrml, vrml
 application/x-wais-source, src|wsrc
+application/x-wine-extension-ini,
 application/x-wintalk, wtk
 application/x-world, svr
 application/x-wri, wri
 application/x-x509-ca-cert, der
 application/x-xz, xz
+application/x-zip,
+application/x-zstd, zst
 application/zip, zip
+application/zlib, z
+!audio/basic, au
 audio/it, it
 audio/make, funk|my|pfunk
 audio/midi, kar
 audio/mid, rmi
+audio/mp4, m4b
 audio/mpeg, m2a|mpa
 audio/ogg, ogg
 audio/s3m, s3m
@@ -180,7 +251,10 @@ audio/tsp-audio, tsi
 audio/tsplayer, tsp
 audio/vnd.qcelp, qcp
 audio/voxware, vox
+audio/x-aiff, aiff|aif
+audio/x-flac, flac
 audio/x-gsm, gsd|gsm
+audio/x-hx-aac-adts,
 audio/x-jam, jam
 audio/x-liveaudio, lam
 audio/x-m4a, m4a
@@ -194,17 +268,24 @@ audio/x-nspaudio, lma
 audio/x-pn-realaudio, ram|rm|rmm|rmp
 audio/x-psid, sid
 audio/x-realaudio, ra
+audio/x-s3m,
 audio/x-twinvq-plugin, vqe|vql
 audio/x-twinvq, vqf
 audio/x-voc, voc
 audio/x-wav, wav
+!audio/x-xbox360-executable, xex
+!audio/x-xbox-executable, xbe
 font/otf,
 font/sfnt,
+font/woff2, woff2
+font/woff, woff
+image/bmp,
 image/cmu-raster, rast
 image/fif, fif
 image/florian, flo|turbot
 image/g3fax, g3
 image/gif, gif
+image/heic, heic
 image/ief, ief|iefs
 image/jpeg, jfif|jfif-tbnl|jpe|jpeg|jpg
 image/jutvision, jut
@@ -213,6 +294,9 @@ image/pict, pic|pict
 image/png, png|x-png
 !image/svg, svg
 !image/svg+xml,
+image/tiff,
+!image/vnd.adobe.photoshop, psd
+!image/vnd.djvu, djvu
 image/vnd.fpx, fpx
 image/vnd.microsoft.icon,
 image/vnd.rn-realflash, rf
@@ -220,9 +304,15 @@ image/vnd.rn-realpix, rp
 image/vnd.wap.wbmp, wbmp
 image/vnd.xiff, xif
 image/webp, webp
+image/wmf,
+image/x-3ds, 3ds
+image/x-award-bioslogo,
 image/x-cmu-raster, ras
+image/x-cur, tga
 image/x-dwg, dwg|dxf|svf
 image/x-eps,
+image/x-exr, exr
+image/x-gem,
 image/x-icns,
 !image/x-icon, ico
 image/x-jg, art
@@ -236,32 +326,30 @@ image/x-portable-graymap, pgm
 image/x-portable-pixmap, ppm
 image/x-quicktime, qif|qti|qtif
 image/x-rgb, rgb
+image/x-tga,
 image/x-tiff, tif|tiff
-image/tiff,
+image/x-win-bitmap,
 !image/x-xcf, xcf
 !image/x-xpixmap, xpm
+image/x-xwindowdump, xwd
+message/news,
 message/rfc822, mht|mhtml|mime
 model/vnd.dwf, dwf
+model/vnd.gdl, gdl
+model/vnd.gs.gdl, gdsl
 model/vrml, wrz
 model/x-pov, pov
 text/asp, asp
 text/css, css
-text/x-sass, sass
-text/x-scss, scss
 text/html, acgi|htm|html|htmls|htx|shtml
 text/javascript, js
 text/mcf, mcf
 text/pascal, pas
+text/PGP,
 text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml
 text/richtext, rt|rtf|rtx
 text/rtf,
 text/scriplet, wsc
-text/x-awk, awk
-!video/x-jng, jng
-video/x-mng, mng
-image/x-cur, tga
-image/x-xwindowdump, xwd
-!image/vnd.adobe.photoshop, psd
 text/tab-separated-values, tsv
 text/troff, man|me|ms|roff|t|tr
 text/uri-list, uji|unis|uri|uris
@@ -273,6 +361,7 @@ text/webviewhtml, htt
 text/x-Algol68,
 text/x-asm, asm|s
 text/x-audiosoft-intra, aip
+text/x-awk, awk
 text/x-bcpl,
 text/x-c, c|cc|h
 text/x-c++, cpp|cxx|c++
@@ -287,23 +376,31 @@ text/x-makefile, am|mak
 text/xml, xml|pom|iml|plist
 text/x-m, m
 text/x-msdos-batch, bat
+text/x-ms-regedit, reg
+text/x-objective-c,
 text/x-pascal, p
 text/x-perl, pl
 text/x-php, php
+text/x-po, po
 text/x-python, py
 text/x-ruby, rb
+text/x-sass, sass
+text/x-scss, scss
 text/x-server-parsed-html, ssi
 text/x-setext, etx
 text/x-sgml, sgm|sgml
 text/x-shellscript, sh
 text/x-speech, talk
+text/x-tcl,
 text/x-tex, tex
 text/x-uil, uil
 text/x-uuencode, uue
 text/x-vcalendar, vcs
+text/x-vcard, vcf
 video/animaflex, afl
 video/avi, avi
 video/avs-video, avs
+video/MP2T,
 video/mp4, mp4
 video/mpeg, m1v|m2v|mpe|mpeg|mpg
 video/quicktime, moov|mov|qt
@@ -318,101 +415,15 @@ video/x-atomic3d-feature, fmf
 video/x-dl, dl
 video/x-dv, dif|dv
 video/x-fli, fli
+video/x-flv, flv
 video/x-isvideo, isu
+!video/x-jng, jng
+video/x-m4v, m4v
+video/x-matroska, mkv
+video/x-mng, mng
 video/x-motion-jpeg, mjpg
 video/x-ms-asf, asf|asx|wmv
+video/x-msvideo, divx
 video/x-qtc, qtc
 video/x-sgi-movie, movie|mv
-application/x-7z-compressed, 7z
-application/vnd.openxmlformats-officedocument.wordprocessingml.document, docx
-text/x-po, po
-application/x-rpm, rpm
-application/x-debian-package, deb
-application/vnd.iccprofile, icm
-application/dicom, dcm
-image/x-exr, exr
-application/vnd.iccprofile, icm
-video/x-matroska, mkv
-application/x-empty,
-model/vnd.gdl, gdl
-model/vnd.gs.gdl, gdsl
-font/woff, woff
-font/woff2, woff2
-application/epub+zip, epub
-application/x-mobipocket-ebook, mobi
-audio/x-flac, flac
-application/x-rar, rar
-video/x-msvideo, divx
-video/x-flv, flv
-application/x-kdelnk,
-text/x-tcl,
-application/ogg, ogv
-application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, xlsx
-application/vnd.ms-cab-compressed, cab
-audio/mp4, m4b
-!image/vnd.djvu, djvu
-application/x-ms-reader, lit
-application/CDFV2-corrupt,
-text/x-vcard, vcf
-application/x-innosetup,
-application/winhelp, hlp
-image/x-tga,
-application/x-wine-extension-ini,
-application/x-cbz, cbz
-application/x-cbr, cbr
-application/x-ms-compress-szdd, fon
-application/x-atari-7800-rom, a78
-application/x-nes-rom, nes
-application/x-font-pfm, pfm
-application/x-gettext-translation,
-image/wmf,
-application/pgp-keys,
-image/x-3ds, 3ds
-application/x-lz4, lz4
-application/vnd.openxmlformats-officedocument.presentationml.presentation, pptx
-application/vnd.oasis.opendocument.presentation, odp
-application/x-msaccess, accdb
-application/vnd.oasis.opendocument.spreadsheet, ods
-audio/x-aiff, aiff|aif
-text/x-ms-regedit, reg
-application/x-gamecube-rom,
-application/x-nintendo-ds-rom,
-text/x-objective-c,
-application/x-font-gdos,
-application/x-apple-diskimage,
-application/x-zstd, zst
-video/x-m4v, m4v
-message/news,
-application/vnd.symbian.install,
-application/x-lzh-compressed,
-application/x-dosdriver,
-application/vnd.tcpdump.pcap, pcap
 x-epoc/x-sisx-app,
-application/x-avira-qua,
-video/MP2T,
-application/x-snappy-framed,
-application/x-lz4+json, jsonlz4
-application/x-dmp, dmp
-application/zlib, z
-application/x-pgp-keyring,
-application/x-gdbm,
-application/x-font-pf2, pf2
-application/x-zip,
-application/x-coredump,
-application/x-java-jmod, jmod
-application/x-terminfo,
-application/x-terminfo2,
-application/x-arc,
-application/vnd.lotus-1-2-3,
-image/x-win-bitmap,
-application/x-maxis-dbpf,
-text/PGP,
-audio/x-hx-aac-adts,
-application/x-chrome-extension,
-image/heic, heic
-image/x-gem,
-application/x-lzma, lzma
-application/warc, warc
-application/x-lz4, lz4
-application/x-lzip, lz
-application/x-lzop, lzo

src/index/elastic.c

@@ -117,7 +117,11 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
 }
 
 void *print_errors(response_t *r) {
-    cJSON *ret_json = cJSON_Parse(r->body);
+    char * tmp = malloc(r->size + 1);
+    memcpy(tmp, r->body, r->size);
+    *(tmp + r->size) = '\0';
+
+    cJSON *ret_json = cJSON_Parse(tmp);
     if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
         cJSON *err;
         cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
@@ -129,6 +133,7 @@ void *print_errors(response_t *r) {
         }
     }
     cJSON_Delete(ret_json);
+    free(tmp);
 }
 
 void _elastic_flush(int max) {

src/io/serialize.c

@@ -193,6 +193,8 @@ void write_document(document_t *doc) {
 
 void thread_cleanup() {
     close(index_fd);
+    cleanup_parse();
+    cleanup_font();
 }
 
 

src/main.c

@@ -6,7 +6,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "1.3.1";
+static const char *const Version = "1.3.3";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",

src/parsing/font.c

@@ -140,6 +140,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
     if (ft_lib == NULL) {
         FT_Init_FreeType(&ft_lib);
     }
+
     if (buf == NULL) {
         return;
     }
@@ -169,6 +170,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
     APPEND_META(doc, meta_name)
 
     if (ScanCtx.tn_size <= 0) {
+        FT_Done_Face(face);
         return;
     }
 
@@ -178,6 +180,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
     err = FT_Set_Pixel_Sizes(face, 0, pixel);
     if (err != 0) {
         LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err, ft_error_string(err))
+        FT_Done_Face(face);
         return;
     }
 
@@ -224,3 +227,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
 
     FT_Done_Face(face);
 }
+
+void cleanup_font() {
+    FT_Done_FreeType(ft_lib);
+}

src/parsing/font.h

@@ -5,5 +5,6 @@
 
 
 void parse_font(const char * buf, size_t buf_len, document_t *doc);
+void cleanup_font();
 
 #endif

src/parsing/media.c

@@ -113,10 +113,13 @@ static AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder,
         // Feed it to decoder
         int decode_ret = avcodec_send_packet(decoder, &avPacket);
         if (decode_ret != 0) {
-            LOG_WARNINGF(doc->filepath,
+            LOG_ERRORF(doc->filepath,
                          "(media.c) avcodec_send_packet() returned error code [%d] %s",
                          decode_ret, av_err2str(decode_ret)
             )
+            av_frame_free(&frame);
+            av_packet_unref(&avPacket);
+            return NULL;
         }
         av_packet_unref(&avPacket);
         receive_ret = avcodec_receive_frame(decoder, frame);
@@ -139,7 +142,7 @@ static void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
 
     AVDictionaryEntry *tag = NULL;
     while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
-        char key[32];
+        char key[256];
         strncpy(key, tag->key, sizeof(key));
 
         char *ptr = key;
@@ -160,7 +163,8 @@ static void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
 }
 
 __always_inline
-static void append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
+static void
+append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
 
     if (is_video) {
         meta_line_t *meta_duration = malloc(sizeof(meta_line_t));

src/parsing/parse.c

@@ -37,7 +37,13 @@ void *read_all(parse_job_t *job, const char *buf, int bytes_read) {
 
         int ret = job->vfile.read(&job->vfile, full_buf + bytes_read, job->info.st_size - bytes_read);
         if (ret < 0) {
-            LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno))
+            free(full_buf);
+
+            if (job->vfile.is_fs_file) {
+                LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno))
+            } else {
+                LOG_ERRORF(job->filepath, "(virtual) read(): [%d] %s", ret, archive_error_string(job->vfile.arc))
+            }
             return NULL;
         }
     }
@@ -92,7 +98,13 @@ void parse(void *arg) {
         // Get mime type with libmagic
         bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
         if (bytes_read < 0) {
-            LOG_WARNINGF(job->filepath, "read() Error: %s", strerror(errno))
+
+            if (job->vfile.is_fs_file) {
+                LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno))
+            } else {
+                LOG_ERRORF(job->filepath, "(virtual) read(): [%d] %s", bytes_read, archive_error_string(job->vfile.arc))
+            }
+
             CLOSE_FILE(job->vfile)
             return;
         }
@@ -119,11 +131,11 @@ void parse(void *arg) {
     } else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) ||
                (mmime == MimeImage && doc.size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) {
 
-//        if (job->vfile.is_fs_file) {
-//            parse_media_filename(job->filepath, &doc);
-//        } else {
-//            parse_media_vfile(&job->vfile, &doc);
-//        }
+        if (job->vfile.is_fs_file) {
+            parse_media_filename(job->filepath, &doc);
+        } else {
+            parse_media_vfile(&job->vfile, &doc);
+        }
 
     } else if (IS_PDF(doc.mime)) {
         void *pdf_buf = read_all(job, (char *) buf, bytes_read);
@@ -180,3 +192,9 @@ void parse(void *arg) {
 
     CLOSE_FILE(job->vfile)
 }
+
+void cleanup_parse() {
+    if (Magic != NULL) {
+        magic_close(Magic);
+    }
+}

src/parsing/parse.h

@@ -10,4 +10,6 @@ void fs_close(struct vfile *f);
 
 void parse(void *arg);
 
+void cleanup_parse();
+
 #endif

src/parsing/pdf.c

@@ -168,7 +168,7 @@ void fill_image(fz_context *ctx, UNUSED(fz_device *dev),
     }
 }
 
-void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
+void parse_pdf(const void *buf, size_t buf_len, document_t *doc) {
 
     if (buf == NULL) {
         return;

src/parsing/pdf.h

@@ -4,6 +4,6 @@
 #include "src/sist.h"
 
 
-void parse_pdf(void *buf, size_t buf_len, document_t *doc);
+void parse_pdf(const void *buf, size_t buf_len, document_t *doc);
 
 #endif

src/util.h

@@ -19,7 +19,7 @@ typedef struct dyn_buffer {
 #include "sist.h"
 
 typedef struct text_buffer {
-    size_t max_size;
+    long max_size;
     int last_char_was_whitespace;
     dyn_buffer_t dyn_buffer;
 } text_buffer_t;
@@ -235,34 +235,48 @@ static void text_buffer_terminate_string(text_buffer_t *buf) {
     }
 }
 
-__always_inline
-static int text_buffer_append_string(text_buffer_t *buf, char *str, size_t len) {
+#define UTF8_END_OF_STRING \
+    (ptr - str >= len || *ptr == 0 || \
+    (0xc0 == (0xe0 & *ptr) && ptr - str > len - 2) || \
+    (0xe0 == (0xf0 & *ptr) && ptr - str > len - 3) || \
+    (0xf0 == (0xf8 & *ptr) && ptr - str > len - 4))
 
-    utf8_int32_t c;
-    if (str == NULL || len < 1 ||
-        (0xf0 == (0xf8 & str[0]) && len < 4) ||
-        (0xe0 == (0xf0 & str[0]) && len < 3) ||
-        (0xc0 == (0xe0 & str[0]) && len == 1) ||
-        *(str) == 0) {
+__always_inline
+static int text_buffer_append_string(text_buffer_t *buf, const char *str, size_t len) {
+
+    char *ptr = str;
+    char *oldPtr = ptr;
+
+    if (str == NULL || UTF8_END_OF_STRING) {
         return 0;
     }
 
-    for (void *v = utf8codepoint(str, &c); c != '\0' && ((char *) v - str + 4) < len; v = utf8codepoint(v, &c)) {
-        if (utf8_validchr2(v)) {
-            text_buffer_append_char(buf, c);
+    utf8_int32_t c;
+    char tmp[4];
+
+    do {
+        ptr = utf8codepoint(ptr, &c);
+        *(int *) tmp = 0x00000000;
+        memcpy(tmp, oldPtr, ptr - oldPtr);
+        oldPtr = ptr;
+
+        if (!utf8_validchr2(tmp)) {
+            continue;
         }
-    }
+
+        int ret = text_buffer_append_char(buf, c);
+
+        if (ret != 0) {
+            return ret;
+        }
+    } while (!UTF8_END_OF_STRING);
+
     return 0;
 }
 
 __always_inline
 static int text_buffer_append_string0(text_buffer_t *buf, char *str) {
-    utf8_int32_t c;
-    for (void *v = utf8codepoint(str, &c); c != '\0'; v = utf8codepoint(v, &c)) {
-        if (utf8_validchr2(v)) {
-            text_buffer_append_char(buf, c);
-        }
-    }
+    return text_buffer_append_string(buf, str, strlen(str));
 }
 
 __always_inline

web/js/search.js

@@ -14,6 +14,12 @@ let searchBusy = true;
 let selectedIndices = [];
 let indexMap = {};
 
+let size_min = 0;
+let size_max = 10000000000000;
+
+let date_min = null;
+let date_max = null;
+
 const CONF = new Settings();
 
 const _defaults = {
@@ -120,9 +126,7 @@ function getDocumentInfo(id) {
 }
 
 function handleTreeClick(tree) {
-    return (event, node, handler) => {
-        event.preventTreeDefault();
-
+    return (node) => {
         if (node.id === "any") {
             if (!node.itree.state.checked) {
                 tree.deselect();
@@ -131,12 +135,10 @@ function handleTreeClick(tree) {
             tree.node("any").deselect();
         }
 
-        handler();
         searchDebounced();
     }
 }
 
-//TODO: filter based on selected indexes, sort mime types
 $.jsonPost("es", {
     aggs: {
         mimeTypes: {
@@ -148,7 +150,7 @@ $.jsonPost("es", {
     },
     size: 0,
 }).then(resp => {
-    resp["aggregations"]["mimeTypes"]["buckets"].forEach(bucket => {
+    resp["aggregations"]["mimeTypes"]["buckets"].sort((a, b) => a.key > b.key).forEach(bucket => {
         let tmp = bucket["key"].split("/");
         let category = tmp[0];
         let mime = tmp[1];
@@ -182,7 +184,7 @@ $.jsonPost("es", {
     new InspireTreeDOM(mimeTree, {
         target: '#mimeTree'
     });
-    mimeTree.on("node.click", handleTreeClick(mimeTree));
+    mimeTree.on("node.state.changed", handleTreeClick(mimeTree));
     mimeTree.deselect();
     mimeTree.node("any").select();
 });
@@ -215,7 +217,7 @@ $.jsonPost("es", {
     new InspireTreeDOM(tagTree, {
         target: '#tagTree'
     });
-    tagTree.on("node.click", handleTreeClick(tagTree));
+    tagTree.on("node.state.changed", handleTreeClick(tagTree));
     tagTree.node("any").select();
     searchBusy = false;
 });
@@ -345,6 +347,14 @@ function search(after = null) {
         filters.push([{terms: {"tag": tags}}]);
     }
 
+    if (date_min && date_max) {
+        filters.push([{range: {mtime: {gte: date_min, lte: date_max}}}])
+    } else if (date_min) {
+        filters.push([{range: {mtime: {gte: date_min}}}])
+    } else if (date_max) {
+        filters.push([{range: {mtime: {lte: date_max}}}])
+    }
+
     let q = {
         "_source": {
             excludes: ["content", "_tie"]
@@ -422,8 +432,6 @@ function search(after = null) {
     });
 }
 
-let size_min = 0;
-let size_max = 10000000000000;
 
 let searchDebounced = _.debounce(function () {
     coolingDown = false;
@@ -431,6 +439,7 @@ let searchDebounced = _.debounce(function () {
 }, 500);
 
 searchBar.addEventListener("keyup", searchDebounced);
+pathBar.addEventListener("keyup", searchDebounced);
 
 //Size slider
 $("#sizeSlider").ionRangeSlider({
@@ -465,6 +474,37 @@ $("#sizeSlider").ionRangeSlider({
     }
 });
 
+//Date slider
+$.jsonPost("es", {
+    aggs: {
+        date_min: {min: {field: "mtime"}},
+        date_max: {max: {field: "mtime"}},
+    },
+    size: 0
+}).then(resp => {
+    $("#dateSlider").ionRangeSlider({
+        type: "double",
+        grid: false,
+        force_edges: true,
+        min: resp["aggregations"]["date_min"]["value"],
+        max: resp["aggregations"]["date_max"]["value"],
+        from: resp["aggregations"]["date_min"]["value"],
+        to: (Date.now() / 1000),
+        min_interval: 3600 * 24 * 7,
+        step: 3600 * 24,
+        drag_interval: true,
+        prettify: function (num) {
+            let date = (new Date(num * 1000));
+            return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
+        },
+        onFinish: function (e) {
+            date_min = e.from === e.min ? null : e.from;
+            date_max = e.to === e.max ? null : e.to;
+            searchDebounced();
+        }
+    });
+})
+
 function updateIndices() {
     let selected = $('#indices').find('option:selected');
     selectedIndices = [];
@@ -492,7 +532,7 @@ function getNextDepth(node) {
             bool: {
                 filter: [
                     {term: {index: node.index}},
-                    {term: {_depth: node.depth + 1}}
+                    {range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
                 ]
             }
         },
@@ -520,12 +560,21 @@ function getNextDepth(node) {
         if (!buckets) {
             return false;
         }
+
+        const paths = [];
+
         return buckets
             .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
             .sort((a, b) => a.key > b.key)
             .map(bucket => {
-                const i = bucket.key.lastIndexOf("/");
-                const name = (i === -1 || i === 1) ? bucket.key : bucket.key.slice(i + 1);
+
+                if (paths.some(n => bucket.key.startsWith(n))) {
+                    return null;
+                }
+
+                const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
+
+                paths.push(bucket.key);
 
                 return {
                     id: bucket.key,
@@ -534,7 +583,7 @@ function getNextDepth(node) {
                     index: node.index,
                     children: true,
                 }
-            })
+            }).filter(x => x !== null)
     })
 }
 
@@ -576,9 +625,7 @@ function createPathTree(target) {
 
     const button = document.querySelector("#pathBarHelper")
     const tooltip = document.querySelector("#pathTreeTooltip")
-    console.log(button)
-    console.log(tooltip)
-    Popper.createPopper(button, tooltip ,{
+    Popper.createPopper(button, tooltip, {
         trigger: "click",
         placement: "right",
     });

web/search.html

@@ -11,7 +11,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">v1.3.0</span>
+    <span class="badge badge-pill version">v1.3.3</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <button style="margin-left: auto" class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings</button>
     <a id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
@@ -20,18 +20,6 @@
 <div class="container">
     <div class="card">
         <div class="card-body">
-
-            <div class="form-group">
-                <div class="input-group">
-                    <div class="input-group-prepend">
-                        <button id="pathBarHelper" class="btn btn-outline-secondary" data-toggle="modal" data-target="#pathTreeModal">
-                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px"><path d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"/></svg>
-                        </button>
-                    </div>
-                    <input id="pathBar" type="search" class="form-control" placeholder="Filter path">
-                </div>
-            </div>
-
             <div class="input-group">
                 <div class="input-group-prepend">
                     <div class="input-group-text">
@@ -51,7 +39,24 @@
                 </div>
             </div>
 
-            <input title="File size" id="sizeSlider" name="size">
+            <div class="row">
+                <div class="col">
+                    <input title="File size" id="sizeSlider" name="size" width="100%">
+                </div>
+
+                <div class="col">
+                    <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
+                        <div class="input-group-prepend">
+                            <button id="pathBarHelper" class="btn btn-outline-secondary" data-toggle="modal" data-target="#pathTreeModal">
+                                <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px"><path d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"/></svg>
+                            </button>
+                        </div>
+                        <input id="pathBar" type="search" class="form-control" placeholder="Filter path">
+                    </div>
+                </div>
+            </div>
+
+            <input title="Date filter" id="dateSlider" name="size" width="100%">
 
             <div class="row">
                 <div class="col">