Escape invalid UTF8 characters simon987/sist2#44, increase magic buffer size

scripts/mime.csv

@@ -429,3 +429,4 @@ video/x-qtc, qtc
 video/x-sgi-movie, movie|mv
 x-epoc/x-sisx-app,
 application/x-zstd-dictionary,
+application/vnd.ms-outlook,

src/io/serialize.c

@@ -251,10 +251,17 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
         } else {
             *(buf.buf + line.ext) = '\0';
         }
-        cJSON_AddStringToObject(document, "name", buf.buf + line.base);
+
+        char tmp[PATH_MAX * 3];
+
+        str_escape(tmp, buf.buf + line.base);
+        cJSON_AddStringToObject(document, "name", tmp);
+
         if (line.base > 0) {
             *(buf.buf + line.base - 1) = '\0';
-            cJSON_AddStringToObject(document, "path", buf.buf);
+
+            str_escape(tmp, buf.buf);
+            cJSON_AddStringToObject(document, "path", tmp);
         } else {
             cJSON_AddStringToObject(document, "path", "");
         }

src/main.c

@@ -20,7 +20,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "2.2.1";
+static const char *const Version = "2.3.0";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",

src/parsing/mime_generated.c

@@ -63,378 +63,379 @@ enum mime {
     application_vnd_ms_excel=655415,
     application_vnd_ms_fontobject=655416,
     application_vnd_ms_opentype=655417 | 0x20000000,
-    application_vnd_ms_pki_certstore=655418,
-    application_vnd_ms_pki_pko=655419,
-    application_vnd_ms_pki_seccat=655420,
-    application_vnd_ms_powerpoint=655421,
-    application_vnd_ms_project=655422,
-    application_vnd_oasis_opendocument_base=655423,
-    application_vnd_oasis_opendocument_formula=655424,
-    application_vnd_oasis_opendocument_graphics=655425,
-    application_vnd_oasis_opendocument_presentation=655426,
-    application_vnd_oasis_opendocument_spreadsheet=655427,
-    application_vnd_oasis_opendocument_text=655428,
-    application_vnd_openxmlformats_officedocument_presentationml_presentation=655429 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655430 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655431 | 0x04000000,
-    application_vnd_symbian_install=655432,
-    application_vnd_tcpdump_pcap=655433,
-    application_vnd_wap_wmlc=655434,
-    application_vnd_wap_wmlscriptc=655435,
-    application_vnd_xara=655436,
-    application_vocaltec_media_desc=655437,
-    application_vocaltec_media_file=655438,
-    application_warc=655439,
-    application_winhelp=655440,
-    application_wordperfect=655441,
-    application_wordperfect6_0=655442,
-    application_wordperfect6_1=655443,
-    application_x_123=655444,
-    application_x_7z_compressed=655445 | 0x10000000,
-    application_x_aim=655446,
-    application_x_apple_diskimage=655447,
-    application_x_arc=655448 | 0x10000000,
-    application_x_archive=655449,
-    application_x_atari_7800_rom=655450,
-    application_x_authorware_bin=655451,
-    application_x_authorware_map=655452,
-    application_x_authorware_seg=655453,
-    application_x_avira_qua=655454,
-    application_x_bcpio=655455,
-    application_x_bittorrent=655456,
-    application_x_bsh=655457,
-    application_x_bytecode_python=655458,
-    application_x_bzip=655459,
-    application_x_bzip2=655460 | 0x08000000,
-    application_x_cbr=655461,
-    application_x_cbz=655462 | 0x40000000,
-    application_x_cdlink=655463,
-    application_x_chat=655464,
-    application_x_chrome_extension=655465,
-    application_x_cocoa=655466,
-    application_x_conference=655467,
-    application_x_coredump=655468,
-    application_x_cpio=655469,
-    application_x_dbf=655470,
-    application_x_dbt=655471,
-    application_x_debian_package=655472,
-    application_x_deepv=655473,
-    application_x_director=655474,
-    application_x_dmp=655475,
-    application_x_dosdriver=655476,
-    application_x_dosexec=655477,
-    application_x_dvi=655478,
-    application_x_elc=655479,
+    application_vnd_ms_outlook=655418,
+    application_vnd_ms_pki_certstore=655419,
+    application_vnd_ms_pki_pko=655420,
+    application_vnd_ms_pki_seccat=655421,
+    application_vnd_ms_powerpoint=655422,
+    application_vnd_ms_project=655423,
+    application_vnd_oasis_opendocument_base=655424,
+    application_vnd_oasis_opendocument_formula=655425,
+    application_vnd_oasis_opendocument_graphics=655426,
+    application_vnd_oasis_opendocument_presentation=655427,
+    application_vnd_oasis_opendocument_spreadsheet=655428,
+    application_vnd_oasis_opendocument_text=655429,
+    application_vnd_openxmlformats_officedocument_presentationml_presentation=655430 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655431 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655432 | 0x04000000,
+    application_vnd_symbian_install=655433,
+    application_vnd_tcpdump_pcap=655434,
+    application_vnd_wap_wmlc=655435,
+    application_vnd_wap_wmlscriptc=655436,
+    application_vnd_xara=655437,
+    application_vocaltec_media_desc=655438,
+    application_vocaltec_media_file=655439,
+    application_warc=655440,
+    application_winhelp=655441,
+    application_wordperfect=655442,
+    application_wordperfect6_0=655443,
+    application_wordperfect6_1=655444,
+    application_x_123=655445,
+    application_x_7z_compressed=655446 | 0x10000000,
+    application_x_aim=655447,
+    application_x_apple_diskimage=655448,
+    application_x_arc=655449 | 0x10000000,
+    application_x_archive=655450,
+    application_x_atari_7800_rom=655451,
+    application_x_authorware_bin=655452,
+    application_x_authorware_map=655453,
+    application_x_authorware_seg=655454,
+    application_x_avira_qua=655455,
+    application_x_bcpio=655456,
+    application_x_bittorrent=655457,
+    application_x_bsh=655458,
+    application_x_bytecode_python=655459,
+    application_x_bzip=655460,
+    application_x_bzip2=655461 | 0x08000000,
+    application_x_cbr=655462,
+    application_x_cbz=655463 | 0x40000000,
+    application_x_cdlink=655464,
+    application_x_chat=655465,
+    application_x_chrome_extension=655466,
+    application_x_cocoa=655467,
+    application_x_conference=655468,
+    application_x_coredump=655469,
+    application_x_cpio=655470,
+    application_x_dbf=655471,
+    application_x_dbt=655472,
+    application_x_debian_package=655473,
+    application_x_deepv=655474,
+    application_x_director=655475,
+    application_x_dmp=655476,
+    application_x_dosdriver=655477,
+    application_x_dosexec=655478,
+    application_x_dvi=655479,
+    application_x_elc=655480,
     application_x_empty=1,
-    application_x_envoy=655481,
-    application_x_esrehber=655482,
-    application_x_excel=655483,
-    application_x_executable=655484,
-    application_x_font_gdos=655485,
-    application_x_font_pf2=655486,
-    application_x_font_pfm=655487,
-    application_x_font_sfn=655488,
-    application_x_font_ttf=655489 | 0x20000000,
-    application_x_fptapplication_x_dbt=655490,
-    application_x_freelance=655491,
-    application_x_gamecube_rom=655492,
-    application_x_gdbm=655493,
-    application_x_gettext_translation=655494,
-    application_x_git=655495,
-    application_x_gsp=655496,
-    application_x_gss=655497,
-    application_x_gtar=655498,
-    application_x_gzip=655499,
-    application_x_hdf=655500,
-    application_x_helpfile=655501,
-    application_x_httpd_imap=655502,
-    application_x_ima=655503,
-    application_x_innosetup=655504,
-    application_x_internett_signup=655505,
-    application_x_inventor=655506,
-    application_x_ip2=655507,
-    application_x_java_applet=655508,
-    application_x_java_commerce=655509,
-    application_x_java_image=655510,
-    application_x_java_jmod=655511,
-    application_x_java_keystore=655512,
-    application_x_kdelnk=655513,
-    application_x_koan=655514,
-    application_x_latex=655515,
-    application_x_livescreen=655516,
-    application_x_lotus=655517,
-    application_x_lz4=655518 | 0x08000000,
-    application_x_lz4_json=655519,
-    application_x_lzh=655520,
-    application_x_lzh_compressed=655521,
-    application_x_lzip=655522 | 0x08000000,
-    application_x_lzma=655523 | 0x08000000,
-    application_x_lzop=655524 | 0x08000000,
-    application_x_lzx=655525,
-    application_x_mach_binary=655526,
-    application_x_mach_executable=655527,
-    application_x_magic_cap_package_1_0=655528,
-    application_x_mathcad=655529,
-    application_x_maxis_dbpf=655530,
-    application_x_meme=655531,
-    application_x_midi=655532,
-    application_x_mif=655533,
-    application_x_mix_transfer=655534,
-    application_x_mobipocket_ebook=655535 | 0x02000000,
-    application_x_ms_compress_szdd=655536,
-    application_x_ms_pdb=655537,
-    application_x_ms_reader=655538,
-    application_x_msaccess=655539,
-    application_x_n64_rom=655540,
-    application_x_navi_animation=655541,
-    application_x_navidoc=655542,
-    application_x_navimap=655543,
-    application_x_navistyle=655544,
-    application_x_nes_rom=655545,
-    application_x_netcdf=655546,
-    application_x_newton_compatible_pkg=655547,
-    application_x_nintendo_ds_rom=655548,
-    application_x_object=655549,
-    application_x_omc=655550,
-    application_x_omcdatamaker=655551,
-    application_x_omcregerator=655552,
-    application_x_pagemaker=655553,
-    application_x_pcl=655554,
-    application_x_pgp_keyring=655555,
-    application_x_pixclscript=655556,
-    application_x_pkcs7_certreqresp=655557,
-    application_x_pkcs7_signature=655558,
-    application_x_project=655559,
-    application_x_qpro=655560,
-    application_x_rar=655561 | 0x10000000,
-    application_x_rpm=655562,
-    application_x_sdp=655563,
-    application_x_sea=655564,
-    application_x_seelogo=655565,
-    application_x_setupscript=655566,
-    application_x_shar=655567,
-    application_x_sharedlib=655568,
-    application_x_shockwave_flash=655569,
-    application_x_snappy_framed=655570,
-    application_x_sprite=655571,
-    application_x_sqlite3=655572,
-    application_x_stargallery_thm=655573,
-    application_x_stuffit=655574,
-    application_x_sv4cpio=655575,
-    application_x_sv4crc=655576,
-    application_x_tar=655577 | 0x10000000,
-    application_x_tbook=655578,
-    application_x_terminfo=655579,
-    application_x_terminfo2=655580,
-    application_x_tex_tfm=655581,
-    application_x_texinfo=655582,
-    application_x_ustar=655583,
-    application_x_visio=655584,
-    application_x_vnd_audioexplosion_mzz=655585,
-    application_x_vnd_ls_xpix=655586,
-    application_x_vrml=655587,
-    application_x_wais_source=655588,
-    application_x_wine_extension_ini=655589,
-    application_x_wintalk=655590,
-    application_x_world=655591,
-    application_x_wri=655592,
-    application_x_x509_ca_cert=655593,
-    application_x_xz=655594 | 0x08000000,
-    application_x_zip=655595,
-    application_x_zstd=655596 | 0x08000000,
-    application_x_zstd_dictionary=655597,
-    application_xml=655598,
-    application_zip=655599 | 0x10000000,
-    application_zlib=655600,
-    audio_basic=458993 | 0x80000000,
-    audio_it=458994,
-    audio_make=458995,
-    audio_mid=458996,
-    audio_midi=458997,
-    audio_mp4=458998,
-    audio_mpeg=458999,
-    audio_ogg=459000,
-    audio_s3m=459001,
-    audio_tsp_audio=459002,
-    audio_tsplayer=459003,
-    audio_vnd_qcelp=459004,
-    audio_voxware=459005,
-    audio_x_aiff=459006,
-    audio_x_flac=459007,
-    audio_x_gsm=459008,
-    audio_x_hx_aac_adts=459009,
-    audio_x_jam=459010,
-    audio_x_liveaudio=459011,
-    audio_x_m4a=459012,
-    audio_x_midi=459013,
-    audio_x_mod=459014,
-    audio_x_mp4a_latm=459015,
-    audio_x_mpeg_3=459016,
-    audio_x_mpequrl=459017,
-    audio_x_nspaudio=459018,
-    audio_x_pn_realaudio=459019,
-    audio_x_psid=459020,
-    audio_x_realaudio=459021,
-    audio_x_s3m=459022,
-    audio_x_twinvq=459023,
-    audio_x_twinvq_plugin=459024,
-    audio_x_voc=459025,
-    audio_x_wav=459026,
-    audio_x_xbox_executable=459027 | 0x80000000,
-    audio_x_xbox360_executable=459028 | 0x80000000,
-    audio_xm=459029,
-    font_otf=327958 | 0x20000000,
-    font_sfnt=327959 | 0x20000000,
-    font_woff=327960 | 0x20000000,
-    font_woff2=327961 | 0x20000000,
-    image_bmp=524570,
-    image_cmu_raster=524571,
-    image_fif=524572,
-    image_florian=524573,
-    image_g3fax=524574,
-    image_gif=524575,
-    image_heic=524576,
-    image_ief=524577,
-    image_jpeg=524578,
-    image_jutvision=524579,
-    image_naplps=524580,
-    image_pict=524581,
-    image_png=524582,
-    image_svg=524583 | 0x80000000,
-    image_svg_xml=524584 | 0x80000000,
-    image_tiff=524585,
-    image_vnd_adobe_photoshop=524586 | 0x80000000,
-    image_vnd_djvu=524587 | 0x80000000,
-    image_vnd_fpx=524588,
-    image_vnd_microsoft_icon=524589,
-    image_vnd_rn_realflash=524590,
-    image_vnd_rn_realpix=524591,
-    image_vnd_wap_wbmp=524592,
-    image_vnd_xiff=524593,
-    image_webp=524594,
-    image_wmf=524595,
-    image_x_3ds=524596,
-    image_x_award_bioslogo=524597,
-    image_x_cmu_raster=524598,
-    image_x_cur=524599,
-    image_x_dwg=524600,
-    image_x_eps=524601,
-    image_x_exr=524602,
-    image_x_gem=524603,
-    image_x_icns=524604,
-    image_x_icon=524605 | 0x80000000,
-    image_x_jg=524606,
-    image_x_jps=524607,
-    image_x_ms_bmp=524608,
-    image_x_niff=524609,
-    image_x_pcx=524610,
-    image_x_pict=524611,
-    image_x_portable_bitmap=524612,
-    image_x_portable_graymap=524613,
-    image_x_portable_pixmap=524614,
-    image_x_quicktime=524615,
-    image_x_rgb=524616,
-    image_x_tga=524617,
-    image_x_tiff=524618,
-    image_x_win_bitmap=524619,
-    image_x_xcf=524620 | 0x80000000,
-    image_x_xpixmap=524621 | 0x80000000,
-    image_x_xwindowdump=524622,
-    message_news=196943,
-    message_rfc822=196944,
-    model_vnd_dwf=65873,
-    model_vnd_gdl=65874,
-    model_vnd_gs_gdl=65875,
-    model_vrml=65876,
-    model_x_pov=65877,
-    text_PGP=590166,
-    text_asp=590167,
-    text_css=590168,
-    text_html=590169 | 0x01000000,
-    text_javascript=590170,
-    text_mcf=590171,
-    text_pascal=590172,
-    text_plain=590173,
-    text_richtext=590174,
-    text_rtf=590175,
-    text_scriplet=590176,
-    text_tab_separated_values=590177,
-    text_troff=590178,
-    text_uri_list=590179,
-    text_vnd_abc=590180,
-    text_vnd_fmi_flexstor=590181,
-    text_vnd_wap_wml=590182,
-    text_vnd_wap_wmlscript=590183,
-    text_webviewhtml=590184,
-    text_x_Algol68=590185,
-    text_x_asm=590186,
-    text_x_audiosoft_intra=590187,
-    text_x_awk=590188,
-    text_x_bcpl=590189,
-    text_x_c=590190,
-    text_x_c__=590191,
-    text_x_component=590192,
-    text_x_diff=590193,
-    text_x_fortran=590194,
-    text_x_java=590195,
-    text_x_la_asf=590196,
-    text_x_lisp=590197,
-    text_x_m=590198,
-    text_x_m4=590199,
-    text_x_makefile=590200,
-    text_x_ms_regedit=590201,
-    text_x_msdos_batch=590202,
-    text_x_objective_c=590203,
-    text_x_pascal=590204,
-    text_x_perl=590205,
-    text_x_php=590206,
-    text_x_po=590207,
-    text_x_python=590208,
-    text_x_ruby=590209,
-    text_x_sass=590210,
-    text_x_scss=590211,
-    text_x_server_parsed_html=590212,
-    text_x_setext=590213,
-    text_x_sgml=590214 | 0x01000000,
-    text_x_shellscript=590215,
-    text_x_speech=590216,
-    text_x_tcl=590217,
-    text_x_tex=590218,
-    text_x_uil=590219,
-    text_x_uuencode=590220,
-    text_x_vcalendar=590221,
-    text_x_vcard=590222,
-    text_xml=590223 | 0x01000000,
-    video_MP2T=393616,
-    video_animaflex=393617,
-    video_avi=393618,
-    video_avs_video=393619,
-    video_mp4=393620,
-    video_mpeg=393621,
-    video_quicktime=393622,
-    video_vdo=393623,
-    video_vivo=393624,
-    video_vnd_rn_realvideo=393625,
-    video_vosaic=393626,
-    video_webm=393627,
-    video_x_amt_demorun=393628,
-    video_x_amt_showrun=393629,
-    video_x_atomic3d_feature=393630,
-    video_x_dl=393631,
-    video_x_dv=393632,
-    video_x_fli=393633,
-    video_x_flv=393634,
-    video_x_isvideo=393635,
-    video_x_jng=393636 | 0x80000000,
-    video_x_m4v=393637,
-    video_x_matroska=393638,
-    video_x_mng=393639,
-    video_x_motion_jpeg=393640,
-    video_x_ms_asf=393641,
-    video_x_msvideo=393642,
-    video_x_qtc=393643,
-    video_x_sgi_movie=393644,
-    x_epoc_x_sisx_app=721325,
+    application_x_envoy=655482,
+    application_x_esrehber=655483,
+    application_x_excel=655484,
+    application_x_executable=655485,
+    application_x_font_gdos=655486,
+    application_x_font_pf2=655487,
+    application_x_font_pfm=655488,
+    application_x_font_sfn=655489,
+    application_x_font_ttf=655490 | 0x20000000,
+    application_x_fptapplication_x_dbt=655491,
+    application_x_freelance=655492,
+    application_x_gamecube_rom=655493,
+    application_x_gdbm=655494,
+    application_x_gettext_translation=655495,
+    application_x_git=655496,
+    application_x_gsp=655497,
+    application_x_gss=655498,
+    application_x_gtar=655499,
+    application_x_gzip=655500,
+    application_x_hdf=655501,
+    application_x_helpfile=655502,
+    application_x_httpd_imap=655503,
+    application_x_ima=655504,
+    application_x_innosetup=655505,
+    application_x_internett_signup=655506,
+    application_x_inventor=655507,
+    application_x_ip2=655508,
+    application_x_java_applet=655509,
+    application_x_java_commerce=655510,
+    application_x_java_image=655511,
+    application_x_java_jmod=655512,
+    application_x_java_keystore=655513,
+    application_x_kdelnk=655514,
+    application_x_koan=655515,
+    application_x_latex=655516,
+    application_x_livescreen=655517,
+    application_x_lotus=655518,
+    application_x_lz4=655519 | 0x08000000,
+    application_x_lz4_json=655520,
+    application_x_lzh=655521,
+    application_x_lzh_compressed=655522,
+    application_x_lzip=655523 | 0x08000000,
+    application_x_lzma=655524 | 0x08000000,
+    application_x_lzop=655525 | 0x08000000,
+    application_x_lzx=655526,
+    application_x_mach_binary=655527,
+    application_x_mach_executable=655528,
+    application_x_magic_cap_package_1_0=655529,
+    application_x_mathcad=655530,
+    application_x_maxis_dbpf=655531,
+    application_x_meme=655532,
+    application_x_midi=655533,
+    application_x_mif=655534,
+    application_x_mix_transfer=655535,
+    application_x_mobipocket_ebook=655536 | 0x02000000,
+    application_x_ms_compress_szdd=655537,
+    application_x_ms_pdb=655538,
+    application_x_ms_reader=655539,
+    application_x_msaccess=655540,
+    application_x_n64_rom=655541,
+    application_x_navi_animation=655542,
+    application_x_navidoc=655543,
+    application_x_navimap=655544,
+    application_x_navistyle=655545,
+    application_x_nes_rom=655546,
+    application_x_netcdf=655547,
+    application_x_newton_compatible_pkg=655548,
+    application_x_nintendo_ds_rom=655549,
+    application_x_object=655550,
+    application_x_omc=655551,
+    application_x_omcdatamaker=655552,
+    application_x_omcregerator=655553,
+    application_x_pagemaker=655554,
+    application_x_pcl=655555,
+    application_x_pgp_keyring=655556,
+    application_x_pixclscript=655557,
+    application_x_pkcs7_certreqresp=655558,
+    application_x_pkcs7_signature=655559,
+    application_x_project=655560,
+    application_x_qpro=655561,
+    application_x_rar=655562 | 0x10000000,
+    application_x_rpm=655563,
+    application_x_sdp=655564,
+    application_x_sea=655565,
+    application_x_seelogo=655566,
+    application_x_setupscript=655567,
+    application_x_shar=655568,
+    application_x_sharedlib=655569,
+    application_x_shockwave_flash=655570,
+    application_x_snappy_framed=655571,
+    application_x_sprite=655572,
+    application_x_sqlite3=655573,
+    application_x_stargallery_thm=655574,
+    application_x_stuffit=655575,
+    application_x_sv4cpio=655576,
+    application_x_sv4crc=655577,
+    application_x_tar=655578 | 0x10000000,
+    application_x_tbook=655579,
+    application_x_terminfo=655580,
+    application_x_terminfo2=655581,
+    application_x_tex_tfm=655582,
+    application_x_texinfo=655583,
+    application_x_ustar=655584,
+    application_x_visio=655585,
+    application_x_vnd_audioexplosion_mzz=655586,
+    application_x_vnd_ls_xpix=655587,
+    application_x_vrml=655588,
+    application_x_wais_source=655589,
+    application_x_wine_extension_ini=655590,
+    application_x_wintalk=655591,
+    application_x_world=655592,
+    application_x_wri=655593,
+    application_x_x509_ca_cert=655594,
+    application_x_xz=655595 | 0x08000000,
+    application_x_zip=655596,
+    application_x_zstd=655597 | 0x08000000,
+    application_x_zstd_dictionary=655598,
+    application_xml=655599,
+    application_zip=655600 | 0x10000000,
+    application_zlib=655601,
+    audio_basic=458994 | 0x80000000,
+    audio_it=458995,
+    audio_make=458996,
+    audio_mid=458997,
+    audio_midi=458998,
+    audio_mp4=458999,
+    audio_mpeg=459000,
+    audio_ogg=459001,
+    audio_s3m=459002,
+    audio_tsp_audio=459003,
+    audio_tsplayer=459004,
+    audio_vnd_qcelp=459005,
+    audio_voxware=459006,
+    audio_x_aiff=459007,
+    audio_x_flac=459008,
+    audio_x_gsm=459009,
+    audio_x_hx_aac_adts=459010,
+    audio_x_jam=459011,
+    audio_x_liveaudio=459012,
+    audio_x_m4a=459013,
+    audio_x_midi=459014,
+    audio_x_mod=459015,
+    audio_x_mp4a_latm=459016,
+    audio_x_mpeg_3=459017,
+    audio_x_mpequrl=459018,
+    audio_x_nspaudio=459019,
+    audio_x_pn_realaudio=459020,
+    audio_x_psid=459021,
+    audio_x_realaudio=459022,
+    audio_x_s3m=459023,
+    audio_x_twinvq=459024,
+    audio_x_twinvq_plugin=459025,
+    audio_x_voc=459026,
+    audio_x_wav=459027,
+    audio_x_xbox_executable=459028 | 0x80000000,
+    audio_x_xbox360_executable=459029 | 0x80000000,
+    audio_xm=459030,
+    font_otf=327959 | 0x20000000,
+    font_sfnt=327960 | 0x20000000,
+    font_woff=327961 | 0x20000000,
+    font_woff2=327962 | 0x20000000,
+    image_bmp=524571,
+    image_cmu_raster=524572,
+    image_fif=524573,
+    image_florian=524574,
+    image_g3fax=524575,
+    image_gif=524576,
+    image_heic=524577,
+    image_ief=524578,
+    image_jpeg=524579,
+    image_jutvision=524580,
+    image_naplps=524581,
+    image_pict=524582,
+    image_png=524583,
+    image_svg=524584 | 0x80000000,
+    image_svg_xml=524585 | 0x80000000,
+    image_tiff=524586,
+    image_vnd_adobe_photoshop=524587 | 0x80000000,
+    image_vnd_djvu=524588 | 0x80000000,
+    image_vnd_fpx=524589,
+    image_vnd_microsoft_icon=524590,
+    image_vnd_rn_realflash=524591,
+    image_vnd_rn_realpix=524592,
+    image_vnd_wap_wbmp=524593,
+    image_vnd_xiff=524594,
+    image_webp=524595,
+    image_wmf=524596,
+    image_x_3ds=524597,
+    image_x_award_bioslogo=524598,
+    image_x_cmu_raster=524599,
+    image_x_cur=524600,
+    image_x_dwg=524601,
+    image_x_eps=524602,
+    image_x_exr=524603,
+    image_x_gem=524604,
+    image_x_icns=524605,
+    image_x_icon=524606 | 0x80000000,
+    image_x_jg=524607,
+    image_x_jps=524608,
+    image_x_ms_bmp=524609,
+    image_x_niff=524610,
+    image_x_pcx=524611,
+    image_x_pict=524612,
+    image_x_portable_bitmap=524613,
+    image_x_portable_graymap=524614,
+    image_x_portable_pixmap=524615,
+    image_x_quicktime=524616,
+    image_x_rgb=524617,
+    image_x_tga=524618,
+    image_x_tiff=524619,
+    image_x_win_bitmap=524620,
+    image_x_xcf=524621 | 0x80000000,
+    image_x_xpixmap=524622 | 0x80000000,
+    image_x_xwindowdump=524623,
+    message_news=196944,
+    message_rfc822=196945,
+    model_vnd_dwf=65874,
+    model_vnd_gdl=65875,
+    model_vnd_gs_gdl=65876,
+    model_vrml=65877,
+    model_x_pov=65878,
+    text_PGP=590167,
+    text_asp=590168,
+    text_css=590169,
+    text_html=590170 | 0x01000000,
+    text_javascript=590171,
+    text_mcf=590172,
+    text_pascal=590173,
+    text_plain=590174,
+    text_richtext=590175,
+    text_rtf=590176,
+    text_scriplet=590177,
+    text_tab_separated_values=590178,
+    text_troff=590179,
+    text_uri_list=590180,
+    text_vnd_abc=590181,
+    text_vnd_fmi_flexstor=590182,
+    text_vnd_wap_wml=590183,
+    text_vnd_wap_wmlscript=590184,
+    text_webviewhtml=590185,
+    text_x_Algol68=590186,
+    text_x_asm=590187,
+    text_x_audiosoft_intra=590188,
+    text_x_awk=590189,
+    text_x_bcpl=590190,
+    text_x_c=590191,
+    text_x_c__=590192,
+    text_x_component=590193,
+    text_x_diff=590194,
+    text_x_fortran=590195,
+    text_x_java=590196,
+    text_x_la_asf=590197,
+    text_x_lisp=590198,
+    text_x_m=590199,
+    text_x_m4=590200,
+    text_x_makefile=590201,
+    text_x_ms_regedit=590202,
+    text_x_msdos_batch=590203,
+    text_x_objective_c=590204,
+    text_x_pascal=590205,
+    text_x_perl=590206,
+    text_x_php=590207,
+    text_x_po=590208,
+    text_x_python=590209,
+    text_x_ruby=590210,
+    text_x_sass=590211,
+    text_x_scss=590212,
+    text_x_server_parsed_html=590213,
+    text_x_setext=590214,
+    text_x_sgml=590215 | 0x01000000,
+    text_x_shellscript=590216,
+    text_x_speech=590217,
+    text_x_tcl=590218,
+    text_x_tex=590219,
+    text_x_uil=590220,
+    text_x_uuencode=590221,
+    text_x_vcalendar=590222,
+    text_x_vcard=590223,
+    text_xml=590224 | 0x01000000,
+    video_MP2T=393617,
+    video_animaflex=393618,
+    video_avi=393619,
+    video_avs_video=393620,
+    video_mp4=393621,
+    video_mpeg=393622,
+    video_quicktime=393623,
+    video_vdo=393624,
+    video_vivo=393625,
+    video_vnd_rn_realvideo=393626,
+    video_vosaic=393627,
+    video_webm=393628,
+    video_x_amt_demorun=393629,
+    video_x_amt_showrun=393630,
+    video_x_atomic3d_feature=393631,
+    video_x_dl=393632,
+    video_x_dv=393633,
+    video_x_fli=393634,
+    video_x_flv=393635,
+    video_x_isvideo=393636,
+    video_x_jng=393637 | 0x80000000,
+    video_x_m4v=393638,
+    video_x_matroska=393639,
+    video_x_mng=393640,
+    video_x_motion_jpeg=393641,
+    video_x_ms_asf=393642,
+    video_x_msvideo=393643,
+    video_x_qtc=393644,
+    video_x_sgi_movie=393645,
+    x_epoc_x_sisx_app=721326,
 };
 char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
 case application_arj: return "application/arj";
@@ -866,6 +867,7 @@ case video_x_qtc: return "video/x-qtc";
 case video_x_sgi_movie: return "video/x-sgi-movie";
 case x_epoc_x_sisx_app: return "x-epoc/x-sisx-app";
 case application_x_zstd_dictionary: return "application/x-zstd-dictionary";
+case application_vnd_ms_outlook: return "application/vnd.ms-outlook";
 default: return NULL;}}
 GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -1816,5 +1818,6 @@ g_hash_table_insert(mime_table, "video/x-qtc", (gpointer)video_x_qtc);
 g_hash_table_insert(mime_table, "video/x-sgi-movie", (gpointer)video_x_sgi_movie);
 g_hash_table_insert(mime_table, "x-epoc/x-sisx-app", (gpointer)x_epoc_x_sisx_app);
 g_hash_table_insert(mime_table, "application/x-zstd-dictionary", (gpointer)application_x_zstd_dictionary);
+g_hash_table_insert(mime_table, "application/vnd.ms-outlook", (gpointer)application_vnd_ms_outlook);
 return mime_table;}
 #endif

src/parsing/parse.c

@@ -49,7 +49,6 @@ void parse(void *arg) {
         return;
     }
 
-
     doc.filepath = job->filepath;
     doc.ext = (short) job->ext;
     doc.base = (short) job->base;
@@ -61,7 +60,7 @@ void parse(void *arg) {
     doc.mtime = job->vfile.info.st_mtim.tv_sec;
 
     uuid_generate(doc.uuid);
-    char *buf[PARSE_BUF_SIZE];
+    char *buf[MAGIC_BUF_SIZE];
 
     if (LogCtx.very_verbose) {
         char uuid_str[UUID_STR_LEN];
@@ -79,7 +78,7 @@ void parse(void *arg) {
 
     if (doc.mime == 0 && !ScanCtx.fast) {
         // Get mime type with libmagic
-        bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
+        bytes_read = job->vfile.read(&job->vfile, buf, MAGIC_BUF_SIZE);
         if (bytes_read < 0) {
 
             if (job->vfile.is_fs_file) {

src/parsing/parse.h

@@ -3,7 +3,7 @@
 
 #include "../sist.h"
 
-#define PARSE_BUF_SIZE 4096
+#define MAGIC_BUF_SIZE 4096 * 6
 
 int fs_read(struct vfile *f, void *buf, size_t size);
 void fs_close(struct vfile *f);

src/static/js/search.js

@@ -465,6 +465,11 @@ function search(after = null) {
             lastDoc = hits[hits.length - 1];
         }
 
+        hits.forEach(hit => {
+            hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
+            hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+        });
+
         if (!after) {
             preload.remove();
             searchResults.appendChild(makeStatsCard(searchResult));

src/static/js/util.js

@@ -64,3 +64,25 @@ function lum(c) {
 
     return 0.2126 * r + 0.7152 * g + 0.0722 * b;
 }
+
+function strUnescape(str) {
+    let result = "";
+
+    for (let i = 0; i < str.length; i++) {
+        const c = str[i];
+        const next = str[i+1];
+
+        if (c === ']') {
+            if (next === ']') {
+                result += c;
+                i += 1;
+            } else {
+                result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
+                i += 2;
+            }
+        } else {
+            result += c;
+        }
+    }
+    return result;
+}

src/static/search.html

@@ -11,7 +11,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.2.1</span>
+    <span class="badge badge-pill version">2.3.0</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <button style="margin-left: auto" class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings</button>
     <a id="theme" class="btn" title="Toggle theme" href="/">Theme</a>

src/util.c

@@ -123,4 +123,92 @@ const char *find_file_in_paths(const char *paths[], const char *filename) {
     return NULL;
 }
 
+#define ESCAPE_CHAR ']'
 
+void str_escape(char *dst, const char *str) {
+    const size_t len = strlen(str);
+
+    char buf[16384];
+    memset(buf + len, 0, 8);
+    strcpy(buf, str);
+
+    char *cur = dst;
+    const char *ptr = buf;
+    const char *oldPtr = ptr;
+
+    utf8_int32_t c;
+    char tmp[16];
+
+    do {
+        ptr = (char *) utf8codepoint(ptr, &c);
+        *(int *) tmp = 0x00000000;
+        size_t code_len = (ptr - oldPtr);
+        memcpy(tmp, oldPtr, code_len);
+        oldPtr = ptr;
+
+        if (!utf8_validchr2(tmp)) {
+            for (int i = 0; i < code_len; i++) {
+                if (tmp[i] == 0) {
+                    break;
+                }
+
+                cur += sprintf(cur, "%c%02X", ESCAPE_CHAR, (unsigned char)tmp[i]);
+            }
+            continue;
+        }
+
+        if (c == ESCAPE_CHAR) {
+            *cur++ = ESCAPE_CHAR;
+            *cur++ = ESCAPE_CHAR;
+            continue;
+        }
+
+        if (((utf8_int32_t) 0xffffff80 & c) == 0) {
+            *(cur++) = (char) c;
+        } else if (((utf8_int32_t) 0xfffff800 & c) == 0) {
+            *(cur++) = 0xc0 | (char) (c >> 6);
+            *(cur++) = 0x80 | (char) (c & 0x3f);
+        } else if (((utf8_int32_t) 0xffff0000 & c) == 0) {
+            *(cur++) = 0xe0 | (char) (c >> 12);
+            *(cur++) = 0x80 | (char) ((c >> 6) & 0x3f);
+            *(cur++) = 0x80 | (char) (c & 0x3f);
+        } else {
+            *(cur++) = 0xf0 | (char) (c >> 18);
+            *(cur++) = 0x80 | (char) ((c >> 12) & 0x3f);
+            *(cur++) = 0x80 | (char) ((c >> 6) & 0x3f);
+            *(cur++) = 0x80 | (char) (c & 0x3f);
+        }
+
+    } while (*ptr != '\0');
+
+    *cur = '\0';
+}
+
+void str_unescape(char *dst, const char *str) {
+    char *cur = dst;
+    const char *ptr = str;
+
+    char tmp[3];
+    tmp[2] = '\0';
+
+    while (*ptr != 0) {
+        char c = *ptr++;
+
+        if (c == ESCAPE_CHAR) {
+            char next = *ptr;
+
+            if (next == ESCAPE_CHAR) {
+                *cur++ = (char)c;
+                ptr += 1;
+            } else {
+                tmp[0] = *(ptr);
+                tmp[1] = *(ptr + 1);
+                *cur++ = (char)strtol(tmp, NULL, 16);
+                ptr += 2;
+            }
+        } else {
+            *cur++ = c;
+        }
+    }
+    *cur = '\0';
+}

src/util.h

@@ -43,4 +43,9 @@ static int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode
 
 const char *find_file_in_paths(const char **paths, const char *filename);
 
+
+void str_escape(char *dst, const char *str);
+
+void str_unescape(char *dst, const char *str);
+
 #endif

src/web/serve.c

@@ -174,12 +174,19 @@ int serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
 
     const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
     const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
+
+    char name_unescaped[PATH_MAX * 3];
+    str_unescape(name_unescaped, name);
+
+    char path_unescaped[PATH_MAX * 3];
+    str_unescape(path_unescaped, path);
+
     const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
 
     char url[8196];
     snprintf(url, sizeof(url),
              "%s%s/%s%s%s",
-             idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext);
+             idx->desc.rewrite_url, path_unescaped, name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
 
     dyn_buffer_t encoded = url_escape(url);
     mg_http_send_redirect(
@@ -198,10 +205,16 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
     const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
     const char *mime = cJSON_GetObjectItem(json, "mime")->valuestring;
 
+    char name_unescaped[PATH_MAX * 3];
+    str_unescape(name_unescaped, name);
+
+    char path_unescaped[PATH_MAX * 3];
+    str_unescape(path_unescaped, path);
+
     char full_path[PATH_MAX];
     snprintf(full_path, PATH_MAX, "%s%s%s%s%s%s",
-             idx->desc.root, path, strlen(path) == 0 ? "" : "/",
-             name, strlen(ext) == 0 ? "" : ".", ext);
+             idx->desc.root, path_unescaped, strlen(path_unescaped) == 0 ? "" : "/",
+             name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
 
     LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path)
 

third-party/libscan

@@ -1 +1 @@
-Subproject commit f363659936bab1d98b92a515610eea5337c0ea39
+Subproject commit fe6232ed8222ee33178d6c50ff2c5ea6312e7dee