simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-19 10:16:42 +00:00
Code Issues Projects Releases Wiki Activity

Add basic JSON/NDJSON support

Browse Source
This commit is contained in:
simon987 2021-09-06 21:27:17 -04:00
parent bbf1aca936
commit 43470e9ce6
6 changed files with 438 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scripts/mime.csv
View File

@ -22,6 +22,7 @@ application/java-archive, jar
application/java, class application/java, class
application/javascript, application/javascript,
application/json, json application/json, json
application/ndjson, jsonl|ndjson
application/marc, mrc application/marc, mrc
application/mbedlet, mbd application/mbedlet, mbd
application/mime, aps application/mime, aps

1 application/arj arj
22 application/java class
23 application/javascript
24 application/json json
25 application/ndjson jsonl|ndjson
26 application/marc mrc
27 application/mbedlet mbd
28 application/mime aps

2
src/ctx.h
View File

@ -15,6 +15,7 @@
#include "libscan/raw/raw.h" #include "libscan/raw/raw.h"
#include "libscan/msdoc/msdoc.h" #include "libscan/msdoc/msdoc.h"
#include "libscan/wpd/wpd.h" #include "libscan/wpd/wpd.h"
#include "libscan/json/json.h"
#include "src/io/store.h" #include "src/io/store.h"
#include <glib.h> #include <glib.h>
@ -62,6 +63,7 @@ typedef struct {
scan_raw_ctx_t raw_ctx; scan_raw_ctx_t raw_ctx;
scan_msdoc_ctx_t msdoc_ctx; scan_msdoc_ctx_t msdoc_ctx;
scan_wpd_ctx_t wpd_ctx; scan_wpd_ctx_t wpd_ctx;
scan_json_ctx_t json_ctx;
} ScanCtx_t; } ScanCtx_t;
typedef struct { typedef struct {

8
src/main.c
View File

@ -259,10 +259,18 @@ void initialize_scan_context(scan_args_t *args) {
ScanCtx.raw_ctx.logf = _logf; ScanCtx.raw_ctx.logf = _logf;
ScanCtx.raw_ctx.store = _store; ScanCtx.raw_ctx.store = _store;
// Wpd
ScanCtx.wpd_ctx.content_size = args->content_size; ScanCtx.wpd_ctx.content_size = args->content_size;
ScanCtx.wpd_ctx.log = _log; ScanCtx.wpd_ctx.log = _log;
ScanCtx.wpd_ctx.logf = _logf; ScanCtx.wpd_ctx.logf = _logf;
ScanCtx.wpd_ctx.wpd_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/wordperfect"); ScanCtx.wpd_ctx.wpd_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/wordperfect");
// Json
ScanCtx.json_ctx.content_size = args->content_size;
ScanCtx.json_ctx.log = _log;
ScanCtx.json_ctx.logf = _logf;
ScanCtx.json_ctx.json_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/json");
ScanCtx.json_ctx.ndjson_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/ndjson");
} }

839
src/parsing/mime_generated.c vendored
View File

@ -35,425 +35,426 @@ enum mime {
application_mime=655387, application_mime=655387,
application_mspowerpoint=655388, application_mspowerpoint=655388,
application_msword=655389, application_msword=655389,
application_netmc=655390, application_ndjson=655390,
application_octet_stream=655391, application_netmc=655391,
application_oda=655392, application_octet_stream=655392,
application_ogg=655393, application_oda=655393,
application_pdf=655394 | 0x40000000, application_ogg=655394,
application_pgp_keys=655395, application_pdf=655395 | 0x40000000,
application_pgp_signature=655396, application_pgp_keys=655396,
application_pkcs7_signature=655397, application_pgp_signature=655397,
application_pkix_cert=655398, application_pkcs7_signature=655398,
application_postscript=655399, application_pkix_cert=655399,
application_pro_eng=655400, application_postscript=655400,
application_ringing_tones=655401, application_pro_eng=655401,
application_smil=655402, application_ringing_tones=655402,
application_solids=655403, application_smil=655403,
application_sounder=655404, application_solids=655404,
application_step=655405, application_sounder=655405,
application_streamingmedia=655406, application_step=655406,
application_vda=655407, application_streamingmedia=655407,
application_vnd_amazon_mobi8_ebook=655408 | 0x02000000, application_vda=655408,
application_vnd_coffeescript=655409, application_vnd_amazon_mobi8_ebook=655409 | 0x02000000,
application_vnd_fdf=655410, application_vnd_coffeescript=655410,
application_vnd_font_fontforge_sfd=655411, application_vnd_fdf=655411,
application_vnd_hp_hpgl=655412, application_vnd_font_fontforge_sfd=655412,
application_vnd_iccprofile=655413, application_vnd_hp_hpgl=655413,
application_vnd_lotus_1_2_3=655414, application_vnd_iccprofile=655414,
application_vnd_ms_cab_compressed=655415, application_vnd_lotus_1_2_3=655415,
application_vnd_ms_excel=655416, application_vnd_ms_cab_compressed=655416,
application_vnd_ms_fontobject=655417, application_vnd_ms_excel=655417,
application_vnd_ms_opentype=655418 | 0x20000000, application_vnd_ms_fontobject=655418,
application_vnd_ms_outlook=655419, application_vnd_ms_opentype=655419 | 0x20000000,
application_vnd_ms_pki_certstore=655420, application_vnd_ms_outlook=655420,
application_vnd_ms_pki_pko=655421, application_vnd_ms_pki_certstore=655421,
application_vnd_ms_pki_seccat=655422, application_vnd_ms_pki_pko=655422,
application_vnd_ms_powerpoint=655423, application_vnd_ms_pki_seccat=655423,
application_vnd_ms_project=655424, application_vnd_ms_powerpoint=655424,
application_vnd_oasis_opendocument_base=655425, application_vnd_ms_project=655425,
application_vnd_oasis_opendocument_formula=655426, application_vnd_oasis_opendocument_base=655426,
application_vnd_oasis_opendocument_graphics=655427, application_vnd_oasis_opendocument_formula=655427,
application_vnd_oasis_opendocument_presentation=655428, application_vnd_oasis_opendocument_graphics=655428,
application_vnd_oasis_opendocument_spreadsheet=655429, application_vnd_oasis_opendocument_presentation=655429,
application_vnd_oasis_opendocument_text=655430, application_vnd_oasis_opendocument_spreadsheet=655430,
application_vnd_openxmlformats_officedocument_presentationml_presentation=655431 | 0x04000000, application_vnd_oasis_opendocument_text=655431,
application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655432 | 0x04000000, application_vnd_openxmlformats_officedocument_presentationml_presentation=655432 | 0x04000000,
application_vnd_openxmlformats_officedocument_wordprocessingml_document=655433 | 0x04000000, application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655433 | 0x04000000,
application_vnd_symbian_install=655434, application_vnd_openxmlformats_officedocument_wordprocessingml_document=655434 | 0x04000000,
application_vnd_tcpdump_pcap=655435, application_vnd_symbian_install=655435,
application_vnd_wap_wmlc=655436, application_vnd_tcpdump_pcap=655436,
application_vnd_wap_wmlscriptc=655437, application_vnd_wap_wmlc=655437,
application_vnd_xara=655438, application_vnd_wap_wmlscriptc=655438,
application_vocaltec_media_desc=655439, application_vnd_xara=655439,
application_vocaltec_media_file=655440, application_vocaltec_media_desc=655440,
application_warc=655441, application_vocaltec_media_file=655441,
application_winhelp=655442, application_warc=655442,
application_wordperfect=655443, application_winhelp=655443,
application_x_123=655444, application_wordperfect=655444,
application_x_7z_compressed=655445 | 0x10000000, application_x_123=655445,
application_x_aim=655446, application_x_7z_compressed=655446 | 0x10000000,
application_x_apple_diskimage=655447, application_x_aim=655447,
application_x_arc=655448 | 0x10000000, application_x_apple_diskimage=655448,
application_x_archive=655449, application_x_arc=655449 | 0x10000000,
application_x_atari_7800_rom=655450, application_x_archive=655450,
application_x_authorware_bin=655451, application_x_atari_7800_rom=655451,
application_x_authorware_map=655452, application_x_authorware_bin=655452,
application_x_authorware_seg=655453, application_x_authorware_map=655453,
application_x_avira_qua=655454, application_x_authorware_seg=655454,
application_x_bcpio=655455, application_x_avira_qua=655455,
application_x_bittorrent=655456, application_x_bcpio=655456,
application_x_bsh=655457, application_x_bittorrent=655457,
application_x_bytecode_python=655458, application_x_bsh=655458,
application_x_bzip=655459, application_x_bytecode_python=655459,
application_x_bzip2=655460 | 0x08000000, application_x_bzip=655460,
application_x_cbr=655461, application_x_bzip2=655461 | 0x08000000,
application_x_cbz=655462, application_x_cbr=655462,
application_x_cdlink=655463, application_x_cbz=655463,
application_x_chat=655464, application_x_cdlink=655464,
application_x_chrome_extension=655465, application_x_chat=655465,
application_x_cocoa=655466, application_x_chrome_extension=655466,
application_x_conference=655467, application_x_cocoa=655467,
application_x_coredump=655468, application_x_conference=655468,
application_x_cpio=655469, application_x_coredump=655469,
application_x_dbf=655470, application_x_cpio=655470,
application_x_dbt=655471, application_x_dbf=655471,
application_x_debian_package=655472, application_x_dbt=655472,
application_x_deepv=655473, application_x_debian_package=655473,
application_x_director=655474, application_x_deepv=655474,
application_x_dmp=655475, application_x_director=655475,
application_x_dosdriver=655476, application_x_dmp=655476,
application_x_dosexec=655477, application_x_dosdriver=655477,
application_x_dvi=655478, application_x_dosexec=655478,
application_x_elc=655479, application_x_dvi=655479,
application_x_elc=655480,
application_x_empty=1, application_x_empty=1,
application_x_envoy=655480, application_x_envoy=655481,
application_x_esrehber=655481, application_x_esrehber=655482,
application_x_excel=655482, application_x_excel=655483,
application_x_executable=655483, application_x_executable=655484,
application_x_font_gdos=655484, application_x_font_gdos=655485,
application_x_font_pf2=655485, application_x_font_pf2=655486,
application_x_font_pfm=655486, application_x_font_pfm=655487,
application_x_font_sfn=655487, application_x_font_sfn=655488,
application_x_font_ttf=655488 | 0x20000000, application_x_font_ttf=655489 | 0x20000000,
application_x_fptapplication_x_dbt=655489, application_x_fptapplication_x_dbt=655490,
application_x_freelance=655490, application_x_freelance=655491,
application_x_gamecube_rom=655491, application_x_gamecube_rom=655492,
application_x_gdbm=655492, application_x_gdbm=655493,
application_x_gettext_translation=655493, application_x_gettext_translation=655494,
application_x_git=655494, application_x_git=655495,
application_x_gsp=655495, application_x_gsp=655496,
application_x_gss=655496, application_x_gss=655497,
application_x_gtar=655497, application_x_gtar=655498,
application_x_gzip=655498, application_x_gzip=655499,
application_x_hdf=655499, application_x_hdf=655500,
application_x_helpfile=655500, application_x_helpfile=655501,
application_x_httpd_imap=655501, application_x_httpd_imap=655502,
application_x_ima=655502, application_x_ima=655503,
application_x_innosetup=655503, application_x_innosetup=655504,
application_x_internett_signup=655504, application_x_internett_signup=655505,
application_x_inventor=655505, application_x_inventor=655506,
application_x_ip2=655506, application_x_ip2=655507,
application_x_java_applet=655507, application_x_java_applet=655508,
application_x_java_commerce=655508, application_x_java_commerce=655509,
application_x_java_image=655509, application_x_java_image=655510,
application_x_java_jmod=655510, application_x_java_jmod=655511,
application_x_java_keystore=655511, application_x_java_keystore=655512,
application_x_kdelnk=655512, application_x_kdelnk=655513,
application_x_koan=655513, application_x_koan=655514,
application_x_latex=655514, application_x_latex=655515,
application_x_livescreen=655515, application_x_livescreen=655516,
application_x_lotus=655516, application_x_lotus=655517,
application_x_lz4=655517 | 0x08000000, application_x_lz4=655518 | 0x08000000,
application_x_lz4_json=655518, application_x_lz4_json=655519,
application_x_lzh=655519, application_x_lzh=655520,
application_x_lzh_compressed=655520, application_x_lzh_compressed=655521,
application_x_lzip=655521 | 0x08000000, application_x_lzip=655522 | 0x08000000,
application_x_lzma=655522 | 0x08000000, application_x_lzma=655523 | 0x08000000,
application_x_lzop=655523 | 0x08000000, application_x_lzop=655524 | 0x08000000,
application_x_lzx=655524, application_x_lzx=655525,
application_x_mach_binary=655525, application_x_mach_binary=655526,
application_x_mach_executable=655526, application_x_mach_executable=655527,
application_x_magic_cap_package_1_0=655527, application_x_magic_cap_package_1_0=655528,
application_x_mathcad=655528, application_x_mathcad=655529,
application_x_maxis_dbpf=655529, application_x_maxis_dbpf=655530,
application_x_meme=655530, application_x_meme=655531,
application_x_midi=655531, application_x_midi=655532,
application_x_mif=655532, application_x_mif=655533,
application_x_mix_transfer=655533, application_x_mix_transfer=655534,
application_x_mobipocket_ebook=655534 | 0x02000000, application_x_mobipocket_ebook=655535 | 0x02000000,
application_x_ms_compress_szdd=655535, application_x_ms_compress_szdd=655536,
application_x_ms_pdb=655536, application_x_ms_pdb=655537,
application_x_ms_reader=655537, application_x_ms_reader=655538,
application_x_msaccess=655538, application_x_msaccess=655539,
application_x_n64_rom=655539, application_x_n64_rom=655540,
application_x_navi_animation=655540, application_x_navi_animation=655541,
application_x_navidoc=655541, application_x_navidoc=655542,
application_x_navimap=655542, application_x_navimap=655543,
application_x_navistyle=655543, application_x_navistyle=655544,
application_x_nes_rom=655544, application_x_nes_rom=655545,
application_x_netcdf=655545, application_x_netcdf=655546,
application_x_newton_compatible_pkg=655546, application_x_newton_compatible_pkg=655547,
application_x_nintendo_ds_rom=655547, application_x_nintendo_ds_rom=655548,
application_x_object=655548, application_x_object=655549,
application_x_omc=655549, application_x_omc=655550,
application_x_omcdatamaker=655550, application_x_omcdatamaker=655551,
application_x_omcregerator=655551, application_x_omcregerator=655552,
application_x_pagemaker=655552, application_x_pagemaker=655553,
application_x_pcl=655553, application_x_pcl=655554,
application_x_pgp_keyring=655554, application_x_pgp_keyring=655555,
application_x_pixclscript=655555, application_x_pixclscript=655556,
application_x_pkcs7_certreqresp=655556, application_x_pkcs7_certreqresp=655557,
application_x_pkcs7_signature=655557, application_x_pkcs7_signature=655558,
application_x_project=655558, application_x_project=655559,
application_x_qpro=655559, application_x_qpro=655560,
application_x_rar=655560 | 0x10000000, application_x_rar=655561 | 0x10000000,
application_x_rpm=655561, application_x_rpm=655562,
application_x_sdp=655562, application_x_sdp=655563,
application_x_sea=655563, application_x_sea=655564,
application_x_seelogo=655564, application_x_seelogo=655565,
application_x_setupscript=655565, application_x_setupscript=655566,
application_x_shar=655566, application_x_shar=655567,
application_x_sharedlib=655567, application_x_sharedlib=655568,
application_x_shockwave_flash=655568, application_x_shockwave_flash=655569,
application_x_snappy_framed=655569, application_x_snappy_framed=655570,
application_x_sprite=655570, application_x_sprite=655571,
application_x_sqlite3=655571, application_x_sqlite3=655572,
application_x_stargallery_thm=655572, application_x_stargallery_thm=655573,
application_x_stuffit=655573, application_x_stuffit=655574,
application_x_sv4cpio=655574, application_x_sv4cpio=655575,
application_x_sv4crc=655575, application_x_sv4crc=655576,
application_x_tar=655576 | 0x10000000, application_x_tar=655577 | 0x10000000,
application_x_tbook=655577, application_x_tbook=655578,
application_x_terminfo=655578, application_x_terminfo=655579,
application_x_terminfo2=655579, application_x_terminfo2=655580,
application_x_tex_tfm=655580, application_x_tex_tfm=655581,
application_x_texinfo=655581, application_x_texinfo=655582,
application_x_ustar=655582, application_x_ustar=655583,
application_x_visio=655583, application_x_visio=655584,
application_x_vnd_audioexplosion_mzz=655584, application_x_vnd_audioexplosion_mzz=655585,
application_x_vnd_ls_xpix=655585, application_x_vnd_ls_xpix=655586,
application_x_vrml=655586, application_x_vrml=655587,
application_x_wais_source=655587, application_x_wais_source=655588,
application_x_wine_extension_ini=655588, application_x_wine_extension_ini=655589,
application_x_wintalk=655589, application_x_wintalk=655590,
application_x_world=655590, application_x_world=655591,
application_x_wri=655591, application_x_wri=655592,
application_x_x509_ca_cert=655592, application_x_x509_ca_cert=655593,
application_x_xz=655593 | 0x08000000, application_x_xz=655594 | 0x08000000,
application_x_zip=655594, application_x_zip=655595,
application_x_zstd=655595 | 0x08000000, application_x_zstd=655596 | 0x08000000,
application_x_zstd_dictionary=655596, application_x_zstd_dictionary=655597,
application_xml=655597, application_xml=655598,
application_zip=655598 | 0x10000000, application_zip=655599 | 0x10000000,
application_zlib=655599, application_zlib=655600,
audio_basic=458992 | 0x80000000, audio_basic=458993 | 0x80000000,
audio_it=458993, audio_it=458994,
audio_make=458994, audio_make=458995,
audio_mid=458995, audio_mid=458996,
audio_midi=458996, audio_midi=458997,
audio_mp4=458997, audio_mp4=458998,
audio_mpeg=458998, audio_mpeg=458999,
audio_ogg=458999, audio_ogg=459000,
audio_s3m=459000, audio_s3m=459001,
audio_tsp_audio=459001, audio_tsp_audio=459002,
audio_tsplayer=459002, audio_tsplayer=459003,
audio_vnd_qcelp=459003, audio_vnd_qcelp=459004,
audio_voxware=459004, audio_voxware=459005,
audio_x_aiff=459005, audio_x_aiff=459006,
audio_x_flac=459006, audio_x_flac=459007,
audio_x_gsm=459007, audio_x_gsm=459008,
audio_x_hx_aac_adts=459008, audio_x_hx_aac_adts=459009,
audio_x_jam=459009, audio_x_jam=459010,
audio_x_liveaudio=459010, audio_x_liveaudio=459011,
audio_x_m4a=459011, audio_x_m4a=459012,
audio_x_midi=459012, audio_x_midi=459013,
audio_x_mod=459013, audio_x_mod=459014,
audio_x_mp4a_latm=459014, audio_x_mp4a_latm=459015,
audio_x_mpeg_3=459015, audio_x_mpeg_3=459016,
audio_x_mpequrl=459016, audio_x_mpequrl=459017,
audio_x_nspaudio=459017, audio_x_nspaudio=459018,
audio_x_pn_realaudio=459018, audio_x_pn_realaudio=459019,
audio_x_psid=459019, audio_x_psid=459020,
audio_x_realaudio=459020, audio_x_realaudio=459021,
audio_x_s3m=459021, audio_x_s3m=459022,
audio_x_twinvq=459022, audio_x_twinvq=459023,
audio_x_twinvq_plugin=459023, audio_x_twinvq_plugin=459024,
audio_x_voc=459024, audio_x_voc=459025,
audio_x_wav=459025, audio_x_wav=459026,
audio_x_xbox_executable=459026 | 0x80000000, audio_x_xbox_executable=459027 | 0x80000000,
audio_x_xbox360_executable=459027 | 0x80000000, audio_x_xbox360_executable=459028 | 0x80000000,
audio_xm=459028, audio_xm=459029,
font_otf=327957 | 0x20000000, font_otf=327958 | 0x20000000,
font_sfnt=327958 | 0x20000000, font_sfnt=327959 | 0x20000000,
font_woff=327959 | 0x20000000, font_woff=327960 | 0x20000000,
font_woff2=327960 | 0x20000000, font_woff2=327961 | 0x20000000,
image_bmp=524569, image_bmp=524570,
image_cmu_raster=524570, image_cmu_raster=524571,
image_fif=524571, image_fif=524572,
image_florian=524572, image_florian=524573,
image_g3fax=524573, image_g3fax=524574,
image_gif=524574, image_gif=524575,
image_heic=524575, image_heic=524576,
image_ief=524576, image_ief=524577,
image_jpeg=524577, image_jpeg=524578,
image_jutvision=524578, image_jutvision=524579,
image_naplps=524579, image_naplps=524580,
image_pict=524580, image_pict=524581,
image_png=524581, image_png=524582,
image_svg=524582 | 0x80000000, image_svg=524583 | 0x80000000,
image_svg_xml=524583 | 0x80000000, image_svg_xml=524584 | 0x80000000,
image_tiff=524584, image_tiff=524585,
image_vnd_adobe_photoshop=524585 | 0x80000000, image_vnd_adobe_photoshop=524586 | 0x80000000,
image_vnd_djvu=524586 | 0x80000000, image_vnd_djvu=524587 | 0x80000000,
image_vnd_fpx=524587, image_vnd_fpx=524588,
image_vnd_microsoft_icon=524588, image_vnd_microsoft_icon=524589,
image_vnd_rn_realflash=524589, image_vnd_rn_realflash=524590,
image_vnd_rn_realpix=524590, image_vnd_rn_realpix=524591,
image_vnd_wap_wbmp=524591, image_vnd_wap_wbmp=524592,
image_vnd_xiff=524592, image_vnd_xiff=524593,
image_webp=524593, image_webp=524594,
image_wmf=524594, image_wmf=524595,
image_x_3ds=524595, image_x_3ds=524596,
image_x_adobe_dng=524596 | 0x00800000, image_x_adobe_dng=524597 | 0x00800000,
image_x_award_bioslogo=524597, image_x_award_bioslogo=524598,
image_x_canon_cr2=524598 | 0x00800000, image_x_canon_cr2=524599 | 0x00800000,
image_x_canon_crw=524599 | 0x00800000, image_x_canon_crw=524600 | 0x00800000,
image_x_cmu_raster=524600, image_x_cmu_raster=524601,
image_x_cur=524601, image_x_cur=524602,
image_x_dcraw=524602 | 0x00800000, image_x_dcraw=524603 | 0x00800000,
image_x_dwg=524603, image_x_dwg=524604,
image_x_eps=524604, image_x_eps=524605,
image_x_epson_erf=524605 | 0x00800000, image_x_epson_erf=524606 | 0x00800000,
image_x_exr=524606, image_x_exr=524607,
image_x_fuji_raf=524607 | 0x00800000, image_x_fuji_raf=524608 | 0x00800000,
image_x_gem=524608, image_x_gem=524609,
image_x_icns=524609, image_x_icns=524610,
image_x_icon=524610 | 0x80000000, image_x_icon=524611 | 0x80000000,
image_x_jg=524611, image_x_jg=524612,
image_x_jps=524612, image_x_jps=524613,
image_x_kodak_dcr=524613 | 0x00800000, image_x_kodak_dcr=524614 | 0x00800000,
image_x_kodak_k25=524614 | 0x00800000, image_x_kodak_k25=524615 | 0x00800000,
image_x_kodak_kdc=524615 | 0x00800000, image_x_kodak_kdc=524616 | 0x00800000,
image_x_minolta_mrw=524616 | 0x00800000, image_x_minolta_mrw=524617 | 0x00800000,
image_x_ms_bmp=524617, image_x_ms_bmp=524618,
image_x_niff=524618, image_x_niff=524619,
image_x_nikon_nef=524619 | 0x00800000, image_x_nikon_nef=524620 | 0x00800000,
image_x_olympus_orf=524620 | 0x00800000, image_x_olympus_orf=524621 | 0x00800000,
image_x_panasonic_raw=524621 | 0x00800000, image_x_panasonic_raw=524622 | 0x00800000,
image_x_pcx=524622, image_x_pcx=524623,
image_x_pentax_pef=524623 | 0x00800000, image_x_pentax_pef=524624 | 0x00800000,
image_x_pict=524624, image_x_pict=524625,
image_x_portable_bitmap=524625, image_x_portable_bitmap=524626,
image_x_portable_graymap=524626, image_x_portable_graymap=524627,
image_x_portable_pixmap=524627, image_x_portable_pixmap=524628,
image_x_quicktime=524628, image_x_quicktime=524629,
image_x_rgb=524629, image_x_rgb=524630,
image_x_sigma_x3f=524630 | 0x00800000, image_x_sigma_x3f=524631 | 0x00800000,
image_x_sony_arw=524631 | 0x00800000, image_x_sony_arw=524632 | 0x00800000,
image_x_sony_sr2=524632 | 0x00800000, image_x_sony_sr2=524633 | 0x00800000,
image_x_sony_srf=524633 | 0x00800000, image_x_sony_srf=524634 | 0x00800000,
image_x_tga=524634, image_x_tga=524635,
image_x_tiff=524635, image_x_tiff=524636,
image_x_win_bitmap=524636, image_x_win_bitmap=524637,
image_x_xcf=524637 | 0x80000000, image_x_xcf=524638 | 0x80000000,
image_x_xpixmap=524638 | 0x80000000, image_x_xpixmap=524639 | 0x80000000,
image_x_xwindowdump=524639, image_x_xwindowdump=524640,
message_news=196960, message_news=196961,
message_rfc822=196961, message_rfc822=196962,
model_vnd_dwf=65890, model_vnd_dwf=65891,
model_vnd_gdl=65891, model_vnd_gdl=65892,
model_vnd_gs_gdl=65892, model_vnd_gs_gdl=65893,
model_vrml=65893, model_vrml=65894,
model_x_pov=65894, model_x_pov=65895,
sist2_sidecar=2, sist2_sidecar=2,
text_PGP=590183, text_PGP=590184,
text_asp=590184, text_asp=590185,
text_css=590185, text_css=590186,
text_html=590186 | 0x01000000, text_html=590187 | 0x01000000,
text_javascript=590187, text_javascript=590188,
text_mcf=590188, text_mcf=590189,
text_pascal=590189, text_pascal=590190,
text_plain=590190, text_plain=590191,
text_richtext=590191, text_richtext=590192,
text_rtf=590192, text_rtf=590193,
text_scriplet=590193, text_scriplet=590194,
text_tab_separated_values=590194, text_tab_separated_values=590195,
text_troff=590195, text_troff=590196,
text_uri_list=590196, text_uri_list=590197,
text_vnd_abc=590197, text_vnd_abc=590198,
text_vnd_fmi_flexstor=590198, text_vnd_fmi_flexstor=590199,
text_vnd_wap_wml=590199, text_vnd_wap_wml=590200,
text_vnd_wap_wmlscript=590200, text_vnd_wap_wmlscript=590201,
text_webviewhtml=590201, text_webviewhtml=590202,
text_x_Algol68=590202, text_x_Algol68=590203,
text_x_asm=590203, text_x_asm=590204,
text_x_audiosoft_intra=590204, text_x_audiosoft_intra=590205,
text_x_awk=590205, text_x_awk=590206,
text_x_bcpl=590206, text_x_bcpl=590207,
text_x_c=590207, text_x_c=590208,
text_x_c__=590208, text_x_c__=590209,
text_x_component=590209, text_x_component=590210,
text_x_diff=590210, text_x_diff=590211,
text_x_fortran=590211, text_x_fortran=590212,
text_x_java=590212, text_x_java=590213,
text_x_la_asf=590213, text_x_la_asf=590214,
text_x_lisp=590214, text_x_lisp=590215,
text_x_m=590215, text_x_m=590216,
text_x_m4=590216, text_x_m4=590217,
text_x_makefile=590217, text_x_makefile=590218,
text_x_ms_regedit=590218, text_x_ms_regedit=590219,
text_x_msdos_batch=590219, text_x_msdos_batch=590220,
text_x_objective_c=590220, text_x_objective_c=590221,
text_x_pascal=590221, text_x_pascal=590222,
text_x_perl=590222, text_x_perl=590223,
text_x_php=590223, text_x_php=590224,
text_x_po=590224, text_x_po=590225,
text_x_python=590225, text_x_python=590226,
text_x_ruby=590226, text_x_ruby=590227,
text_x_sass=590227, text_x_sass=590228,
text_x_scss=590228, text_x_scss=590229,
text_x_server_parsed_html=590229, text_x_server_parsed_html=590230,
text_x_setext=590230, text_x_setext=590231,
text_x_sgml=590231 | 0x01000000, text_x_sgml=590232 | 0x01000000,
text_x_shellscript=590232, text_x_shellscript=590233,
text_x_speech=590233, text_x_speech=590234,
text_x_tcl=590234, text_x_tcl=590235,
text_x_tex=590235, text_x_tex=590236,
text_x_uil=590236, text_x_uil=590237,
text_x_uuencode=590237, text_x_uuencode=590238,
text_x_vcalendar=590238, text_x_vcalendar=590239,
text_x_vcard=590239, text_x_vcard=590240,
text_xml=590240 | 0x01000000, text_xml=590241 | 0x01000000,
video_MP2T=393633, video_MP2T=393634,
video_animaflex=393634, video_animaflex=393635,
video_avi=393635, video_avi=393636,
video_avs_video=393636, video_avs_video=393637,
video_mp4=393637, video_mp4=393638,
video_mpeg=393638, video_mpeg=393639,
video_quicktime=393639, video_quicktime=393640,
video_vdo=393640, video_vdo=393641,
video_vivo=393641, video_vivo=393642,
video_vnd_rn_realvideo=393642, video_vnd_rn_realvideo=393643,
video_vosaic=393643, video_vosaic=393644,
video_webm=393644, video_webm=393645,
video_x_amt_demorun=393645, video_x_amt_demorun=393646,
video_x_amt_showrun=393646, video_x_amt_showrun=393647,
video_x_atomic3d_feature=393647, video_x_atomic3d_feature=393648,
video_x_dl=393648, video_x_dl=393649,
video_x_dv=393649, video_x_dv=393650,
video_x_fli=393650, video_x_fli=393651,
video_x_flv=393651, video_x_flv=393652,
video_x_isvideo=393652, video_x_isvideo=393653,
video_x_jng=393653 | 0x80000000, video_x_jng=393654 | 0x80000000,
video_x_m4v=393654, video_x_m4v=393655,
video_x_matroska=393655, video_x_matroska=393656,
video_x_mng=393656, video_x_mng=393657,
video_x_motion_jpeg=393657, video_x_motion_jpeg=393658,
video_x_ms_asf=393658, video_x_ms_asf=393659,
video_x_msvideo=393659, video_x_msvideo=393660,
video_x_qtc=393660, video_x_qtc=393661,
video_x_sgi_movie=393661, video_x_sgi_movie=393662,
x_epoc_x_sisx_app=721342, x_epoc_x_sisx_app=721343,
}; };
char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) { char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
case application_arj: return "application/arj"; case application_arj: return "application/arj";
@ -480,6 +481,7 @@ case application_java_archive: return "application/java-archive";
case application_java: return "application/java"; case application_java: return "application/java";
case application_javascript: return "application/javascript"; case application_javascript: return "application/javascript";
case application_json: return "application/json"; case application_json: return "application/json";
case application_ndjson: return "application/ndjson";
case application_marc: return "application/marc"; case application_marc: return "application/marc";
case application_mbedlet: return "application/mbedlet"; case application_mbedlet: return "application/mbedlet";
case application_mime: return "application/mime"; case application_mime: return "application/mime";
@ -930,6 +932,8 @@ g_hash_table_insert(ext_table, "inf", (gpointer)application_inf);
g_hash_table_insert(ext_table, "jar", (gpointer)application_java_archive); g_hash_table_insert(ext_table, "jar", (gpointer)application_java_archive);
g_hash_table_insert(ext_table, "class", (gpointer)application_java); g_hash_table_insert(ext_table, "class", (gpointer)application_java);
g_hash_table_insert(ext_table, "json", (gpointer)application_json); g_hash_table_insert(ext_table, "json", (gpointer)application_json);
g_hash_table_insert(ext_table, "jsonl", (gpointer)application_ndjson);
g_hash_table_insert(ext_table, "ndjson", (gpointer)application_ndjson);
g_hash_table_insert(ext_table, "mrc", (gpointer)application_marc); g_hash_table_insert(ext_table, "mrc", (gpointer)application_marc);
g_hash_table_insert(ext_table, "mbd", (gpointer)application_mbedlet); g_hash_table_insert(ext_table, "mbd", (gpointer)application_mbedlet);
g_hash_table_insert(ext_table, "aps", (gpointer)application_mime); g_hash_table_insert(ext_table, "aps", (gpointer)application_mime);
@ -1474,6 +1478,7 @@ g_hash_table_insert(mime_table, "application/java-archive", (gpointer)applicatio
g_hash_table_insert(mime_table, "application/java", (gpointer)application_java); g_hash_table_insert(mime_table, "application/java", (gpointer)application_java);
g_hash_table_insert(mime_table, "application/javascript", (gpointer)application_javascript); g_hash_table_insert(mime_table, "application/javascript", (gpointer)application_javascript);
g_hash_table_insert(mime_table, "application/json", (gpointer)application_json); g_hash_table_insert(mime_table, "application/json", (gpointer)application_json);
g_hash_table_insert(mime_table, "application/ndjson", (gpointer)application_ndjson);
g_hash_table_insert(mime_table, "application/marc", (gpointer)application_marc); g_hash_table_insert(mime_table, "application/marc", (gpointer)application_marc);
g_hash_table_insert(mime_table, "application/mbedlet", (gpointer)application_mbedlet); g_hash_table_insert(mime_table, "application/mbedlet", (gpointer)application_mbedlet);
g_hash_table_insert(mime_table, "application/mime", (gpointer)application_mime); g_hash_table_insert(mime_table, "application/mime", (gpointer)application_mime);

6
src/parsing/parse.c
View File

@ -182,8 +182,10 @@ void parse(void *arg) {
return; return;
} else if (is_msdoc(&ScanCtx.msdoc_ctx, doc->mime)) { } else if (is_msdoc(&ScanCtx.msdoc_ctx, doc->mime)) {
parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc); parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc);
} else if (is_wpd(&ScanCtx.wpd_ctx, doc->mime)) { } else if (is_json(&ScanCtx.json_ctx, doc->mime)) {
parse_wpd(&ScanCtx.wpd_ctx, &job->vfile, doc); parse_json(&ScanCtx.json_ctx, &job->vfile, doc);
} else if (is_ndjson(&ScanCtx.json_ctx, doc->mime)) {
parse_ndjson(&ScanCtx.json_ctx, &job->vfile, doc);
} }
abort: abort:

2
src/parsing/parse.h
View File

@ -3,7 +3,7 @@
#include "../sist.h" #include "../sist.h"
#define MAGIC_BUF_SIZE 4096 * 6 #define MAGIC_BUF_SIZE (4096 * 6)
int fs_read(struct vfile *f, void *buf, size_t size); int fs_read(struct vfile *f, void *buf, size_t size);
void fs_close(struct vfile *f); void fs_close(struct vfile *f);