From 6075c21a3ac5eeebda6ee4b8f326b1c9cb1a8170 Mon Sep 17 00:00:00 2001 From: Yatao Li Date: Sun, 23 Jan 2022 04:41:13 +0800 Subject: [PATCH] do not throttle writer/index thread pools --- src/cli.c | 4 ++-- src/cli.h | 2 +- src/main.c | 10 +++++----- src/tpool.c | 6 ++++-- src/tpool.h | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/cli.c b/src/cli.c index afd0163..665d109 100644 --- a/src/cli.c +++ b/src/cli.c @@ -257,8 +257,8 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { args->max_memory_buffer = DEFAULT_MAX_MEM_BUFFER; } - if (args->throttle_memory_threshold <= 0) { - args->throttle_memory_threshold = DEFAULT_THROTTLE_MEMORY_THRESHOLD; + if (args->scan_mem_limit <= 0) { + args->scan_mem_limit = DEFAULT_THROTTLE_MEMORY_THRESHOLD; } if (args->list_path != NULL) { diff --git a/src/cli.h b/src/cli.h index 1cd92ba..b74fe96 100644 --- a/src/cli.h +++ b/src/cli.h @@ -10,7 +10,7 @@ typedef struct scan_args { int size; int content_size; int threads; - int throttle_memory_threshold; + int scan_mem_limit; char *incremental; char *output; char *rewrite_url; diff --git a/src/main.c b/src/main.c index 669037f..b7f3134 100644 --- a/src/main.c +++ b/src/main.c @@ -253,7 +253,7 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.threads = args->threads; ScanCtx.depth = args->depth; - ScanCtx.mem_limit = args->throttle_memory_threshold * 1024 * 1024; + ScanCtx.mem_limit = args->scan_mem_limit * 1024 * 1024; strncpy(ScanCtx.index.path, args->output, sizeof(ScanCtx.index.path)); strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name)); @@ -383,10 +383,10 @@ void sist2_scan(scan_args_t *args) { load_incremental_index(args); } - ScanCtx.pool = tpool_create(args->threads, thread_cleanup, TRUE, TRUE); + ScanCtx.pool = tpool_create(ScanCtx.threads, thread_cleanup, TRUE, TRUE, ScanCtx.mem_limit); tpool_start(ScanCtx.pool); - ScanCtx.writer_pool = tpool_create(1, writer_cleanup, TRUE, FALSE); + ScanCtx.writer_pool = tpool_create(1, writer_cleanup, TRUE, FALSE, 0); tpool_start(ScanCtx.writer_pool); if (args->list_path) { @@ -467,7 +467,7 @@ void sist2_index(index_args_t *args) { f = index_json; } - IndexCtx.pool = tpool_create(args->threads, elastic_cleanup, FALSE, args->print == 0); + IndexCtx.pool = tpool_create(args->threads, elastic_cleanup, FALSE, args->print == 0, 0); tpool_start(IndexCtx.pool); READ_INDICES(file_path, args->index_path, { @@ -587,7 +587,7 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Scan options"), OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"), - OPT_STRING(0, "mem-throttle", &scan_args->throttle_memory_threshold, "Total memory threshold in MB for scan throttling. DEFAULT=0"), + OPT_STRING(0, "mem-throttle", &scan_args->scan_mem_limit, "Total memory threshold in MB for scan throttling. DEFAULT=0"), OPT_FLOAT('q', "quality", &scan_args->quality, "Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=3"), OPT_INTEGER(0, "size", &scan_args->size, diff --git a/src/tpool.c b/src/tpool.c index a10f2e8..3419f85 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -29,6 +29,7 @@ typedef struct tpool { int done_cnt; int busy_cnt; int throttle_stuck_cnt; + size_t mem_limit; int free_arg; int stop; @@ -171,7 +172,7 @@ static void *tpool_worker(void *arg) { if (work != NULL) { stuck_notified = 0; - while(!pool->stop && ScanCtx.mem_limit > 0 && _get_total_mem() >= ScanCtx.mem_limit) { + while(!pool->stop && pool->mem_limit > 0 && _get_total_mem() >= pool->mem_limit) { if (!stuck_notified && throttle_ms >= 90000) { // notify the pool that this thread is stuck. pthread_mutex_lock(&(pool->work_mutex)); @@ -299,7 +300,7 @@ void tpool_destroy(tpool_t *pool) { * Create a thread pool * @param thread_cnt Worker threads count */ -tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int free_arg, int print_progress) { +tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int free_arg, int print_progress, size_t mem_limit) { tpool_t *pool = malloc(sizeof(tpool_t)); pool->thread_cnt = thread_cnt; @@ -307,6 +308,7 @@ tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int free_arg, int pri pool->done_cnt = 0; pool->busy_cnt = 0; pool->throttle_stuck_cnt = 0; + pool->mem_limit = mem_limit; pool->stop = FALSE; pool->free_arg = free_arg; pool->cleanup_func = cleanup_func; diff --git a/src/tpool.h b/src/tpool.h index 6c84a93..71d202f 100644 --- a/src/tpool.h +++ b/src/tpool.h @@ -8,7 +8,7 @@ typedef struct tpool tpool_t; typedef void (*thread_func_t)(void *arg); -tpool_t *tpool_create(int num, void (*cleanup_func)(), int free_arg, int print_progress); +tpool_t *tpool_create(int num, void (*cleanup_func)(), int free_arg, int print_progress, size_t mem_limit); void tpool_start(tpool_t *pool); void tpool_destroy(tpool_t *pool);