diff --git a/config.go b/config.go index 5e669d5..af84b34 100644 --- a/config.go +++ b/config.go @@ -12,6 +12,8 @@ import ( var config struct { ServerUrl string Token string + ServerTimeout time.Duration + Recheck time.Duration ChunkSize uint Retries int Workers int @@ -26,6 +28,8 @@ var config struct { const ( ConfServerUrl = "server.url" ConfToken = "server.token" + ConfServerTimeout = "server.timeout" + ConfRecheck = "server.recheck" ConfChunkSize = "server.upload_chunk" ConfTasks = "crawl.tasks" ConfRetries = "crawl.retries" @@ -46,6 +50,7 @@ func prepareConfig() { viper.SetDefault(ConfAllocStats, 0) viper.SetDefault(ConfVerbose, false) viper.SetDefault(ConfPrintHTTP, false) + viper.SetDefault(ConfRecheck, 3 * time.Second) viper.SetDefault(ConfChunkSize, "1 MB") } @@ -69,6 +74,10 @@ func readConfig() { configMissing(ConfToken) } + config.ServerTimeout = viper.GetDuration(ConfServerTimeout) + + config.Recheck = viper.GetDuration(ConfRecheck) + config.ChunkSize = viper.GetSizeInBytes(ConfChunkSize) if config.ChunkSize < 100 { configOOB(ConfChunkSize, config.ChunkSize) diff --git a/config.yml b/config.yml index 945a503..7b4385a 100644 --- a/config.yml +++ b/config.yml @@ -2,8 +2,19 @@ server: # Connection URL url: http://od-db.mine.terorie.com/api + # Server auth token token: + + # Request timeout + timeout: 60s + + # Recheck interval + # The crawler periodically asks the server + # for new jobs. Sets the minimum wait time + # between /task/get requests to the server. + recheck: 1s + # Upload chunk size # If the value is too high, the upload fails. upload_chunk: 1 MB diff --git a/main.go b/main.go index 16ae21d..1ca415d 100644 --- a/main.go +++ b/main.go @@ -59,7 +59,7 @@ func cmdBase(_ *cli.Context) error { inRemotes := make(chan *OD) go Schedule(forceCtx, inRemotes) - ticker := time.NewTicker(3 * time.Second) + ticker := time.NewTicker(config.Recheck) defer ticker.Stop() for { select { diff --git a/server.go b/server.go index 5c2c0c3..d469c75 100644 --- a/server.go +++ b/server.go @@ -17,7 +17,9 @@ const ( fileListChunkSize int64 = 5000000 // 5 mb ) -var serverClient = http.DefaultClient +var serverClient = http.Client { + Timeout: config.ServerTimeout, +} func FetchTask() (t *Task, err error) { res, err := serverClient.PostForm(