simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 02:16:43 +00:00
Code Issues Projects Releases Wiki Activity

Wait time control in config

Browse Source
This commit is contained in:
Richard Patel 2018-11-27 19:47:30 +01:00
parent b1bf59adef
commit e82768ff80
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
4 changed files with 21 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
config.go
View File

@ -33,7 +33,11 @@ const (
ConfToken = "server.token" ConfToken = "server.token"
ConfServerTimeout = "server.timeout" ConfServerTimeout = "server.timeout"
ConfRecheck = "server.recheck" ConfRecheck = "server.recheck"
ConfCooldown = "server.cooldown"
ConfChunkSize = "server.upload_chunk" ConfChunkSize = "server.upload_chunk"
ConfUploadRetries = "server.upload_retries"
ConfUploadRetryInterval = "server.upload_retry_interval"
ConfTasks = "crawl.tasks" ConfTasks = "crawl.tasks"
ConfRetries = "crawl.retries" ConfRetries = "crawl.retries"
ConfWorkers = "crawl.connections" ConfWorkers = "crawl.connections"
@ -41,6 +45,7 @@ const (
ConfDialTimeout = "crawl.dial_timeout" ConfDialTimeout = "crawl.dial_timeout"
ConfTimeout = "crawl.timeout" ConfTimeout = "crawl.timeout"
ConfJobBufferSize = "crawl.job_buffer" ConfJobBufferSize = "crawl.job_buffer"
ConfCrawlStats = "output.crawl_stats" ConfCrawlStats = "output.crawl_stats"
ConfAllocStats = "output.resource_stats" ConfAllocStats = "output.resource_stats"
ConfVerbose = "output.verbose" ConfVerbose = "output.verbose"
@ -54,7 +59,7 @@ func prepareConfig() {
viper.SetDefault(ConfTasks, 3) viper.SetDefault(ConfTasks, 3)
viper.SetDefault(ConfUserAgent, "") viper.SetDefault(ConfUserAgent, "")
viper.SetDefault(ConfDialTimeout, 10 * time.Second) viper.SetDefault(ConfDialTimeout, 10 * time.Second)
viper.SetDefault(ConfTimeout, 30 * time.Second) viper.SetDefault(ConfTimeout, 60 * time.Second)
viper.SetDefault(ConfJobBufferSize, 5000) viper.SetDefault(ConfJobBufferSize, 5000)
viper.SetDefault(ConfCrawlStats, 3 * time.Second) viper.SetDefault(ConfCrawlStats, 3 * time.Second)
viper.SetDefault(ConfAllocStats, 0) viper.SetDefault(ConfAllocStats, 0)
@ -62,7 +67,10 @@ func prepareConfig() {
viper.SetDefault(ConfPrintHTTP, false) viper.SetDefault(ConfPrintHTTP, false)
viper.SetDefault(ConfLogFile, "") viper.SetDefault(ConfLogFile, "")
viper.SetDefault(ConfRecheck, 3 * time.Second) viper.SetDefault(ConfRecheck, 3 * time.Second)
viper.SetDefault(ConfCooldown, 30 * time.Second)
viper.SetDefault(ConfChunkSize, "1 MB") viper.SetDefault(ConfChunkSize, "1 MB")
viper.SetDefault(ConfUploadRetries, 10)
viper.SetDefault(ConfUploadRetryInterval, 30 * time.Second)
} }
func readConfig() { func readConfig() {

7
config.yml
View File

@ -15,10 +15,17 @@ server:
# between /task/get requests to the server. # between /task/get requests to the server.
recheck: 1s recheck: 1s
# Time to wait after receiving an error
# from the server. Doesn't apply to uploads.
cooldown: 30s
# Upload chunk size # Upload chunk size
# If the value is too high, the upload fails. # If the value is too high, the upload fails.
upload_chunk: 1 MB upload_chunk: 1 MB
upload_retries: 10
upload_retry_interval: 30s
# Log output settings # Log output settings
output: output:
# Crawl statistics # Crawl statistics

4
main.go
View File

@ -84,7 +84,7 @@ func cmdBase(_ *cli.Context) error {
if err != nil { if err != nil {
logrus.WithError(err). logrus.WithError(err).
Error("Failed to get new task") Error("Failed to get new task")
time.Sleep(30 * time.Second) time.Sleep(viper.GetDuration(ConfCooldown))
continue continue
} }
if t == nil { if t == nil {
@ -111,7 +111,7 @@ func cmdBase(_ *cli.Context) error {
} else if err != nil { } else if err != nil {
logrus.WithError(err). logrus.WithError(err).
Error("Failed to get new task") Error("Failed to get new task")
time.Sleep(30 * time.Second) time.Sleep(viper.GetDuration(ConfCooldown))
continue continue
} }
ScheduleTask(inRemotes, t, &baseUri) ScheduleTask(inRemotes, t, &baseUri)

5
server.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"io" "io"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
@ -102,10 +103,10 @@ func uploadChunks(websiteId uint64, f *os.File) error {
multi.Close() multi.Close()
for retries := 0; retries < 10; retries++ { for retries := 0; retries < viper.GetInt(ConfUploadRetries); retries++ {
if retries > 0 { if retries > 0 {
// Error occurred, retry upload // Error occurred, retry upload
time.Sleep(30 * time.Second) time.Sleep(viper.GetDuration(ConfUploadRetryInterval))
} }
req, err := http.NewRequest( req, err := http.NewRequest(