simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 18:36:43 +00:00
Code Issues Projects Releases Wiki Activity

Change default config

Browse Source
This commit is contained in:
simon987 2019-04-04 20:49:10 -04:00
parent 860fa79327
commit 84c10e1981
2 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config.go
View File

@ -67,15 +67,15 @@ func prepareConfig() {
pf.StringVar(&configFile, "config", "", "Config file") pf.StringVar(&configFile, "config", "", "Config file")
configFile = os.Getenv("OD_CONFIG") configFile = os.Getenv("OD_CONFIG")
pf.String(ConfTrackerUrl, "http://tt.the-eye.eu/api", "task_tracker api URL") pf.String(ConfTrackerUrl, "https://tt.the-eye.eu/api", "task_tracker api URL")
pf.String(ConfTrackerProject, "3", "task_tracker project id") pf.String(ConfTrackerProject, "1", "task_tracker project id")
pf.String(ConfWsBucketScheme, "ws", "ws_bucket scheme") pf.String(ConfWsBucketScheme, "wss", "ws_bucket scheme")
pf.String(ConfWsBucketHost, "localhost:3020", "ws_bucket host") //todo def val pf.String(ConfWsBucketHost, "wsb.the-eye.eu", "ws_bucket host")
pf.String(ConfTrackerAlias, "crawler", "task_tracker worker alias") pf.String(ConfTrackerAlias, "changeme", "task_tracker worker alias")
pf.Duration(ConfServerTimeout, 60*time.Second, "OD-DB request timeout") pf.Duration(ConfServerTimeout, 60*time.Second, "OD-DB request timeout")
@ -89,7 +89,7 @@ func prepareConfig() {
pf.Duration(ConfUploadRetryInterval, 30*time.Second, "OD-DB: Time to wait between upload retries") pf.Duration(ConfUploadRetryInterval, 30*time.Second, "OD-DB: Time to wait between upload retries")
pf.Uint(ConfTasks, 100, "Crawler: Max concurrent tasks") pf.Uint(ConfTasks, 25, "Crawler: Max concurrent tasks")
pf.Uint(ConfWorkers, 4, "Crawler: Connections per server") pf.Uint(ConfWorkers, 4, "Crawler: Connections per server")

10
config.yml
View File

@ -1,14 +1,14 @@
# OD-Database server settings # OD-Database server settings
server: server:
# Connection URL # Connection URL
url: http://localhost:3010 url: https://tt.the-eye.eu/api
# OD-Database project id (for crawling) # OD-Database project id (for crawling)
project: 3 project: 1
# Your worker alias # Your worker alias
alias: changeme alias: changeme
# Websocket bucket host & scheme (ws/wss) # Websocket bucket host & scheme (ws/wss)
ws_bucket_host: localhost:3020 ws_bucket_host: https://wsb.the-eye.eu
ws_bucket_scheme: ws ws_bucket_scheme: wss
# Request timeout # Request timeout
timeout: 60s timeout: 60s
@ -47,7 +47,7 @@ output:
# Crawler settings # Crawler settings
crawl: crawl:
# Number of sites that can be processed at once # Number of sites that can be processed at once
tasks: 100 tasks: 25
# Number of connections per site # Number of connections per site
# Please be careful with this setting! # Please be careful with this setting!