simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 23:29:02 +00:00
Code Issues Projects Releases Wiki Activity

Fix crawl descent

Browse Source
This commit is contained in:
Richard Patel
2018-10-28 03:06:18 +01:00
parent a507110787
commit 1c33346f45
5 changed files with 31 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
worker.go
View File

@@ -59,7 +59,7 @@ func DoJob(job *Job, f *File) (newJobs []Job, err error) {
// File
if strings.HasSuffix(job.Uri.Path, "/") {
// Dir
links, err := GetDir(job.Uri, f)
links, err := GetDir(job, f)
if err != nil {
logrus.WithError(err).
WithField("url", job.Uri.String()).
@@ -75,6 +75,10 @@ func DoJob(job *Job, f *File) (newJobs []Job, err error) {
Fails: 0,
})
}
logrus.WithFields(logrus.Fields{
"url": job.UriStr,
"files": len(links),
}).Debug("Listed")
} else {
err := GetFile(job.Uri, f)
if err != nil {