simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Add urfave/cli app

Browse Source
This commit is contained in:
Richard Patel
2018-10-28 17:59:46 +01:00
parent b1c40767e0
commit bfd7302be8
3 changed files with 71 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawl.go
View File

@@ -19,10 +19,6 @@ var client fasthttp.Client
var ErrRateLimit = errors.New("too many requests")
var ErrForbidden = errors.New("access denied")
func NewRemoteDir(u url.URL) *OD {
return &OD{ BaseUri: u }
}
func GetDir(j *Job, f *File) (links []url.URL, err error) {
f.IsDir = true
f.Name = path.Base(j.Uri.Path)