simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 10:26:43 +00:00
Code Issues Projects Releases Wiki Activity

Saved path should not contain file name

Browse Source
This commit is contained in:
Simon 2018-11-16 13:58:12 -05:00
parent 3f85cf679b
commit 1e78cea7e7
1 changed files with 38 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
crawl.go
View File

@ -29,10 +29,14 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
err = client.DoTimeout(req, res, config.Timeout) err = client.DoTimeout(req, res, config.Timeout)
fasthttp.ReleaseRequest(req) fasthttp.ReleaseRequest(req)
if err != nil { return } if err != nil {
return
}
err = checkStatusCode(res.StatusCode()) err = checkStatusCode(res.StatusCode())
if err != nil { return } if err != nil {
return
}
body := res.Body() body := res.Body()
doc := html.NewTokenizer(bytes.NewReader(body)) doc := html.NewTokenizer(bytes.NewReader(body))
@ -83,7 +87,9 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
var link fasturl.URL var link fasturl.URL
err = j.Uri.ParseRel(&link, href) err = j.Uri.ParseRel(&link, href)
if err != nil { continue } if err != nil {
continue
}
if link.Scheme != j.Uri.Scheme || if link.Scheme != j.Uri.Scheme ||
link.Host != j.Uri.Host || link.Host != j.Uri.Host ||
@ -106,7 +112,7 @@ func GetFile(u fasturl.URL, f *File) (err error) {
f.IsDir = false f.IsDir = false
u.Path = path.Clean(u.Path) u.Path = path.Clean(u.Path)
f.Name = path.Base(u.Path) f.Name = path.Base(u.Path)
f.Path = strings.Trim(u.Path, "/") f.Path = strings.Trim(path.Dir(u.Path), "/")
req := fasthttp.AcquireRequest() req := fasthttp.AcquireRequest()
req.Header.SetMethod("HEAD") req.Header.SetMethod("HEAD")
@ -119,10 +125,14 @@ func GetFile(u fasturl.URL, f *File) (err error) {
err = client.DoTimeout(req, res, config.Timeout) err = client.DoTimeout(req, res, config.Timeout)
fasthttp.ReleaseRequest(req) fasthttp.ReleaseRequest(req)
if err != nil { return } if err != nil {
return
}
err = checkStatusCode(res.StatusCode()) err = checkStatusCode(res.StatusCode())
if err != nil { return } if err != nil {
return
}
f.applyContentLength(string(res.Header.Peek("content-length"))) f.applyContentLength(string(res.Header.Peek("content-length")))
f.applyLastModified(string(res.Header.Peek("last-modified"))) f.applyLastModified(string(res.Header.Peek("last-modified")))
@ -143,23 +153,37 @@ func (f *File) HashDir(links []fasturl.URL) (o redblackhash.Key) {
} }
func (f *File) applyContentLength(v string) { func (f *File) applyContentLength(v string) {
if v == "" { return } if v == "" {
return
}
size, err := strconv.ParseInt(v, 10, 64) size, err := strconv.ParseInt(v, 10, 64)
if err != nil { return } if err != nil {
if size < 0 { return } return
}
if size < 0 {
return
}
f.Size = size f.Size = size
} }
func (f *File) applyLastModified(v string) { func (f *File) applyLastModified(v string) {
if v == "" { return } if v == "" {
return
}
var err error var err error
f.MTime, err = time.Parse(time.RFC1123, v) f.MTime, err = time.Parse(time.RFC1123, v)
if err == nil { return } if err == nil {
return
}
f.MTime, err = time.Parse(time.RFC850, v) f.MTime, err = time.Parse(time.RFC850, v)
if err == nil { return } if err == nil {
return
}
// TODO Parse asctime // TODO Parse asctime
f.MTime, err = time.Parse("2006-01-02", v[:10]) f.MTime, err = time.Parse("2006-01-02", v[:10])
if err == nil { return } if err == nil {
return
}
} }
func checkStatusCode(status int) error { func checkStatusCode(status int) error {