simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-18 18:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Optimizing with hexa :P

Browse Source
This commit is contained in:
Richard Patel 2018-11-15 23:51:31 +01:00
parent ac0b8d2d0b
commit 084b3a5903
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 2 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crawl.go
View File

@ -39,7 +39,6 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
doc := html.NewTokenizer(bytes.NewReader(body))
var linkHref string
var linkTexts []string
for {
tokenType := doc.Next()
token := doc.Token()
@ -58,20 +57,13 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
}
}
case html.TextToken:
if linkHref != "" {
linkTexts = append(linkTexts, token.Data)
}
case html.EndTagToken:
if linkHref != "" && token.DataAtom == atom.A {
// Copy params
href := linkHref
linkText := strings.Join(linkTexts, " ")
// Reset params
linkHref = ""
linkTexts = nil
// TODO Optimized decision tree
if strings.LastIndexByte(href, '?') != -1 {
@ -83,11 +75,9 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
goto nextToken
}
}
for _, entry := range fileNameBlackList {
if strings.Contains(linkText, entry) {
if strings.Contains(href, "../") {
goto nextToken
}
}
var link fasturl.URL
err = j.Uri.ParseRel(&link, href)
@ -194,10 +184,3 @@ var urlBlackList = [...]string {
"..",
"/",
}
var fileNameBlackList = [...]string {
"Parent Directory",
" Parent Directory",
"../",
}