simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-18 18:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Optimizing with hexa :P

Browse Source
This commit is contained in:
Richard Patel 2018-11-15 23:51:31 +01:00
parent ac0b8d2d0b
commit 084b3a5903
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 2 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
crawl.go
View File

@ -39,7 +39,6 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
doc := html.NewTokenizer(bytes.NewReader(body)) doc := html.NewTokenizer(bytes.NewReader(body))
var linkHref string var linkHref string
var linkTexts []string
for { for {
tokenType := doc.Next() tokenType := doc.Next()
token := doc.Token() token := doc.Token()
@ -58,20 +57,13 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
} }
} }
case html.TextToken:
if linkHref != "" {
linkTexts = append(linkTexts, token.Data)
}
case html.EndTagToken: case html.EndTagToken:
if linkHref != "" && token.DataAtom == atom.A { if linkHref != "" && token.DataAtom == atom.A {
// Copy params // Copy params
href := linkHref href := linkHref
linkText := strings.Join(linkTexts, " ")
// Reset params // Reset params
linkHref = "" linkHref = ""
linkTexts = nil
// TODO Optimized decision tree // TODO Optimized decision tree
if strings.LastIndexByte(href, '?') != -1 { if strings.LastIndexByte(href, '?') != -1 {
@ -83,10 +75,8 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
goto nextToken goto nextToken
} }
} }
for _, entry := range fileNameBlackList { if strings.Contains(href, "../") {
if strings.Contains(linkText, entry) { goto nextToken
goto nextToken
}
} }
var link fasturl.URL var link fasturl.URL
@ -194,10 +184,3 @@ var urlBlackList = [...]string {
"..", "..",
"/", "/",
} }
var fileNameBlackList = [...]string {
"Parent Directory",
" Parent Directory",
"../",
}