simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 18:36:43 +00:00
Code Issues Projects Releases Wiki Activity

Use finite state machine

Browse Source
This commit is contained in:
Richard Patel 2018-10-27 16:55:00 +02:00
parent 442a2cf8a7
commit 76c8c13d49
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 2 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crawl.go
View File

@ -8,7 +8,6 @@ import (
"golang.org/x/net/html/atom" "golang.org/x/net/html/atom"
"net/url" "net/url"
"os" "os"
"regexp"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@ -28,8 +27,6 @@ var visited int64
var in chan<- url.URL var in chan<- url.URL
var out <-chan url.URL var out <-chan url.URL
var matchHeader = regexp.MustCompile("([\\w-]+): (.*)")
type File struct { type File struct {
Name string `json:"name"` Name string `json:"name"`
Size int64 `json:"size"` Size int64 `json:"size"`
@ -188,24 +185,12 @@ func fileInfo(u url.URL, f *File) (err error) {
// TODO Inefficient af // TODO Inefficient af
header := res.Header.Header() header := res.Header.Header()
s := time.Now() f.ParseHeader(header)
for i := 0; i < 10000; i++ {
f.ParseHeaderRegex(header)
}
println(time.Since(s).String())
return nil return nil
} }
func (f *File) ParseHeaderRegex(h []byte) { func (f *File) ParseHeader(h []byte) {
for _, parts := range matchHeader.FindAllSubmatch(h, -1) {
k := string(parts[1])
v := string(parts[2])
f.applyHeader(k, v)
}
}
func (f *File) ParseHeaderMachine(h []byte) {
var k1, k2 int var k1, k2 int
var v1, v2 int var v1, v2 int