simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-18 18:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Hash links

Browse Source
This commit is contained in:
Richard Patel 2018-11-06 02:01:53 +01:00
parent ed5e35f005
commit a8c27b2d21
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
2 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
crawl.go
View File

@ -156,6 +156,14 @@ func (f *File) HashDir(links []fasturl.URL) (o redblackhash.Key) {
return
}
func HashString(s string) (o redblackhash.Key) {
h, _ := blake2b.New256(nil)
h.Write([]byte(s))
sum := h.Sum(nil)
copy(o[:redblackhash.KeySize], sum)
return
}
func (f *File) ParseHeader(h []byte) {
var k1, k2 int
var v1, v2 int

10
worker.go
View File

@ -86,15 +86,17 @@ func DoJob(job *Job, f *File) (newJobs []Job, err error) {
}
for _, link := range links {
uriStr := link.String()
// Skip already queued links
//if _, old := job.OD.Scanned.LoadOrStore(link, true); old {
// continue
//}
linkHash := HashString(uriStr)
if job.OD.LoadOrStoreKey(&linkHash) {
continue
}
job.OD.Wait.Add(1)
newJobs = append(newJobs, Job{
OD: job.OD,
Uri: link,
UriStr: link.String(),
UriStr: uriStr,
Fails: 0,
})
}