simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Exclude dups in dir instead of keeping hashes of links

Browse Source
This commit is contained in:
Richard Patel
2018-11-11 23:11:30 +01:00
parent 9c8174dd8d
commit 4c071171eb
2 changed files with 17 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crawl.go
View File

@@ -155,14 +155,6 @@ func (f *File) HashDir(links []fasturl.URL) (o redblackhash.Key) {
return
}
func HashString(s string) (o redblackhash.Key) {
h, _ := blake2b.New256(nil)
h.Write([]byte(s))
sum := h.Sum(nil)
copy(o[:redblackhash.KeySize], sum)
return
}
func (f *File) applyContentLength(v string) {
if v == "" { return }
size, err := strconv.ParseInt(v, 10, 64)