mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-12-13 23:29:02 +00:00
Unescape results & don't recrawl 404
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/terorie/oddb-go/fasturl"
|
||||
"os"
|
||||
"path"
|
||||
"sync/atomic"
|
||||
@@ -91,6 +92,8 @@ func (t *Task) collect(results chan File) error {
|
||||
defer f.Close()
|
||||
|
||||
for result := range results {
|
||||
result.Path = fasturl.PathUnescape(result.Path)
|
||||
result.Name = fasturl.PathUnescape(result.Name)
|
||||
resJson, err := json.Marshal(result)
|
||||
if err != nil { panic(err) }
|
||||
_, err = f.Write(resJson)
|
||||
|
||||
Reference in New Issue
Block a user