Unescape results & don't recrawl 404

This commit is contained in:
Richard Patel
2018-11-17 01:21:20 +01:00
parent 145d37f84a
commit f1687679ab
5 changed files with 80 additions and 15 deletions

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"github.com/sirupsen/logrus"
"github.com/terorie/oddb-go/fasturl"
"os"
"path"
"sync/atomic"
@@ -91,6 +92,8 @@ func (t *Task) collect(results chan File) error {
defer f.Close()
for result := range results {
result.Path = fasturl.PathUnescape(result.Path)
result.Name = fasturl.PathUnescape(result.Name)
resJson, err := json.Marshal(result)
if err != nil { panic(err) }
_, err = f.Write(resJson)