simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 10:26:43 +00:00
Code Issues Projects Releases Wiki Activity

Fix: make crawled dir

Browse Source
This commit is contained in:
Richard Patel 2018-11-17 13:36:35 +01:00
parent 73ba848e17
commit 8060556089
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scheduler.go
View File

@ -73,12 +73,17 @@ func ScheduleTask(remotes chan<- *OD, t *Task, u *fasturl.URL) {
func (o *OD) Watch(results chan File) { func (o *OD) Watch(results chan File) {
filePath := path.Join("crawled", fmt.Sprintf("%d.json", o.Task.WebsiteId)) filePath := path.Join("crawled", fmt.Sprintf("%d.json", o.Task.WebsiteId))
// Open crawl results file // Open crawl results file
// TODO Print errors
err := os.MkdirAll("crawled", 0755)
if err != nil { return }
f, err := os.OpenFile( f, err := os.OpenFile(
filePath, filePath,
os.O_CREATE | os.O_RDWR | os.O_TRUNC, os.O_CREATE | os.O_RDWR | os.O_TRUNC,
0644, 0644,
) )
if err != nil { return } if err != nil { return }
defer f.Close() defer f.Close()
defer os.Remove(filePath) defer os.Remove(filePath)
@ -144,9 +149,6 @@ func (t *Task) Collect(results chan File, f *os.File, done *sync.Mutex) {
} }
func (t *Task) collect(results chan File, f *os.File) error { func (t *Task) collect(results chan File, f *os.File) error {
err := os.MkdirAll("crawled", 0755)
if err != nil { return err }
for result := range results { for result := range results {
result.Path = fasturl.PathUnescape(result.Path) result.Path = fasturl.PathUnescape(result.Path)
result.Name = fasturl.PathUnescape(result.Name) result.Name = fasturl.PathUnescape(result.Name)