mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-04-19 18:36:43 +00:00
Retry /task/upload
This commit is contained in:
parent
8ed2cf3b93
commit
ac8221b109
40
server.go
40
server.go
@ -11,6 +11,7 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var serverClient = http.Client {
|
var serverClient = http.Client {
|
||||||
@ -101,25 +102,38 @@ func uploadChunks(websiteId uint64, f *os.File) error {
|
|||||||
|
|
||||||
multi.Close()
|
multi.Close()
|
||||||
|
|
||||||
req, err := http.NewRequest(
|
for retried := false; true; retried = true {
|
||||||
http.MethodPost,
|
err = nil
|
||||||
config.ServerUrl + "/task/upload",
|
if retried {
|
||||||
&b)
|
// Error occurred, retry upload
|
||||||
req.Header.Set("content-type", multi.FormDataContentType())
|
time.Sleep(5 * time.Second)
|
||||||
if err != nil { return err }
|
}
|
||||||
|
|
||||||
res, err := serverClient.Do(req)
|
req, err := http.NewRequest(
|
||||||
if err != nil { return err }
|
http.MethodPost,
|
||||||
res.Body.Close()
|
config.ServerUrl + "/task/upload",
|
||||||
|
&b)
|
||||||
|
req.Header.Set("content-type", multi.FormDataContentType())
|
||||||
|
if err != nil { continue }
|
||||||
|
|
||||||
if res.StatusCode != http.StatusOK {
|
res, err := serverClient.Do(req)
|
||||||
return fmt.Errorf("failed to upload list part %d: %s",
|
if err != nil { continue }
|
||||||
iter, res.Status)
|
res.Body.Close()
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
logrus.WithField("status", res.Status).
|
||||||
|
WithField("part", iter).
|
||||||
|
Errorf("Upload failed")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload successful
|
||||||
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
logrus.WithField("id", websiteId).
|
logrus.WithField("id", websiteId).
|
||||||
WithField("part", iter).
|
WithField("part", iter).
|
||||||
Infof("Uploading files chunk")
|
Infof("Uploaded files chunk")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user