simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 10:26:43 +00:00
Code Issues Projects Releases Wiki Activity

Switch to simon987/task_tracker

Browse Source
This commit is contained in:
Richard Patel 2019-02-28 23:51:26 +01:00
parent 0b3f0d87fe
commit 60471a081e
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
2 changed files with 46 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
model.go
View File

@ -7,19 +7,26 @@ import (
"time" "time"
) )
type ResultCode int
const (
TR_OK = ResultCode(iota)
TR_FAIL
TR_SKIP
)
type Task struct { type Task struct {
WebsiteId uint64 `json:"website_id"` WebsiteId uint64 `json:"website_id"`
Url string `json:"url"` Url string `json:"url"`
} }
type TaskResult struct { type TaskResult struct {
StatusCode string `json:"status_code"` ResultCode ResultCode `json:"status_code"`
FileCount uint64 `json:"file_count"` FileCount uint64 `json:"file_count"`
ErrorCount uint64 `json:"-"` ErrorCount uint64 `json:"-"`
StartTime time.Time `json:"-"` StartTime time.Time `json:"-"`
StartTimeUnix int64 `json:"start_time"` StartTimeUnix int64 `json:"start_time"`
EndTimeUnix int64 `json:"end_time"` EndTimeUnix int64 `json:"end_time"`
WebsiteId uint64 `json:"website_id"` WebsiteId uint64 `json:"website_id"`
} }
type Job struct { type Job struct {

34
server.go
View File

@ -144,11 +144,18 @@ func uploadChunks(websiteId uint64, f *os.File) error {
} }
func uploadResult(result *TaskResult) (err error) { func uploadResult(result *TaskResult) (err error) {
resultEnc, err := json.Marshal(result) req := releaseTaskRequest {
TaskId: int64(result.WebsiteId),
ResultCode: result.ResultCode,
// TODO What is verification
Verification: 0,
}
resultEnc, err := json.Marshal(&req)
if err != nil { panic(err) } if err != nil { panic(err) }
res, err := serverClient.PostForm( res, err := serverClient.PostForm(
config.ServerUrl + "/task/complete", config.ServerUrl + "/task/release",
url.Values { url.Values {
"token": {config.Token}, "token": {config.Token},
"result": {string(resultEnc)}, "result": {string(resultEnc)},
@ -165,6 +172,8 @@ func uploadResult(result *TaskResult) (err error) {
} }
func CancelTask(websiteId uint64) (err error) { func CancelTask(websiteId uint64) (err error) {
// TODO Remove, no endpoint in task_tracker
res, err := serverClient.PostForm( res, err := serverClient.PostForm(
config.ServerUrl + "/task/cancel", config.ServerUrl + "/task/cancel",
url.Values{ url.Values{
@ -188,3 +197,24 @@ func (t *ServerTripper) RoundTrip(req *http.Request) (res *http.Response, err er
req.Header.Set("User-Agent", serverUserAgent) req.Header.Set("User-Agent", serverUserAgent)
return http.DefaultTransport.RoundTrip(req) return http.DefaultTransport.RoundTrip(req)
} }
const mimeJSON = "application/json"
// https://github.com/simon987/task_tracker/blob/master/api/models.go
type submitTaskRequest struct {
Project int64 `json:"project"`
MaxRetries int64 `json:"max_retries"`
Recipe string `json:"recipe"`
Priority int64 `json:"priority"`
MaxAssignTime int64 `json:"max_assign_time"`
Hash64 int64 `json:"hash_u64"`
UniqueString string `json:"unique_string"`
VerificationCount int64 `json:"verification_count"`
}
type releaseTaskRequest struct {
TaskId int64 `json:"task_id"`
ResultCode ResultCode `json:"result"`
Verification int64 `json:"verification"`
}