Bulk submit

api/main.go

@@ -109,6 +109,7 @@ func New() *WebAPI {
 	api.router.POST("/project/reclaim_assigned_tasks/:id", LogRequestMiddleware(api.ReclaimAssignedTasks))
 
 	api.router.POST("/task/submit", LogRequestMiddleware(api.SubmitTask))
+	api.router.POST("/task/bulk_submit", LogRequestMiddleware(api.BulkSubmitTask))
 	api.router.GET("/task/get/:project", LogRequestMiddleware(api.GetTaskFromProject))
 	api.router.POST("/task/release", LogRequestMiddleware(api.ReleaseTask))
 

api/models.go

@@ -222,6 +222,28 @@ func (req *SubmitTaskRequest) IsValid() bool {
 	return true
 }
 
+type BulkSubmitTaskRequest struct {
+	Requests []SubmitTaskRequest `json:"requests"`
+}
+
+func (reqs BulkSubmitTaskRequest) IsValid() bool {
+
+	if reqs.Requests == nil {
+		return false
+	}
+
+	if len(reqs.Requests) == 0 {
+		return false
+	}
+
+	for _, req := range reqs.Requests {
+		if !req.IsValid() {
+			return false
+		}
+	}
+	return true
+}
+
 type ReleaseTaskRequest struct {
 	TaskId       int64              `json:"task_id"`
 	Result       storage.TaskResult `json:"result"`

api/rate.go

@@ -5,7 +5,7 @@ import (
 	"time"
 )
 
-func (api *WebAPI) ReserveSubmit(pid int64) *rate.Reservation {
+func (api *WebAPI) ReserveSubmit(pid int64, count int) *rate.Reservation {
 
 	limiter, ok := api.SubmitLimiters.Load(pid)
 	if !ok {
@@ -18,7 +18,7 @@ func (api *WebAPI) ReserveSubmit(pid int64) *rate.Reservation {
 		api.SubmitLimiters.Store(pid, limiter)
 	}
 
-	return limiter.(*rate.Limiter).ReserveN(time.Now(), 1)
+	return limiter.(*rate.Limiter).ReserveN(time.Now(), count)
 }
 
 func (api *WebAPI) ReserveAssign(pid int64) *rate.Reservation {

api/task.go

@@ -32,6 +32,18 @@ func (api *WebAPI) SubmitTask(r *Request) {
 		}, 400)
 		return
 	}
+
+	if !createReq.IsValid() {
+		logrus.WithFields(logrus.Fields{
+			"req": createReq,
+		}).Warn("Invalid task")
+		r.Json(JsonResponse{
+			Ok:      false,
+			Message: "Invalid task",
+		}, 400)
+		return
+	}
+
 	task := &storage.Task{
 		MaxRetries:        createReq.MaxRetries,
 		Recipe:            createReq.Recipe,
@@ -41,18 +53,7 @@ func (api *WebAPI) SubmitTask(r *Request) {
 		VerificationCount: createReq.VerificationCount,
 	}
 
-	if !createReq.IsValid() {
-		logrus.WithFields(logrus.Fields{
-			"task": task,
-		}).Warn("Invalid task")
-		r.Json(JsonResponse{
-			Ok:      false,
-			Message: "Invalid task",
-		}, 400)
-		return
-	}
-
-	reservation := api.ReserveSubmit(createReq.Project)
+	reservation := api.ReserveSubmit(createReq.Project, 1)
 	if reservation == nil {
 		r.Json(JsonResponse{
 			Ok:      false,
@@ -91,6 +92,103 @@ func (api *WebAPI) SubmitTask(r *Request) {
 	})
 }
 
+func (api *WebAPI) BulkSubmitTask(r *Request) {
+
+	worker, err := api.validateSecret(r)
+	if err != nil {
+		r.Json(JsonResponse{
+			Ok:      false,
+			Message: err.Error(),
+		}, 401)
+		return
+	}
+
+	createReq := &BulkSubmitTaskRequest{}
+	err = json.Unmarshal(r.Ctx.Request.Body(), createReq)
+	if err != nil || createReq.Requests == nil || len(createReq.Requests) == 0 {
+		r.Json(JsonResponse{
+			Ok:      false,
+			Message: "Could not parse request",
+		}, 400)
+		return
+	}
+	if !createReq.IsValid() {
+		logrus.WithFields(logrus.Fields{
+			"req": createReq,
+		}).Warn("Invalid request")
+		r.Json(JsonResponse{
+			Ok:      false,
+			Message: "Invalid request",
+		}, 400)
+		return
+	}
+
+	saveRequests := make([]storage.SaveTaskRequest, len(createReq.Requests))
+	projectId := createReq.Requests[0].Project
+	for i, req := range createReq.Requests {
+
+		if req.Project != projectId {
+			r.Json(JsonResponse{
+				Ok:      false,
+				Message: "All the tasks in a bulk submit must be of the same project",
+			}, 400)
+			return
+		}
+
+		if req.UniqueString != "" {
+			req.Hash64 = int64(siphash.Hash(1, 2, []byte(req.UniqueString)))
+		}
+
+		saveRequests[i] = storage.SaveTaskRequest{
+			Task: &storage.Task{
+				MaxRetries:        req.MaxRetries,
+				Recipe:            req.Recipe,
+				Priority:          req.Priority,
+				AssignTime:        0,
+				MaxAssignTime:     req.MaxAssignTime,
+				VerificationCount: req.VerificationCount,
+			},
+			Project:  projectId,
+			WorkerId: worker.Id,
+			Hash64:   req.Hash64,
+		}
+	}
+
+	reservation := api.ReserveSubmit(projectId, len(saveRequests))
+	if reservation == nil {
+		r.Json(JsonResponse{
+			Ok:      false,
+			Message: "Project not found",
+		}, 404)
+		return
+	}
+	delay := reservation.DelayFrom(time.Now()).Seconds()
+	if delay > 0 {
+		r.Json(JsonResponse{
+			Ok:             false,
+			Message:        "Too many requests",
+			RateLimitDelay: delay,
+		}, 429)
+		reservation.Cancel()
+		return
+	}
+
+	saveErrors := api.Database.BulkSaveTask(saveRequests)
+
+	if saveErrors == nil {
+		r.Json(JsonResponse{
+			Ok:      false,
+			Message: "Fatal error during bulk insert, see server logs",
+		}, 400)
+		reservation.Cancel()
+		return
+	}
+
+	r.OkJson(JsonResponse{
+		Ok: true,
+	})
+}
+
 func (api *WebAPI) GetTaskFromProject(r *Request) {
 
 	worker, err := api.validateSecret(r)