mirror of
https://github.com/simon987/task_tracker.git
synced 2025-04-19 10:16:41 +00:00
Add per project rate limit
This commit is contained in:
parent
9acf6e27c1
commit
3415f95337
@ -9,6 +9,7 @@ import (
|
|||||||
"github.com/simon987/task_tracker/config"
|
"github.com/simon987/task_tracker/config"
|
||||||
"github.com/simon987/task_tracker/storage"
|
"github.com/simon987/task_tracker/storage"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
type WebAPI struct {
|
type WebAPI struct {
|
||||||
@ -18,6 +19,8 @@ type WebAPI struct {
|
|||||||
SessionConfig sessions.Config
|
SessionConfig sessions.Config
|
||||||
Session *sessions.Sessions
|
Session *sessions.Sessions
|
||||||
Cron *cron.Cron
|
Cron *cron.Cron
|
||||||
|
AssignLimiters sync.Map
|
||||||
|
SubmitLimiters sync.Map
|
||||||
}
|
}
|
||||||
|
|
||||||
type RequestHandler func(*Request)
|
type RequestHandler func(*Request)
|
||||||
@ -98,7 +101,6 @@ func New() *WebAPI {
|
|||||||
|
|
||||||
api.router.POST("/task/submit", LogRequestMiddleware(api.SubmitTask))
|
api.router.POST("/task/submit", LogRequestMiddleware(api.SubmitTask))
|
||||||
api.router.GET("/task/get/:project", LogRequestMiddleware(api.GetTaskFromProject))
|
api.router.GET("/task/get/:project", LogRequestMiddleware(api.GetTaskFromProject))
|
||||||
api.router.GET("/task/get", LogRequestMiddleware(api.GetTask))
|
|
||||||
api.router.POST("/task/release", LogRequestMiddleware(api.ReleaseTask))
|
api.router.POST("/task/release", LogRequestMiddleware(api.ReleaseTask))
|
||||||
|
|
||||||
api.router.POST("/git/receivehook", LogRequestMiddleware(api.ReceiveGitWebHook))
|
api.router.POST("/git/receivehook", LogRequestMiddleware(api.ReceiveGitWebHook))
|
||||||
|
@ -3,6 +3,7 @@ package api
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"github.com/simon987/task_tracker/storage"
|
"github.com/simon987/task_tracker/storage"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -14,6 +15,7 @@ const (
|
|||||||
type JsonResponse struct {
|
type JsonResponse struct {
|
||||||
Ok bool `json:"ok"`
|
Ok bool `json:"ok"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
|
RateLimitDelay string `json:"rate_limit_delay,omitempty"`
|
||||||
Content interface{} `json:"content,omitempty"`
|
Content interface{} `json:"content,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -124,6 +126,8 @@ type CreateProjectRequest struct {
|
|||||||
Public bool `json:"public"`
|
Public bool `json:"public"`
|
||||||
Hidden bool `json:"hidden"`
|
Hidden bool `json:"hidden"`
|
||||||
Chain int64 `json:"chain"`
|
Chain int64 `json:"chain"`
|
||||||
|
AssignRate rate.Limit `json:"assign_rate"`
|
||||||
|
SubmitRate rate.Limit `json:"submit_rate"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (req *CreateProjectRequest) isValid() bool {
|
func (req *CreateProjectRequest) isValid() bool {
|
||||||
@ -149,6 +153,8 @@ type UpdateProjectRequest struct {
|
|||||||
Hidden bool `json:"hidden"`
|
Hidden bool `json:"hidden"`
|
||||||
Chain int64 `json:"chain"`
|
Chain int64 `json:"chain"`
|
||||||
Paused bool `json:"paused"`
|
Paused bool `json:"paused"`
|
||||||
|
AssignRate rate.Limit `json:"assign_rate"`
|
||||||
|
SubmitRate rate.Limit `json:"submit_rate"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (req *UpdateProjectRequest) isValid() bool {
|
func (req *UpdateProjectRequest) isValid() bool {
|
||||||
|
@ -5,6 +5,7 @@ import (
|
|||||||
"github.com/Sirupsen/logrus"
|
"github.com/Sirupsen/logrus"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/simon987/task_tracker/storage"
|
"github.com/simon987/task_tracker/storage"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
"strconv"
|
"strconv"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -62,6 +63,13 @@ func (api *WebAPI) CreateProject(r *Request) {
|
|||||||
}, 400)
|
}, 400)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if createReq.AssignRate == 0 {
|
||||||
|
createReq.AssignRate = rate.Inf
|
||||||
|
}
|
||||||
|
if createReq.SubmitRate == 0 {
|
||||||
|
createReq.SubmitRate = rate.Inf
|
||||||
|
}
|
||||||
project := &storage.Project{
|
project := &storage.Project{
|
||||||
Name: createReq.Name,
|
Name: createReq.Name,
|
||||||
Version: createReq.Version,
|
Version: createReq.Version,
|
||||||
@ -72,6 +80,8 @@ func (api *WebAPI) CreateProject(r *Request) {
|
|||||||
Public: createReq.Public,
|
Public: createReq.Public,
|
||||||
Hidden: createReq.Hidden,
|
Hidden: createReq.Hidden,
|
||||||
Chain: createReq.Chain,
|
Chain: createReq.Chain,
|
||||||
|
AssignRate: createReq.AssignRate,
|
||||||
|
SubmitRate: createReq.SubmitRate,
|
||||||
}
|
}
|
||||||
|
|
||||||
if !createReq.isValid() {
|
if !createReq.isValid() {
|
||||||
@ -166,6 +176,8 @@ func (api *WebAPI) UpdateProject(r *Request) {
|
|||||||
Hidden: updateReq.Hidden,
|
Hidden: updateReq.Hidden,
|
||||||
Chain: updateReq.Chain,
|
Chain: updateReq.Chain,
|
||||||
Paused: updateReq.Paused,
|
Paused: updateReq.Paused,
|
||||||
|
AssignRate: updateReq.AssignRate,
|
||||||
|
SubmitRate: updateReq.SubmitRate,
|
||||||
}
|
}
|
||||||
sess := api.Session.StartFasthttp(r.Ctx)
|
sess := api.Session.StartFasthttp(r.Ctx)
|
||||||
manager := sess.Get("manager")
|
manager := sess.Get("manager")
|
||||||
|
40
api/rate.go
Normal file
40
api/rate.go
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (api *WebAPI) ReserveSubmit(pid int64) *rate.Reservation {
|
||||||
|
|
||||||
|
limiter, ok := api.SubmitLimiters.Load(pid)
|
||||||
|
if !ok {
|
||||||
|
project := api.Database.GetProject(pid)
|
||||||
|
if project == nil {
|
||||||
|
return &rate.Reservation{}
|
||||||
|
}
|
||||||
|
|
||||||
|
limiter = rate.NewLimiter(project.SubmitRate, 1)
|
||||||
|
api.SubmitLimiters.Store(pid, limiter)
|
||||||
|
}
|
||||||
|
|
||||||
|
return limiter.(*rate.Limiter).ReserveN(time.Now(), 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (api *WebAPI) ReserveAssign(pid int64) *rate.Reservation {
|
||||||
|
|
||||||
|
limiter, ok := api.AssignLimiters.Load(pid)
|
||||||
|
if !ok {
|
||||||
|
project := api.Database.GetProject(pid)
|
||||||
|
if project == nil {
|
||||||
|
return &rate.Reservation{}
|
||||||
|
}
|
||||||
|
|
||||||
|
limiter = rate.NewLimiter(project.AssignRate, 1)
|
||||||
|
api.AssignLimiters.Store(pid, limiter)
|
||||||
|
fmt.Printf("Create")
|
||||||
|
}
|
||||||
|
|
||||||
|
return limiter.(*rate.Limiter).ReserveN(time.Now(), 1)
|
||||||
|
}
|
63
api/task.go
63
api/task.go
@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/dchest/siphash"
|
"github.com/dchest/siphash"
|
||||||
"github.com/simon987/task_tracker/storage"
|
"github.com/simon987/task_tracker/storage"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (api *WebAPI) SubmitTask(r *Request) {
|
func (api *WebAPI) SubmitTask(r *Request) {
|
||||||
@ -53,6 +54,17 @@ func (api *WebAPI) SubmitTask(r *Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
reservation := api.ReserveSubmit(createReq.Project)
|
||||||
|
delay := reservation.DelayFrom(time.Now()).Seconds()
|
||||||
|
if delay > 0 {
|
||||||
|
r.Json(JsonResponse{
|
||||||
|
Ok: false,
|
||||||
|
Message: "Too many requests",
|
||||||
|
RateLimitDelay: strconv.FormatFloat(delay, 'f', -1, 64),
|
||||||
|
}, 429)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if createReq.UniqueString != "" {
|
if createReq.UniqueString != "" {
|
||||||
//TODO: Load key from config
|
//TODO: Load key from config
|
||||||
createReq.Hash64 = int64(siphash.Hash(1, 2, []byte(createReq.UniqueString)))
|
createReq.Hash64 = int64(siphash.Hash(1, 2, []byte(createReq.UniqueString)))
|
||||||
@ -65,6 +77,7 @@ func (api *WebAPI) SubmitTask(r *Request) {
|
|||||||
Ok: false,
|
Ok: false,
|
||||||
Message: err.Error(),
|
Message: err.Error(),
|
||||||
}, 400)
|
}, 400)
|
||||||
|
reservation.Cancel()
|
||||||
} else {
|
} else {
|
||||||
r.OkJson(JsonResponse{
|
r.OkJson(JsonResponse{
|
||||||
Ok: true,
|
Ok: true,
|
||||||
@ -84,47 +97,33 @@ func (api *WebAPI) GetTaskFromProject(r *Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
project, err := strconv.ParseInt(r.Ctx.UserValue("project").(string), 10, 64)
|
project, err := strconv.ParseInt(r.Ctx.UserValue("project").(string), 10, 64)
|
||||||
handleErr(err, r)
|
if err != nil || project <= 0 {
|
||||||
task := api.Database.GetTaskFromProject(worker, project)
|
|
||||||
|
|
||||||
if task == nil {
|
|
||||||
|
|
||||||
r.OkJson(JsonResponse{
|
|
||||||
Ok: false,
|
|
||||||
Message: "No task available",
|
|
||||||
})
|
|
||||||
|
|
||||||
} else {
|
|
||||||
|
|
||||||
r.OkJson(JsonResponse{
|
|
||||||
Ok: true,
|
|
||||||
Content: GetTaskResponse{
|
|
||||||
Task: task,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (api *WebAPI) GetTask(r *Request) {
|
|
||||||
|
|
||||||
worker, err := api.validateSignature(r)
|
|
||||||
if err != nil {
|
|
||||||
r.Json(JsonResponse{
|
r.Json(JsonResponse{
|
||||||
Ok: false,
|
Ok: false,
|
||||||
Message: err.Error(),
|
Message: "Invalid project id",
|
||||||
}, 403)
|
}, 400)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
task := api.Database.GetTask(worker)
|
reservation := api.ReserveAssign(project)
|
||||||
if task == nil {
|
delay := reservation.DelayFrom(time.Now()).Seconds()
|
||||||
|
if delay > 0 {
|
||||||
|
r.Json(JsonResponse{
|
||||||
|
Ok: false,
|
||||||
|
Message: "Too many requests",
|
||||||
|
RateLimitDelay: strconv.FormatFloat(delay, 'f', -1, 64),
|
||||||
|
}, 429)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
task := api.Database.GetTaskFromProject(worker, project)
|
||||||
|
|
||||||
|
if task == nil {
|
||||||
r.OkJson(JsonResponse{
|
r.OkJson(JsonResponse{
|
||||||
Ok: false,
|
Ok: false,
|
||||||
Message: "No task available",
|
Message: "No task available",
|
||||||
})
|
})
|
||||||
|
reservation.CancelAt(time.Now())
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
r.OkJson(JsonResponse{
|
r.OkJson(JsonResponse{
|
||||||
@ -134,8 +133,8 @@ func (api *WebAPI) GetTask(r *Request) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
|
}
|
||||||
func (api WebAPI) validateSignature(r *Request) (*storage.Worker, error) {
|
func (api WebAPI) validateSignature(r *Request) (*storage.Worker, error) {
|
||||||
|
|
||||||
widStr := string(r.Ctx.Request.Header.Peek("X-Worker-Id"))
|
widStr := string(r.Ctx.Request.Header.Peek("X-Worker-Id"))
|
||||||
|
2
jenkins/Jenkinsfile
vendored
2
jenkins/Jenkinsfile
vendored
@ -68,7 +68,7 @@ pipeline {
|
|||||||
node('master') {
|
node('master') {
|
||||||
unstash 'webdist'
|
unstash 'webdist'
|
||||||
unstash 'apidist'
|
unstash 'apidist'
|
||||||
sshCommand remote: remote, command: "rm -rf tt_api config.yml schema.sql webroot deploy.sh"
|
sshCommand remote: remote, command: "cd task_tracker && rm -rf tt_api config.yml schema.sql webroot deploy.sh"
|
||||||
sshPut remote: remote, from: 'tt_api', into: 'task_tracker/tt_api'
|
sshPut remote: remote, from: 'tt_api', into: 'task_tracker/tt_api'
|
||||||
sshPut remote: remote, from: 'config.yml', into: 'task_tracker/config.yml'
|
sshPut remote: remote, from: 'config.yml', into: 'task_tracker/config.yml'
|
||||||
sshPut remote: remote, from: 'schema.sql', into: 'task_tracker/schema.sql'
|
sshPut remote: remote, from: 'schema.sql', into: 'task_tracker/schema.sql'
|
||||||
|
@ -25,7 +25,10 @@ CREATE TABLE project
|
|||||||
git_repo TEXT NOT NULL,
|
git_repo TEXT NOT NULL,
|
||||||
version TEXT NOT NULL,
|
version TEXT NOT NULL,
|
||||||
motd TEXT NOT NULL,
|
motd TEXT NOT NULL,
|
||||||
secret TEXT NOT NULL DEFAULT '{}'
|
secret TEXT NOT NULL DEFAULT '{}',
|
||||||
|
webhook_secret TEXT NOT NULL,
|
||||||
|
assign_rate DOUBLE PRECISION NOT NULL,
|
||||||
|
submit_rate DOUBLE PRECISION NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE worker_access
|
CREATE TABLE worker_access
|
||||||
|
@ -3,6 +3,7 @@ package storage
|
|||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"github.com/Sirupsen/logrus"
|
"github.com/Sirupsen/logrus"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -18,6 +19,8 @@ type Project struct {
|
|||||||
Hidden bool `json:"hidden"`
|
Hidden bool `json:"hidden"`
|
||||||
Chain int64 `json:"chain"`
|
Chain int64 `json:"chain"`
|
||||||
Paused bool `json:"paused"`
|
Paused bool `json:"paused"`
|
||||||
|
AssignRate rate.Limit `json:"assign_rate"`
|
||||||
|
SubmitRate rate.Limit `json:"submit_rate"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type AssignedTasks struct {
|
type AssignedTasks struct {
|
||||||
@ -29,10 +32,11 @@ func (database *Database) SaveProject(project *Project, webhookSecret string) (i
|
|||||||
db := database.getDB()
|
db := database.getDB()
|
||||||
|
|
||||||
row := db.QueryRow(`INSERT INTO project (name, git_repo, clone_url, version, priority,
|
row := db.QueryRow(`INSERT INTO project (name, git_repo, clone_url, version, priority,
|
||||||
motd, public, hidden, chain, paused, webhook_secret)
|
motd, public, hidden, chain, paused, webhook_secret, assign_rate, submit_rate)
|
||||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,NULLIF($9, 0),$10,$11) RETURNING id`,
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,NULLIF($9, 0),$10,$11,$12,$13) RETURNING id`,
|
||||||
project.Name, project.GitRepo, project.CloneUrl, project.Version, project.Priority, project.Motd,
|
project.Name, project.GitRepo, project.CloneUrl, project.Version, project.Priority, project.Motd,
|
||||||
project.Public, project.Hidden, project.Chain, project.Paused, webhookSecret)
|
project.Public, project.Hidden, project.Chain, project.Paused, webhookSecret, project.AssignRate,
|
||||||
|
project.SubmitRate)
|
||||||
|
|
||||||
var id int64
|
var id int64
|
||||||
err := row.Scan(&id)
|
err := row.Scan(&id)
|
||||||
@ -58,7 +62,7 @@ func (database *Database) GetProject(id int64) *Project {
|
|||||||
|
|
||||||
db := database.getDB()
|
db := database.getDB()
|
||||||
row := db.QueryRow(`SELECT id, priority, name, clone_url, git_repo, version,
|
row := db.QueryRow(`SELECT id, priority, name, clone_url, git_repo, version,
|
||||||
motd, public, hidden, COALESCE(chain, 0), paused
|
motd, public, hidden, COALESCE(chain, 0), paused, assign_rate, submit_rate
|
||||||
FROM project WHERE id=$1`, id)
|
FROM project WHERE id=$1`, id)
|
||||||
|
|
||||||
project, err := scanProject(row)
|
project, err := scanProject(row)
|
||||||
@ -81,7 +85,7 @@ func scanProject(row *sql.Row) (*Project, error) {
|
|||||||
|
|
||||||
p := &Project{}
|
p := &Project{}
|
||||||
err := row.Scan(&p.Id, &p.Priority, &p.Name, &p.CloneUrl, &p.GitRepo, &p.Version,
|
err := row.Scan(&p.Id, &p.Priority, &p.Name, &p.CloneUrl, &p.GitRepo, &p.Version,
|
||||||
&p.Motd, &p.Public, &p.Hidden, &p.Chain, &p.Paused)
|
&p.Motd, &p.Public, &p.Hidden, &p.Chain, &p.Paused, &p.AssignRate, &p.SubmitRate)
|
||||||
|
|
||||||
return p, err
|
return p, err
|
||||||
}
|
}
|
||||||
@ -90,7 +94,8 @@ func (database *Database) GetProjectWithRepoName(repoName string) *Project {
|
|||||||
|
|
||||||
db := database.getDB()
|
db := database.getDB()
|
||||||
row := db.QueryRow(`SELECT id, priority, name, clone_url, git_repo, version,
|
row := db.QueryRow(`SELECT id, priority, name, clone_url, git_repo, version,
|
||||||
motd, public, hidden, COALESCE(chain, 0), paused FROM project WHERE LOWER(git_repo)=$1`,
|
motd, public, hidden, COALESCE(chain, 0), paused, assign_rate, submit_rate
|
||||||
|
FROM project WHERE LOWER(git_repo)=$1`,
|
||||||
strings.ToLower(repoName))
|
strings.ToLower(repoName))
|
||||||
|
|
||||||
project, err := scanProject(row)
|
project, err := scanProject(row)
|
||||||
@ -109,11 +114,13 @@ func (database *Database) UpdateProject(project *Project) error {
|
|||||||
db := database.getDB()
|
db := database.getDB()
|
||||||
|
|
||||||
res, err := db.Exec(`UPDATE project
|
res, err := db.Exec(`UPDATE project
|
||||||
SET (priority, name, clone_url, git_repo, version, motd, public, hidden, chain, paused) =
|
SET (priority, name, clone_url, git_repo, version, motd, public, hidden, chain, paused,
|
||||||
($1,$2,$3,$4,$5,$6,$7,$8,NULLIF($9, 0), $10)
|
assign_rate, submit_rate) =
|
||||||
WHERE id=$11`,
|
($1,$2,$3,$4,$5,$6,$7,$8,NULLIF($9, 0), $10,$11,$12)
|
||||||
|
WHERE id=$13`,
|
||||||
project.Priority, project.Name, project.CloneUrl, project.GitRepo, project.Version, project.Motd,
|
project.Priority, project.Name, project.CloneUrl, project.GitRepo, project.Version, project.Motd,
|
||||||
project.Public, project.Hidden, project.Chain, project.Paused, project.Id)
|
project.Public, project.Hidden, project.Chain, project.Paused, project.AssignRate, project.SubmitRate,
|
||||||
|
project.Id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/Sirupsen/logrus"
|
"github.com/Sirupsen/logrus"
|
||||||
@ -69,75 +68,6 @@ func (database *Database) SaveTask(task *Task, project int64, hash64 int64, wid
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (database *Database) GetTask(worker *Worker) *Task {
|
|
||||||
|
|
||||||
db := database.getDB()
|
|
||||||
|
|
||||||
row := db.QueryRow(`
|
|
||||||
UPDATE task
|
|
||||||
SET assignee=$1, assign_time=extract(epoch from now() at time zone 'utc')
|
|
||||||
WHERE id IN
|
|
||||||
(
|
|
||||||
SELECT task.id
|
|
||||||
FROM task
|
|
||||||
INNER JOIN project project on task.project = project.id
|
|
||||||
LEFT JOIN worker_verifies_task wvt on task.id = wvt.task AND wvt.worker=$1
|
|
||||||
WHERE NOT project.paused AND assignee IS NULL AND task.status=1
|
|
||||||
AND (project.public OR (
|
|
||||||
SELECT a.role_assign AND not a.request FROM worker_access a WHERE a.worker=$1 AND a.project=project.id
|
|
||||||
))
|
|
||||||
AND wvt.task IS NULL
|
|
||||||
ORDER BY project.priority DESC, task.priority DESC
|
|
||||||
LIMIT 1
|
|
||||||
)
|
|
||||||
RETURNING id`, worker.Id)
|
|
||||||
var id int64
|
|
||||||
|
|
||||||
err := row.Scan(&id)
|
|
||||||
if err != nil {
|
|
||||||
logrus.WithError(err).WithFields(logrus.Fields{
|
|
||||||
"worker": worker,
|
|
||||||
}).Trace("No task available")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
logrus.WithFields(logrus.Fields{
|
|
||||||
"id": id,
|
|
||||||
"worker": worker,
|
|
||||||
}).Trace("Database.getTask UPDATE task")
|
|
||||||
|
|
||||||
task := getTaskById(id, db)
|
|
||||||
|
|
||||||
return task
|
|
||||||
}
|
|
||||||
|
|
||||||
func getTaskById(id int64, db *sql.DB) *Task {
|
|
||||||
|
|
||||||
row := db.QueryRow(`
|
|
||||||
SELECT task.id, task.priority, task.project, assignee, retries, max_retries,
|
|
||||||
status, recipe, max_assign_time, assign_time, verification_count, project.priority, project.name,
|
|
||||||
project.clone_url, project.git_repo, project.version, project.motd, project.public, COALESCE(project.chain,0) FROM task
|
|
||||||
INNER JOIN project project ON task.project = project.id
|
|
||||||
WHERE task.id=$1`, id)
|
|
||||||
project := &Project{}
|
|
||||||
task := &Task{}
|
|
||||||
task.Project = project
|
|
||||||
|
|
||||||
err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
|
|
||||||
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
|
|
||||||
&task.AssignTime, &task.VerificationCount, &project.Priority, &project.Name,
|
|
||||||
&project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public,
|
|
||||||
&project.Chain)
|
|
||||||
handleErr(err)
|
|
||||||
|
|
||||||
logrus.WithFields(logrus.Fields{
|
|
||||||
"id": id,
|
|
||||||
"task": task,
|
|
||||||
}).Trace("Database.getTaskById SELECT task")
|
|
||||||
|
|
||||||
return task
|
|
||||||
}
|
|
||||||
|
|
||||||
func (database Database) ReleaseTask(id int64, workerId int64, result TaskResult, verification int64) bool {
|
func (database Database) ReleaseTask(id int64, workerId int64, result TaskResult, verification int64) bool {
|
||||||
|
|
||||||
db := database.getDB()
|
db := database.getDB()
|
||||||
@ -205,12 +135,24 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
logrus.WithFields(logrus.Fields{
|
row = db.QueryRow(`
|
||||||
"id": id,
|
SELECT task.id, task.priority, task.project, assignee, retries, max_retries,
|
||||||
"worker": worker,
|
status, recipe, max_assign_time, assign_time, verification_count, project.priority, project.name,
|
||||||
}).Trace("Database.getTask UPDATE task")
|
project.clone_url, project.git_repo, project.version, project.motd, project.public, COALESCE(project.chain,0),
|
||||||
|
project.assign_rate, project.submit_rate
|
||||||
|
FROM task
|
||||||
|
INNER JOIN project project ON task.project = project.id
|
||||||
|
WHERE task.id=$1`, id)
|
||||||
|
project := &Project{}
|
||||||
|
task := &Task{}
|
||||||
|
task.Project = project
|
||||||
|
|
||||||
task := getTaskById(id, db)
|
err = row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
|
||||||
|
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
|
||||||
|
&task.AssignTime, &task.VerificationCount, &project.Priority, &project.Name,
|
||||||
|
&project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public,
|
||||||
|
&project.Chain, &project.AssignRate, &project.SubmitRate)
|
||||||
|
handleErr(err)
|
||||||
|
|
||||||
return task
|
return task
|
||||||
}
|
}
|
||||||
|
@ -21,6 +21,8 @@ func TestCreateGetProject(t *testing.T) {
|
|||||||
Motd: "motd",
|
Motd: "motd",
|
||||||
Public: true,
|
Public: true,
|
||||||
Hidden: false,
|
Hidden: false,
|
||||||
|
AssignRate: 10.0,
|
||||||
|
SubmitRate: 20.0,
|
||||||
})
|
})
|
||||||
|
|
||||||
id := resp.Content.Id
|
id := resp.Content.Id
|
||||||
@ -64,6 +66,12 @@ func TestCreateGetProject(t *testing.T) {
|
|||||||
if getResp.Project.Hidden != false {
|
if getResp.Project.Hidden != false {
|
||||||
t.Error()
|
t.Error()
|
||||||
}
|
}
|
||||||
|
if getResp.Project.SubmitRate != 20.0 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
|
if getResp.Project.AssignRate != 10.0 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCreateProjectInvalid(t *testing.T) {
|
func TestCreateProjectInvalid(t *testing.T) {
|
||||||
@ -114,6 +122,8 @@ func TestUpdateProjectValid(t *testing.T) {
|
|||||||
CloneUrl: "CloneUrlA",
|
CloneUrl: "CloneUrlA",
|
||||||
GitRepo: "GitRepoA",
|
GitRepo: "GitRepoA",
|
||||||
Priority: 1,
|
Priority: 1,
|
||||||
|
AssignRate: 3,
|
||||||
|
SubmitRate: 3,
|
||||||
}).Content.Id
|
}).Content.Id
|
||||||
|
|
||||||
updateResp := updateProject(api.UpdateProjectRequest{
|
updateResp := updateProject(api.UpdateProjectRequest{
|
||||||
@ -125,6 +135,8 @@ func TestUpdateProjectValid(t *testing.T) {
|
|||||||
Public: false,
|
Public: false,
|
||||||
Hidden: true,
|
Hidden: true,
|
||||||
Paused: true,
|
Paused: true,
|
||||||
|
AssignRate: 1,
|
||||||
|
SubmitRate: 2,
|
||||||
}, pid, testAdminCtx)
|
}, pid, testAdminCtx)
|
||||||
|
|
||||||
if updateResp.Ok != true {
|
if updateResp.Ok != true {
|
||||||
@ -154,6 +166,12 @@ func TestUpdateProjectValid(t *testing.T) {
|
|||||||
if proj.Project.Paused != true {
|
if proj.Project.Paused != true {
|
||||||
t.Error()
|
t.Error()
|
||||||
}
|
}
|
||||||
|
if proj.Project.AssignRate != 1 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
|
if proj.Project.SubmitRate != 2 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUpdateProjectInvalid(t *testing.T) {
|
func TestUpdateProjectInvalid(t *testing.T) {
|
||||||
|
80
test/api_rate_test.go
Normal file
80
test/api_rate_test.go
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
package test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/simon987/task_tracker/api"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAssignRateLimit(t *testing.T) {
|
||||||
|
|
||||||
|
project := createProjectAsAdmin(api.CreateProjectRequest{
|
||||||
|
SubmitRate: 2,
|
||||||
|
AssignRate: 2,
|
||||||
|
Name: "testassignratelimit",
|
||||||
|
GitRepo: "testassignratelimit",
|
||||||
|
CloneUrl: "testassignratelimit",
|
||||||
|
}).Content.Id
|
||||||
|
|
||||||
|
w := genWid()
|
||||||
|
requestAccess(api.CreateWorkerAccessRequest{
|
||||||
|
Project: project,
|
||||||
|
Submit: true,
|
||||||
|
Assign: true,
|
||||||
|
}, w)
|
||||||
|
acceptAccessRequest(project, w.Id, testAdminCtx)
|
||||||
|
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
createTask(api.SubmitTaskRequest{
|
||||||
|
Project: project,
|
||||||
|
Recipe: fmt.Sprintf("%d", i),
|
||||||
|
}, w)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastResp TaskAR
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
lastResp = getTaskFromProject(project, w)
|
||||||
|
}
|
||||||
|
|
||||||
|
if lastResp.Ok != false {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
|
if len(lastResp.Message) <= 0 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubmitRateLimit(t *testing.T) {
|
||||||
|
|
||||||
|
project := createProjectAsAdmin(api.CreateProjectRequest{
|
||||||
|
SubmitRate: 2,
|
||||||
|
AssignRate: 2,
|
||||||
|
Name: "testsubmitratlimit",
|
||||||
|
GitRepo: "testsubmitratelimit",
|
||||||
|
CloneUrl: "testsubmitratelimit",
|
||||||
|
}).Content.Id
|
||||||
|
|
||||||
|
w := genWid()
|
||||||
|
requestAccess(api.CreateWorkerAccessRequest{
|
||||||
|
Project: project,
|
||||||
|
Submit: true,
|
||||||
|
Assign: true,
|
||||||
|
}, w)
|
||||||
|
acceptAccessRequest(project, w.Id, testAdminCtx)
|
||||||
|
|
||||||
|
var lastResp api.JsonResponse
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
lastResp = createTask(api.SubmitTaskRequest{
|
||||||
|
Project: project,
|
||||||
|
Recipe: fmt.Sprintf("%d", i),
|
||||||
|
}, w)
|
||||||
|
}
|
||||||
|
|
||||||
|
if lastResp.Ok != false {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
|
if len(lastResp.Message) <= 0 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -57,7 +57,7 @@ func TestCreateTaskInvalidProject(t *testing.T) {
|
|||||||
|
|
||||||
func TestGetTaskInvalidWid(t *testing.T) {
|
func TestGetTaskInvalidWid(t *testing.T) {
|
||||||
|
|
||||||
resp := getTask(nil)
|
resp := getTaskFromProject(testProject, genWid())
|
||||||
|
|
||||||
if resp.Ok != false {
|
if resp.Ok != false {
|
||||||
t.Error()
|
t.Error()
|
||||||
@ -70,7 +70,7 @@ func TestGetTaskInvalidWid(t *testing.T) {
|
|||||||
|
|
||||||
func TestGetTaskInvalidWorker(t *testing.T) {
|
func TestGetTaskInvalidWorker(t *testing.T) {
|
||||||
|
|
||||||
resp := getTask(&storage.Worker{
|
resp := getTaskFromProject(testProject, &storage.Worker{
|
||||||
Id: -1,
|
Id: -1,
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -144,6 +144,8 @@ func TestCreateGetTask(t *testing.T) {
|
|||||||
GitRepo: "myrepo",
|
GitRepo: "myrepo",
|
||||||
Priority: 999,
|
Priority: 999,
|
||||||
Public: true,
|
Public: true,
|
||||||
|
AssignRate: 2,
|
||||||
|
SubmitRate: 2,
|
||||||
}).Content.Id
|
}).Content.Id
|
||||||
|
|
||||||
worker := genWid()
|
worker := genWid()
|
||||||
@ -197,6 +199,12 @@ func TestCreateGetTask(t *testing.T) {
|
|||||||
if taskResp.Task.Project.Public != true {
|
if taskResp.Task.Project.Public != true {
|
||||||
t.Error()
|
t.Error()
|
||||||
}
|
}
|
||||||
|
if taskResp.Task.Project.AssignRate == 1 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
|
if taskResp.Task.Project.SubmitRate != 2 {
|
||||||
|
t.Error()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func createTasks(prefix string) (int64, int64) {
|
func createTasks(prefix string) (int64, int64) {
|
||||||
@ -279,36 +287,6 @@ func TestTaskProjectPriority(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTaskPriority(t *testing.T) {
|
|
||||||
|
|
||||||
wid := genWid()
|
|
||||||
|
|
||||||
// Clean other tasks
|
|
||||||
for i := 0; i < 20; i++ {
|
|
||||||
getTask(wid)
|
|
||||||
}
|
|
||||||
|
|
||||||
createTasks("")
|
|
||||||
|
|
||||||
t1 := getTask(wid).Content
|
|
||||||
t2 := getTask(wid).Content
|
|
||||||
t3 := getTask(wid).Content
|
|
||||||
t4 := getTask(wid).Content
|
|
||||||
|
|
||||||
if t1.Task.Recipe != "high2" {
|
|
||||||
t.Error()
|
|
||||||
}
|
|
||||||
if t2.Task.Recipe != "high1" {
|
|
||||||
t.Error()
|
|
||||||
}
|
|
||||||
if t3.Task.Recipe != "low2" {
|
|
||||||
t.Error()
|
|
||||||
}
|
|
||||||
if t4.Task.Recipe != "low1" {
|
|
||||||
t.Error()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTaskNoAccess(t *testing.T) {
|
func TestTaskNoAccess(t *testing.T) {
|
||||||
|
|
||||||
worker := genWid()
|
worker := genWid()
|
||||||
@ -400,15 +378,6 @@ func TestTaskHasAccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNoMoreTasks(t *testing.T) {
|
|
||||||
|
|
||||||
worker := genWid()
|
|
||||||
|
|
||||||
for i := 0; i < 15; i++ {
|
|
||||||
getTask(worker)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReleaseTaskSuccess(t *testing.T) {
|
func TestReleaseTaskSuccess(t *testing.T) {
|
||||||
|
|
||||||
worker := genWid()
|
worker := genWid()
|
||||||
|
@ -26,7 +26,9 @@ CREATE TABLE project
|
|||||||
version TEXT NOT NULL,
|
version TEXT NOT NULL,
|
||||||
motd TEXT NOT NULL,
|
motd TEXT NOT NULL,
|
||||||
secret TEXT NOT NULL DEFAULT '{}',
|
secret TEXT NOT NULL DEFAULT '{}',
|
||||||
webhook_secret TEXT NOT NULL
|
webhook_secret TEXT NOT NULL,
|
||||||
|
assign_rate DOUBLE PRECISION NOT NULL,
|
||||||
|
submit_rate DOUBLE PRECISION NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE worker_access
|
CREATE TABLE worker_access
|
||||||
|
Loading…
x
Reference in New Issue
Block a user