simon987/task_tracker
1
0
Fork 0
mirror of https://github.com/simon987/task_tracker.git synced 2025-04-10 05:56:42 +00:00
Code Issues Projects Releases Wiki Activity

Performance patch, version bump

Browse Source
This commit is contained in:
simon 2019-09-21 14:32:18 -04:00
parent 77b4da0653
commit 3123abceb6
34 changed files with 362 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
api/log.go
View File

@ -5,7 +5,6 @@ import (
"errors"
"github.com/simon987/task_tracker/config"
"github.com/sirupsen/logrus"
"github.com/valyala/fasthttp"
"os"
"time"
)
@ -16,22 +15,6 @@ func (e *LogRequest) Time() time.Time {
return t
}
func LogRequestMiddleware(h RequestHandler) fasthttp.RequestHandler {
return fasthttp.RequestHandler(func(ctx *fasthttp.RequestCtx) {
ctx.Response.Header.Add("Access-Control-Allow-Headers", "Content-Type")
ctx.Response.Header.Add("Access-Control-Allow-Methods", "GET, POST, OPTION")
ctx.Response.Header.Add("Access-Control-Allow-Origin", "*")
logrus.WithFields(logrus.Fields{
"path": string(ctx.Path()),
"header": ctx.Request.Header.String(),
}).Trace(string(ctx.Method()))
h(&Request{Ctx: ctx})
})
}
func (api *WebAPI) SetupLogger() {
writer, err := os.OpenFile("log.txt", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {

94
api/main.go
View File

@ -27,7 +27,13 @@ type RequestHandler func(*Request)
var info = Info{
Name: "task_tracker",
Version: "1.0",
Version: "1.1",
}
func Middleware(h RequestHandler) fasthttp.RequestHandler {
return fasthttp.RequestHandler(func(ctx *fasthttp.RequestCtx) {
h(&Request{Ctx: ctx})
})
}
func Index(r *Request) {
@ -77,56 +83,56 @@ func New() *WebAPI {
Name: info.Name,
}
api.router.GET("/", LogRequestMiddleware(Index))
api.router.GET("/", Middleware(Index))
api.router.POST("/log/trace", LogRequestMiddleware(api.LogTrace))
api.router.POST("/log/info", LogRequestMiddleware(api.LogInfo))
api.router.POST("/log/warn", LogRequestMiddleware(api.LogWarn))
api.router.POST("/log/error", LogRequestMiddleware(api.LogError))
api.router.POST("/log/trace", Middleware(api.LogTrace))
api.router.POST("/log/info", Middleware(api.LogInfo))
api.router.POST("/log/warn", Middleware(api.LogWarn))
api.router.POST("/log/error", Middleware(api.LogError))
api.router.POST("/worker/create", LogRequestMiddleware(api.CreateWorker))
api.router.POST("/worker/update", LogRequestMiddleware(api.UpdateWorker))
api.router.POST("/worker/set_paused", LogRequestMiddleware(api.WorkerSetPaused))
api.router.GET("/worker/get/:id", LogRequestMiddleware(api.GetWorker))
api.router.GET("/worker/stats", LogRequestMiddleware(api.GetAllWorkerStats))
api.router.POST("/worker/create", Middleware(api.CreateWorker))
api.router.POST("/worker/update", Middleware(api.UpdateWorker))
api.router.POST("/worker/set_paused", Middleware(api.WorkerSetPaused))
api.router.GET("/worker/get/:id", Middleware(api.GetWorker))
api.router.GET("/worker/stats", Middleware(api.GetAllWorkerStats))
api.router.POST("/project/create", LogRequestMiddleware(api.CreateProject))
api.router.GET("/project/get/:id", LogRequestMiddleware(api.GetProject))
api.router.POST("/project/update/:id", LogRequestMiddleware(api.UpdateProject))
api.router.GET("/project/list", LogRequestMiddleware(api.GetProjectList))
api.router.GET("/project/monitoring-between/:id", LogRequestMiddleware(api.GetSnapshotsWithinRange))
api.router.GET("/project/monitoring/:id", LogRequestMiddleware(api.GetNSnapshots))
api.router.GET("/project/assignees/:id", LogRequestMiddleware(api.GetAssigneeStatsForProject))
api.router.GET("/project/access_list/:id", LogRequestMiddleware(api.GetWorkerAccessListForProject))
api.router.POST("/project/request_access", LogRequestMiddleware(api.CreateWorkerAccess))
api.router.POST("/project/accept_request/:id/:wid", LogRequestMiddleware(api.AcceptAccessRequest))
api.router.POST("/project/reject_request/:id/:wid", LogRequestMiddleware(api.RejectAccessRequest))
api.router.GET("/project/secret/:id", LogRequestMiddleware(api.GetSecret))
api.router.POST("/project/secret/:id", LogRequestMiddleware(api.SetSecret))
api.router.GET("/project/webhook_secret/:id", LogRequestMiddleware(api.GetWebhookSecret))
api.router.POST("/project/webhook_secret/:id", LogRequestMiddleware(api.SetWebhookSecret))
api.router.POST("/project/reset_failed_tasks/:id", LogRequestMiddleware(api.ResetFailedTasks))
api.router.POST("/project/hard_reset/:id", LogRequestMiddleware(api.HardReset))
api.router.POST("/project/reclaim_assigned_tasks/:id", LogRequestMiddleware(api.ReclaimAssignedTasks))
api.router.POST("/project/create", Middleware(api.CreateProject))
api.router.GET("/project/get/:id", Middleware(api.GetProject))
api.router.POST("/project/update/:id", Middleware(api.UpdateProject))
api.router.GET("/project/list", Middleware(api.GetProjectList))
api.router.GET("/project/monitoring-between/:id", Middleware(api.GetSnapshotsWithinRange))
api.router.GET("/project/monitoring/:id", Middleware(api.GetNSnapshots))
api.router.GET("/project/assignees/:id", Middleware(api.GetAssigneeStatsForProject))
api.router.GET("/project/access_list/:id", Middleware(api.GetWorkerAccessListForProject))
api.router.POST("/project/request_access", Middleware(api.CreateWorkerAccess))
api.router.POST("/project/accept_request/:id/:wid", Middleware(api.AcceptAccessRequest))
api.router.POST("/project/reject_request/:id/:wid", Middleware(api.RejectAccessRequest))
api.router.GET("/project/secret/:id", Middleware(api.GetSecret))
api.router.POST("/project/secret/:id", Middleware(api.SetSecret))
api.router.GET("/project/webhook_secret/:id", Middleware(api.GetWebhookSecret))
api.router.POST("/project/webhook_secret/:id", Middleware(api.SetWebhookSecret))
api.router.POST("/project/reset_failed_tasks/:id", Middleware(api.ResetFailedTasks))
api.router.POST("/project/hard_reset/:id", Middleware(api.HardReset))
api.router.POST("/project/reclaim_assigned_tasks/:id", Middleware(api.ReclaimAssignedTasks))
api.router.POST("/task/submit", LogRequestMiddleware(api.SubmitTask))
api.router.POST("/task/bulk_submit", LogRequestMiddleware(api.BulkSubmitTask))
api.router.GET("/task/get/:project", LogRequestMiddleware(api.GetTaskFromProject))
api.router.POST("/task/release", LogRequestMiddleware(api.ReleaseTask))
api.router.POST("/task/submit", Middleware(api.SubmitTask))
api.router.POST("/task/bulk_submit", Middleware(api.BulkSubmitTask))
api.router.GET("/task/get/:project", Middleware(api.GetTaskFromProject))
api.router.POST("/task/release", Middleware(api.ReleaseTask))
api.router.POST("/git/receivehook", LogRequestMiddleware(api.ReceiveGitWebHook))
api.router.POST("/git/receivehook", Middleware(api.ReceiveGitWebHook))
api.router.POST("/logs", LogRequestMiddleware(api.GetLog))
api.router.POST("/logs", Middleware(api.GetLog))
api.router.POST("/register", LogRequestMiddleware(api.Register))
api.router.POST("/login", LogRequestMiddleware(api.Login))
api.router.GET("/logout", LogRequestMiddleware(api.Logout))
api.router.GET("/account", LogRequestMiddleware(api.GetAccountDetails))
api.router.GET("/manager/list", LogRequestMiddleware(api.GetManagerList))
api.router.GET("/manager/list_for_project/:id", LogRequestMiddleware(api.GetManagerListWithRoleOn))
api.router.GET("/manager/promote/:id", LogRequestMiddleware(api.PromoteManager))
api.router.GET("/manager/demote/:id", LogRequestMiddleware(api.DemoteManager))
api.router.POST("/manager/set_role_for_project/:id", LogRequestMiddleware(api.SetManagerRoleOnProject))
api.router.POST("/register", Middleware(api.Register))
api.router.POST("/login", Middleware(api.Login))
api.router.GET("/logout", Middleware(api.Logout))
api.router.GET("/account", Middleware(api.GetAccountDetails))
api.router.GET("/manager/list", Middleware(api.GetManagerList))
api.router.GET("/manager/list_for_project/:id", Middleware(api.GetManagerListWithRoleOn))
api.router.GET("/manager/promote/:id", Middleware(api.PromoteManager))
api.router.GET("/manager/demote/:id", Middleware(api.DemoteManager))
api.router.POST("/manager/set_role_for_project/:id", Middleware(api.SetManagerRoleOnProject))
api.router.NotFound = func(ctx *fasthttp.RequestCtx) {

8
api/models.go
View File

@ -196,13 +196,13 @@ type GetWorkerAccessListForProjectResponse struct {
type SubmitTaskRequest struct {
Project int64 `json:"project"`
MaxRetries int64 `json:"max_retries"`
MaxRetries int16 `json:"max_retries"`
Recipe string `json:"recipe"`
Priority int64 `json:"priority"`
Priority int16 `json:"priority"`
MaxAssignTime int64 `json:"max_assign_time"`
Hash64 int64 `json:"hash_u64"`
UniqueString string `json:"unique_string"`
VerificationCount int64 `json:"verification_count"`
VerificationCount int16 `json:"verification_count"`
}
func (req *SubmitTaskRequest) IsValid() bool {
@ -212,7 +212,7 @@ func (req *SubmitTaskRequest) IsValid() bool {
if len(req.Recipe) <= 0 {
return false
}
if req.Hash64 != 0 && req.UniqueString != "" {
if req.Hash64 != 0 && len(req.UniqueString) != 0 {
return false
}
if req.Project == 0 {

6
api/project.go
View File

@ -165,6 +165,12 @@ func (api *WebAPI) UpdateProject(r *Request) {
return
}
if updateReq.AssignRate == 0 {
updateReq.AssignRate = rate.Inf
}
if updateReq.SubmitRate == 0 {
updateReq.SubmitRate = rate.Inf
}
project := &storage.Project{
Id: id,
Name: updateReq.Name,

25
api/task.go
View File

@ -33,6 +33,10 @@ func (api *WebAPI) SubmitTask(r *Request) {
return
}
if createReq.VerificationCount == 0 {
createReq.VerificationCount = 1
}
if !createReq.IsValid() {
logrus.WithFields(logrus.Fields{
"req": createReq,
@ -46,7 +50,7 @@ func (api *WebAPI) SubmitTask(r *Request) {
task := &storage.Task{
MaxRetries: createReq.MaxRetries,
Recipe: createReq.Recipe,
Recipe: string(createReq.Recipe),
Priority: createReq.Priority,
AssignTime: 0,
MaxAssignTime: createReq.MaxAssignTime,
@ -72,7 +76,7 @@ func (api *WebAPI) SubmitTask(r *Request) {
return
}
if createReq.UniqueString != "" {
if len(createReq.UniqueString) != 0 {
createReq.Hash64 = int64(siphash.Hash(1, 2, []byte(createReq.UniqueString)))
}
@ -135,14 +139,17 @@ func (api *WebAPI) BulkSubmitTask(r *Request) {
return
}
if req.UniqueString != "" {
if len(req.UniqueString) != 0 {
req.Hash64 = int64(siphash.Hash(1, 2, []byte(req.UniqueString)))
}
if req.VerificationCount == 0 {
req.VerificationCount = 1
}
saveRequests[i] = storage.SaveTaskRequest{
Task: &storage.Task{
MaxRetries: req.MaxRetries,
Recipe: req.Recipe,
Recipe: string(req.Recipe),
Priority: req.Priority,
AssignTime: 0,
MaxAssignTime: req.MaxAssignTime,
@ -184,6 +191,8 @@ func (api *WebAPI) BulkSubmitTask(r *Request) {
return
}
logrus.Info(saveErrors)
r.OkJson(JsonResponse{
Ok: true,
})
@ -263,7 +272,7 @@ func (api *WebAPI) validateSecret(r *Request) (*storage.Worker, error) {
if widStr == "" {
return nil, errors.New("worker id not specified")
}
if bytes.Equal(secretHeader, []byte("")) {
if len(secretHeader) == 0 {
return nil, errors.New("secret is not specified")
}
@ -290,12 +299,6 @@ func (api *WebAPI) validateSecret(r *Request) (*storage.Worker, error) {
secretLen, _ := base64.StdEncoding.Decode(secret, secretHeader)
matches := bytes.Equal(worker.Secret, secret[:secretLen])
logrus.WithFields(logrus.Fields{
"expected": string(worker.Secret),
"header": string(secretHeader),
"matches": matches,
}).Trace("Validating Worker secret")
if !matches {
return nil, errors.New("invalid secret")
}

14
client/models.go
View File

@ -21,7 +21,19 @@ type AssignTaskResponse struct {
Message string `json:"message"`
RateLimitDelay float64 `json:"rate_limit_delay,omitempty"`
Content struct {
Task *storage.Task `json:"task"`
Task *struct {
Id int64 `json:"id"`
Priority int16 `json:"priority"`
Project *storage.Project `json:"project"`
Assignee int64 `json:"assignee"`
Retries int16 `json:"retries"`
MaxRetries int64 `json:"max_retries"`
Status storage.TaskStatus `json:"status"`
Recipe string `json:"recipe"`
MaxAssignTime int64 `json:"max_assign_time"`
AssignTime int64 `json:"assign_time"`
VerificationCount int16 `json:"verification_count"`
} `json:"task"`
} `json:"content"`
}

0
config.yml Executable file → Normal file
View File

1
jenkins/Jenkinsfile vendored
View File

@ -7,6 +7,7 @@ remote.knownHosts = '/var/lib/jenkins/.ssh/known_hosts'
remote.allowAnyHosts = true
remote.retryCount = 3
remote.retryWaitSec = 3
remote.port = 2299
logLevel = 'FINER'
pipeline {

0
jenkins/deploy.sh Executable file → Normal file
View File

9
main/main.go
View File

@ -3,18 +3,10 @@ package main
import (
"github.com/simon987/task_tracker/api"
"github.com/simon987/task_tracker/config"
//"github.com/simon987/task_tracker/storage"
"math/rand"
"time"
)
func tmpDebugSetup() {
//db := storage.Database{}
//db.Reset()
}
func main() {
rand.Seed(time.Now().UTC().UnixNano())
@ -22,6 +14,5 @@ func main() {
webApi := api.New()
webApi.SetupLogger()
tmpDebugSetup()
webApi.Run()
}

20
schema.sql Executable file → Normal file
View File

@ -41,6 +41,8 @@ CREATE TABLE worker_access
request boolean,
primary key (worker, project)
);
CREATE INDEX worker_index ON worker_access (worker);
CREATE INDEX project_index ON worker_access (project);
CREATE TABLE task
(
@ -50,22 +52,28 @@ CREATE TABLE task
assignee INTEGER REFERENCES worker (id),
max_assign_time INTEGER DEFAULT 0,
assign_time INTEGER DEFAULT NULL,
verification_count INTEGER DEFAULT 0,
verification_count SMALLINT DEFAULT 0,
priority SMALLINT DEFAULT 0,
retries SMALLINT DEFAULT 0,
max_retries SMALLINT,
status SMALLINT DEFAULT 1,
recipe TEXT,
UNIQUE (project, hash64)
recipe TEXT
);
CREATE INDEX priority_desc_index ON task (priority DESC);
CREATE INDEX assignee_index ON task (assignee);
CREATE INDEX verifcnt_index ON task (verification_count);
CREATE UNIQUE INDEX project_hash_unique ON task (project, hash64);
CREATE TABLE worker_verifies_task
(
verification_hash BIGINT NOT NULL,
task BIGINT REFERENCES task (id) ON DELETE CASCADE NOT NULL,
task INT REFERENCES task (id) ON DELETE CASCADE NOT NULL,
worker INT REFERENCES worker (id) NOT NULL
);
CREATE INDEX task_index ON worker_verifies_task (task);
CREATE TABLE log_entry
(
level INTEGER NOT NULL,
@ -150,7 +158,7 @@ $$
DECLARE
res INT = NULL;
BEGIN
DELETE FROM task WHERE id = tid AND assignee = wid AND verification_count < 2 RETURNING project INTO res;
DELETE FROM task WHERE id = tid AND assignee = wid AND verification_count = 1 RETURNING project INTO res;
IF res IS NULL THEN
INSERT INTO worker_verifies_task (worker, verification_hash, task)
@ -171,7 +179,7 @@ BEGIN
LIMIT 1) >= task.verification_count RETURNING task.id INTO res;
IF res IS NULL THEN
UPDATE task SET assignee= NULL WHERE id = tid AND assignee = wid;
UPDATE task SET assignee=NULL WHERE id = tid AND assignee = wid;
end if;
end if;

17
storage/database.go
View File

@ -15,6 +15,12 @@ type Database struct {
saveTaskStmt *sql.Stmt
workerCache map[int64]*Worker
projectCache map[int64]*Project
// [worker][project]
submitAccessCache map[int64]map[int64]bool
assignAccessCache map[int64]map[int64]bool
assignMutex *sync.Mutex
}
@ -22,6 +28,9 @@ func New() *Database {
d := Database{}
d.workerCache = make(map[int64]*Worker)
d.projectCache = make(map[int64]*Project)
d.assignAccessCache = make(map[int64]map[int64]bool)
d.submitAccessCache = make(map[int64]map[int64]bool)
d.assignMutex = &sync.Mutex{}
d.init()
@ -68,10 +77,12 @@ func (database *Database) getDB() *sql.DB {
db.SetMaxOpenConns(50)
database.db = db
} else {
err := database.db.Ping()
handleErr(err)
}
return database.db
}
func (database *Database) invalidateAccessCache(worker int64) {
delete(database.submitAccessCache, worker)
delete(database.assignAccessCache, worker)
}

12
storage/project.go
View File

@ -55,11 +55,17 @@ func (database *Database) SaveProject(project *Project, webhookSecret string) (i
"project": project,
}).Trace("Database.saveProject INSERT project")
database.projectCache[id] = project
return id, nil
}
func (database *Database) GetProject(id int64) *Project {
if database.projectCache[id] != nil {
return database.projectCache[id]
}
db := database.getDB()
row := db.QueryRow(`SELECT id, priority, name, clone_url, git_repo, version,
motd, public, hidden, COALESCE(chain, 0), paused, assign_rate, submit_rate
@ -76,7 +82,9 @@ func (database *Database) GetProject(id int64) *Project {
logrus.WithFields(logrus.Fields{
"id": id,
"project": project,
}).Trace("Database.saveProject SELECT project")
}).Trace("Database.getProject SELECT project")
database.projectCache[id] = project
return project
}
@ -132,6 +140,8 @@ func (database *Database) UpdateProject(project *Project) error {
"rowsAffected": rowsAffected,
}).Trace("Database.updateProject UPDATE project")
database.projectCache[project.Id] = project
return nil
}

172
storage/task.go
View File

@ -1,23 +1,24 @@
package storage
import (
"database/sql"
"errors"
"fmt"
"github.com/lib/pq"
"github.com/sirupsen/logrus"
)
type Task struct {
Id int64 `json:"id"`
Priority int64 `json:"priority"`
Priority int16 `json:"priority"`
Project *Project `json:"project"`
Assignee int64 `json:"assignee"`
Retries int64 `json:"retries"`
MaxRetries int64 `json:"max_retries"`
Retries int16 `json:"retries"`
MaxRetries int16 `json:"max_retries"`
Status TaskStatus `json:"status"`
Recipe string `json:"recipe"`
MaxAssignTime int64 `json:"max_assign_time"`
AssignTime int64 `json:"assign_time"`
VerificationCount int64 `json:"verification_count"`
VerificationCount int16 `json:"verification_count"`
}
type TaskStatus int
@ -42,16 +43,62 @@ type SaveTaskRequest struct {
WorkerId int64
}
func (database *Database) SaveTask(task *Task, project int64, hash64 int64, wid int64) error {
func (database *Database) checkAccess(workerId, projectId int64, assign, submit bool) bool {
if database.submitAccessCache[workerId] == nil {
database.submitAccessCache[workerId] = make(map[int64]bool)
database.assignAccessCache[workerId] = make(map[int64]bool)
} else {
_, ok := database.submitAccessCache[workerId][projectId]
if ok {
if assign && !database.assignAccessCache[workerId][projectId] {
return false
}
if submit && !database.submitAccessCache[workerId][projectId] {
return false
}
return true
}
}
db := database.getDB()
res, err := db.Exec(fmt.Sprintf(`
INSERT INTO task (project, max_retries, recipe, priority, max_assign_time, hash64,verification_count)
SELECT $1,$2,$3,$4,$5,NULLIF(%d, 0),$6 FROM worker_access
WHERE role_submit AND NOT request AND worker=$7 AND project=$1`, hash64),
project, task.MaxRetries, task.Recipe, task.Priority, task.MaxAssignTime, task.VerificationCount,
wid)
row := db.QueryRow(`SELECT role_assign, role_submit FROM worker_access
WHERE worker=$1 and project=$2 AND NOT request`,
workerId, projectId)
var hasAssign, hasSubmit bool
err := row.Scan(&hasAssign, &hasSubmit)
database.submitAccessCache[workerId][projectId] = hasSubmit
database.assignAccessCache[workerId][projectId] = hasAssign
if err != nil {
return false
}
if !hasAssign && assign {
return false
}
if !hasSubmit && submit {
return false
}
return true
}
func (database *Database) SaveTask(task *Task, project int64, hash64 int64, wid int64) error {
if !database.checkAccess(wid, project, false, true) {
return errors.New("unauthorized task submit")
}
db := database.getDB()
_, err := db.Exec(`INSERT INTO task
(project, max_retries, recipe, priority, max_assign_time, hash64, verification_count)
VALUES ($1,$2,$3,$4,$5,$6,$7)`,
project, task.MaxRetries, task.Recipe, task.Priority, task.MaxAssignTime,
makeNullableInt(hash64), task.VerificationCount)
if err != nil {
logrus.WithError(err).WithFields(logrus.Fields{
"task": task,
@ -59,45 +106,55 @@ func (database *Database) SaveTask(task *Task, project int64, hash64 int64, wid
return err
}
rowsAffected, err := res.RowsAffected()
handleErr(err)
logrus.WithFields(logrus.Fields{
"rowsAffected": rowsAffected,
"task": task,
}).Trace("Database.saveTask INSERT task")
if rowsAffected == 0 {
return errors.New("unauthorized task submit")
}
return nil
}
func makeNullableInt(i int64) sql.NullInt64 {
if i == 0 {
return sql.NullInt64{Valid: false}
} else {
return sql.NullInt64{
Valid: true,
Int64: i,
}
}
}
func (database Database) BulkSaveTask(bulkSaveTaskReqs []SaveTaskRequest) []error {
if !database.checkAccess(bulkSaveTaskReqs[0].WorkerId, bulkSaveTaskReqs[0].Project,
false, true) {
return []error{errors.New("unauthorized task submit")}
}
db := database.getDB()
txn, err := db.Begin()
handleErr(err)
errs := make([]error, len(bulkSaveTaskReqs))
for i, req := range bulkSaveTaskReqs {
res, err := db.Exec(fmt.Sprintf(`
INSERT INTO task (project, max_retries, recipe, priority, max_assign_time, hash64,verification_count)
SELECT $1,$2,$3,$4,$5,NULLIF(%d, 0),$6 FROM worker_access
WHERE role_submit AND NOT request AND worker=$7 AND project=$1`, req.Hash64),
req.Project, req.Task.MaxRetries, req.Task.Recipe, req.Task.Priority,
req.Task.MaxAssignTime, req.Task.VerificationCount,
req.WorkerId)
errs[i] = err
stmt, _ := txn.Prepare(pq.CopyIn(
"task",
"project", "max_retries", "recipe", "priority",
"max_assign_time", "hash64", "verification_count",
))
if res != nil {
rowsAffected, _ := res.RowsAffected()
if rowsAffected == 0 {
errs[i] = errors.New("unauthorized task submit")
}
for i, req := range bulkSaveTaskReqs {
_, err = stmt.Exec(req.Project, req.Task.MaxRetries, req.Task.Recipe,
req.Task.Priority, req.Task.MaxAssignTime, makeNullableInt(req.Hash64),
req.Task.VerificationCount)
if err != nil {
errs[i] = err
}
}
_, err = stmt.Exec()
err = stmt.Close()
handleErr(err)
err = txn.Commit()
handleErr(err)
return errs
}
@ -107,7 +164,7 @@ func (database Database) ReleaseTask(id int64, workerId int64, result TaskResult
var taskUpdated bool
if result == TR_OK {
row := db.QueryRow(fmt.Sprintf(`SELECT release_task_ok(%d,%d,%d)`, workerId, id, verification))
row := db.QueryRow(`SELECT release_task_ok($1,$2,$3)`, workerId, id, verification)
err := row.Scan(&taskUpdated)
if err != nil {
@ -128,13 +185,6 @@ func (database Database) ReleaseTask(id int64, workerId int64, result TaskResult
taskUpdated = rowsAffected == 1
}
logrus.WithFields(logrus.Fields{
"taskUpdated": taskUpdated,
"taskId": id,
"workerId": workerId,
"verification": verification,
}).Trace("Database.ReleaseTask")
return taskUpdated
}
@ -161,35 +211,21 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T
ORDER BY task.priority DESC
LIMIT 1
)
RETURNING task.id`, worker.Id, projectId)
RETURNING task.id, task.priority, assignee, retries, max_retries,
status, recipe, max_assign_time, assign_time, verification_count`, worker.Id, projectId)
var id int64
err := row.Scan(&id)
database.assignMutex.Unlock()
task := &Task{}
err := row.Scan(&task.Id, &task.Priority, &task.Assignee,
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
&task.AssignTime, &task.VerificationCount)
if err != nil {
return nil
}
row = db.QueryRow(`
SELECT task.id, task.priority, task.project, assignee, retries, max_retries,
status, recipe, max_assign_time, assign_time, verification_count, project.priority, project.name,
project.clone_url, project.git_repo, project.version, project.motd, project.public, COALESCE(project.chain,0),
project.assign_rate, project.submit_rate
FROM task
INNER JOIN project project ON task.project = project.id
WHERE task.id=$1`, id)
project := &Project{}
task := &Task{}
task.Project = project
err = row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
&task.AssignTime, &task.VerificationCount, &project.Priority, &project.Name,
&project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public,
&project.Chain, &project.AssignRate, &project.SubmitRate)
handleErr(err)
task.Project = database.GetProject(projectId)
return task
}

28
storage/worker.go
View File

@ -71,31 +71,6 @@ func (database *Database) GetWorker(id int64) *Worker {
return worker
}
func (database *Database) GrantAccess(workerId int64, projectId int64) bool {
db := database.getDB()
res, err := db.Exec(`UPDATE worker_access SET
request=FALSE WHERE worker=$1 AND project=$2`,
workerId, projectId)
if err != nil {
logrus.WithFields(logrus.Fields{
"workerId": workerId,
"projectId": projectId,
}).WithError(err).Warn("Database.GrantAccess INSERT")
return false
}
rowsAffected, _ := res.RowsAffected()
logrus.WithFields(logrus.Fields{
"rowsAffected": rowsAffected,
"workerId": workerId,
"projectId": projectId,
}).Trace("Database.GrantAccess INSERT")
return rowsAffected == 1
}
func (database *Database) UpdateWorker(worker *Worker) bool {
db := database.getDB()
@ -139,6 +114,7 @@ func (database *Database) SaveAccessRequest(wa *WorkerAccess) bool {
func (database *Database) AcceptAccessRequest(worker int64, projectId int64) bool {
db := database.getDB()
database.invalidateAccessCache(worker)
res, err := db.Exec(`UPDATE worker_access SET request=FALSE
WHERE worker=$1 AND project=$2`, worker, projectId)
@ -156,6 +132,8 @@ func (database *Database) AcceptAccessRequest(worker int64, projectId int64) boo
func (database *Database) RejectAccessRequest(workerId int64, projectId int64) bool {
db := database.getDB()
database.invalidateAccessCache(workerId)
res, err := db.Exec(`DELETE FROM worker_access WHERE worker=$1 AND project=$2`,
workerId, projectId)
handleErr(err)

5
test/api_project_test.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/simon987/task_tracker/api"
"github.com/simon987/task_tracker/storage"
"golang.org/x/time/rate"
"io/ioutil"
"net/http"
"testing"
@ -136,7 +137,7 @@ func TestUpdateProjectValid(t *testing.T) {
Hidden: true,
Paused: true,
AssignRate: 1,
SubmitRate: 2,
SubmitRate: 0,
Version: "VersionB",
}, pid, testAdminCtx)
@ -173,7 +174,7 @@ func TestUpdateProjectValid(t *testing.T) {
if proj.Project.AssignRate != 1 {
t.Error()
}
if proj.Project.SubmitRate != 2 {
if proj.Project.SubmitRate != rate.Inf {
t.Error()
}
}

41
test/api_task_bench_test.go
View File

@ -2,6 +2,7 @@ package test
import (
"github.com/simon987/task_tracker/api"
"github.com/simon987/task_tracker/storage"
"strconv"
"testing"
)
@ -17,6 +18,13 @@ func BenchmarkCreateTaskRemote(b *testing.B) {
worker := genWid()
requestAccess(api.CreateWorkerAccessRequest{
Submit: true,
Assign: false,
Project: resp.Content.Id,
}, worker)
acceptAccessRequest(resp.Content.Id, worker.Id, testAdminCtx)
b.ResetTimer()
for i := 0; i < b.N; i++ {
createTask(api.SubmitTaskRequest{
@ -27,3 +35,36 @@ func BenchmarkCreateTaskRemote(b *testing.B) {
}, worker)
}
}
func BenchmarkCreateTask(b *testing.B) {
resp := createProjectAsAdmin(api.CreateProjectRequest{
Name: "BenchmarkCreateTask" + strconv.Itoa(b.N),
GitRepo: "benchmark_test" + strconv.Itoa(b.N),
Version: "f09e8c9r0w839x0c43",
CloneUrl: "http://localhost",
})
worker := genWid()
requestAccess(api.CreateWorkerAccessRequest{
Submit: true,
Assign: false,
Project: resp.Content.Id,
}, worker)
acceptAccessRequest(resp.Content.Id, worker.Id, testAdminCtx)
db := storage.New()
b.ResetTimer()
p := db.GetProject(resp.Content.Id)
for i := 0; i < b.N; i++ {
db.SaveTask(&storage.Task{
Project: p,
Priority: 0,
Recipe: "{}",
MaxRetries: 1,
}, resp.Content.Id, 0, worker.Id)
}
}

52
test/api_task_test.go
View File

@ -405,6 +405,7 @@ func TestReleaseTaskSuccess(t *testing.T) {
Project: pid,
Recipe: "{}",
MaxRetries: 3,
Hash64: math.MaxInt64,
}, worker)
task := getTaskFromProject(pid, worker).Content.Task
@ -923,25 +924,20 @@ func TestTaskSubmitInvalidDoesntGiveRateLimit(t *testing.T) {
func TestBulkTaskSubmitValid(t *testing.T) {
proj := createProjectAsAdmin(api.CreateProjectRequest{
Name: "testbulksubmit",
CloneUrl: "testbulkprojectsubmit",
GitRepo: "testbulkprojectsubmit",
}).Content.Id
r := bulkSubmitTask(api.BulkSubmitTaskRequest{
Requests: []api.SubmitTaskRequest{
{
Recipe: "1234",
Project: proj,
Project: testProject,
},
{
Recipe: "1234",
Project: proj,
Project: testProject,
},
{
Recipe: "1234",
Project: proj,
Project: testProject,
Hash64: 8565956259293726066,
},
},
}, testWorker)
@ -1015,6 +1011,44 @@ func TestBulkTaskSubmitInvalid2(t *testing.T) {
}
}
func TestTaskGetUnauthorizedWithCache(t *testing.T) {
pid := createProjectAsAdmin(api.CreateProjectRequest{
Name: "testtaskgetunauthorizedcache",
GitRepo: "testtaskgetunauthorizedcache",
CloneUrl: "testtaskgettunauthorizedcache",
}).Content.Id
w := genWid()
requestAccess(api.CreateWorkerAccessRequest{
Project: pid,
Submit: true,
Assign: true,
}, w)
acceptAccessRequest(pid, w.Id, testAdminCtx)
r1 := createTask(api.SubmitTaskRequest{
Project: pid,
Recipe: "ssss",
}, w)
// removed access, cache should be invalidated
rejectAccessRequest(pid, w.Id, testAdminCtx)
r2 := createTask(api.SubmitTaskRequest{
Project: pid,
Recipe: "ssss",
}, w)
if r1.Ok != true {
t.Error()
}
if r2.Ok != false {
t.Error()
}
}
func bulkSubmitTask(request api.BulkSubmitTaskRequest, worker *storage.Worker) (ar api.JsonResponse) {
r := Post("/task/bulk_submit", request, worker, nil)
UnmarshalResponse(r, &ar)

21
test/schema.sql Executable file → Normal file
View File

@ -41,6 +41,8 @@ CREATE TABLE worker_access
request boolean,
primary key (worker, project)
);
CREATE INDEX worker_index ON worker_access (worker);
CREATE INDEX project_index ON worker_access (project);
CREATE TABLE task
(
@ -50,22 +52,28 @@ CREATE TABLE task
assignee INTEGER REFERENCES worker (id),
max_assign_time INTEGER DEFAULT 0,
assign_time INTEGER DEFAULT NULL,
verification_count INTEGER DEFAULT 0,
verification_count SMALLINT DEFAULT 0,
priority SMALLINT DEFAULT 0,
retries SMALLINT DEFAULT 0,
max_retries SMALLINT,
status SMALLINT DEFAULT 1,
recipe TEXT,
UNIQUE (project, hash64)
recipe TEXT
);
CREATE INDEX priority_desc_index ON task (priority DESC);
CREATE INDEX assignee_index ON task (assignee);
CREATE INDEX verifcnt_index ON task (verification_count);
CREATE UNIQUE INDEX project_hash_unique ON task (project, hash64);
CREATE TABLE worker_verifies_task
(
verification_hash BIGINT NOT NULL,
task BIGINT REFERENCES task (id) ON DELETE CASCADE NOT NULL,
task INT REFERENCES task (id) ON DELETE CASCADE NOT NULL,
worker INT REFERENCES worker (id) NOT NULL
);
CREATE INDEX task_index ON worker_verifies_task (task);
CREATE TABLE log_entry
(
level INTEGER NOT NULL,
@ -150,7 +158,7 @@ $$
DECLARE
res INT = NULL;
BEGIN
DELETE FROM task WHERE id = tid AND assignee = wid AND verification_count < 2 RETURNING project INTO res;
DELETE FROM task WHERE id = tid AND assignee = wid AND verification_count = 1 RETURNING project INTO res;
IF res IS NULL THEN
INSERT INTO worker_verifies_task (worker, verification_hash, task)
@ -171,11 +179,10 @@ BEGIN
LIMIT 1) >= task.verification_count RETURNING task.id INTO res;
IF res IS NULL THEN
UPDATE task SET assignee= NULL WHERE id = tid AND assignee = wid;
UPDATE task SET assignee=NULL WHERE id = tid AND assignee = wid;
end if;
end if;
RETURN res IS NOT NULL;
END;
$$ LANGUAGE 'plpgsql';

0
web/angular/e2e/protractor.conf.js Executable file → Normal file
View File

0
web/angular/e2e/src/app.e2e-spec.ts Executable file → Normal file
View File

0
web/angular/e2e/src/app.po.ts Executable file → Normal file
View File

0
web/angular/e2e/tsconfig.e2e.json Executable file → Normal file
View File

63
web/angular/package-lock.json generated
View File

@ -4323,9 +4323,9 @@
}
},
"fstream": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.11.tgz",
"integrity": "sha1-XB+x8RdHcRTwYyoOtLcbPLD9MXE=",
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
"integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
"dev": true,
"optional": true,
"requires": {
@ -6050,9 +6050,9 @@
}
},
"lodash": {
"version": "4.17.11",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
},
"lodash.clonedeep": {
"version": "4.5.0",
@ -6519,9 +6519,9 @@
}
},
"mixin-deep": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
"integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
"integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
"dev": true,
"requires": {
"for-in": "^1.0.2",
@ -8514,9 +8514,9 @@
"dev": true
},
"set-value": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
"integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
"integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
"dev": true,
"requires": {
"extend-shallow": "^2.0.1",
@ -9366,14 +9366,14 @@
"dev": true
},
"tar": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz",
"integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=",
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz",
"integrity": "sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==",
"dev": true,
"optional": true,
"requires": {
"block-stream": "*",
"fstream": "^1.0.2",
"fstream": "^1.0.12",
"inherits": "2"
}
},
@ -9881,38 +9881,15 @@
"dev": true
},
"union-value": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
"integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
"integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
"dev": true,
"requires": {
"arr-union": "^3.1.0",
"get-value": "^2.0.6",
"is-extendable": "^0.1.1",
"set-value": "^0.4.3"
},
"dependencies": {
"extend-shallow": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
"integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
"dev": true,
"requires": {
"is-extendable": "^0.1.0"
}
},
"set-value": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
"integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
"dev": true,
"requires": {
"extend-shallow": "^2.0.1",
"is-extendable": "^0.1.1",
"is-plain-object": "^2.0.1",
"to-object-path": "^0.3.0"
}
}
"set-value": "^2.0.1"
}
},
"unique-filename": {

2
web/angular/package.json
View File

@ -25,7 +25,7 @@
"@ngx-translate/http-loader": "^4.0.0",
"chart.js": "^2.7.3",
"core-js": "^2.5.4",
"lodash": "^4.17.11",
"lodash": "^4.17.15",
"moment": "^2.23.0",
"rxjs": "~6.3.3",
"tslib": "^1.9.0",

0
web/angular/src/app/api.service.ts Executable file → Normal file
View File

0
web/angular/src/app/app-routing.module.ts Executable file → Normal file
View File

0
web/angular/src/app/app.component.css Executable file → Normal file
View File

0
web/angular/src/app/app.component.html Executable file → Normal file
View File

0
web/angular/src/app/app.module.ts Executable file → Normal file
View File

0
web/angular/src/app/project-list/project-list.component.css Executable file → Normal file
View File

0
web/angular/src/app/project-list/project-list.component.html Executable file → Normal file
View File

0
web/angular/src/app/project-list/project-list.component.ts Executable file → Normal file
View File

2
web/angular/src/assets/i18n/en.json
View File

@ -115,7 +115,7 @@
"subtitle": "Real-time data for all projects",
"paused": "(paused)",
"pause": "Pause",
"manage": "Manager workers"
"manage": "Manage workers"
},
"perms": {
"title": "Project permissions",