simon987/task_tracker
1
0
Fork 0
mirror of https://github.com/simon987/task_tracker.git synced 2025-04-19 02:06:45 +00:00
Code Issues Projects Releases Wiki Activity

bunch of probably useless micro optimisation

Browse Source
This commit is contained in:
simon987 2019-01-29 21:35:31 -05:00
parent 58f20aa33d
commit d3188c512d
11 changed files with 153 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/log.go
View File

@ -13,8 +13,8 @@ import (
type RequestHandler func(*Request) type RequestHandler func(*Request)
type GetLogRequest struct { type GetLogRequest struct {
Level logrus.Level `json:"level"` Level storage.LogLevel `json:"level"`
Since int64 `json:"since"` Since int64 `json:"since"`
} }
type LogRequest struct { type LogRequest struct {

4
api/task.go
View File

@ -75,8 +75,8 @@ func (api *WebAPI) TaskCreate(r *Request) {
if err != nil { if err != nil {
r.Json(CreateTaskResponse{ r.Json(CreateTaskResponse{
Ok: false, Ok: false,
Message: err.Error(), //todo: hide sensitive error? Message: err.Error(),
}, 500) }, 400)
} else { } else {
r.OkJson(CreateTaskResponse{ r.OkJson(CreateTaskResponse{
Ok: true, Ok: true,

32
schema.sql
View File

@ -3,17 +3,6 @@ DROP TABLE IF EXISTS worker_identity, worker, project, task, log_entry,
DROP TYPE IF EXISTS status; DROP TYPE IF EXISTS status;
DROP TYPE IF EXISTS log_level; DROP TYPE IF EXISTS log_level;
CREATE TYPE status as ENUM (
'new',
'failed',
'closed',
'timeout'
);
CREATE TYPE log_level as ENUM (
'fatal', 'panic', 'error', 'warning', 'info', 'debug', 'trace'
);
CREATE TABLE worker_identity CREATE TABLE worker_identity
( (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
@ -53,25 +42,24 @@ CREATE TABLE worker_has_access_to_project
CREATE TABLE task CREATE TABLE task
( (
hash64 BIGINT DEFAULT NULL UNIQUE,
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
priority INTEGER DEFAULT 0,
project INTEGER REFERENCES project (id), project INTEGER REFERENCES project (id),
assignee INTEGER REFERENCES worker (id), assignee INTEGER REFERENCES worker (id),
retries INTEGER DEFAULT 0, max_assign_time INTEGER DEFAULT 0,
max_retries INTEGER, assign_time INTEGER DEFAULT 0,
status Status DEFAULT 'new', priority SMALLINT DEFAULT 0,
recipe TEXT, retries SMALLINT DEFAULT 0,
max_assign_time INTEGER DEFAULT 0, max_retries SMALLINT,
assign_time INTEGER DEFAULT 0, status SMALLINT DEFAULT 1,
hash64 BIGINT DEFAULT NULL UNIQUE recipe TEXT
); );
CREATE TABLE log_entry CREATE TABLE log_entry
( (
level log_level, level INTEGER,
message TEXT, message TEXT,
message_data TEXT, message_data TEXT,
timestamp INT timestamp INTEGER
); );

3
storage/database.go
View File

@ -10,7 +10,8 @@ import (
) )
type Database struct { type Database struct {
db *sql.DB db *sql.DB
saveTaskStmt *sql.Stmt
} }
func (database *Database) Reset() { func (database *Database) Reset() {

46
storage/log.go
View File

@ -7,11 +7,24 @@ import (
) )
type LogEntry struct { type LogEntry struct {
Message string `json:"message"` Message string `json:"message"`
Timestamp int64 `json:"timestamp"` Timestamp int64 `json:"timestamp"`
Data string `json:"data"` Data string `json:"data"`
Level string `json:"level"` Level LogLevel `json:"level"`
} }
type LogLevel int
const (
FATAL LogLevel = 1
PANIC LogLevel = 2
ERROR LogLevel = 3
WARN LogLevel = 4
INFO LogLevel = 5
DEBUG LogLevel = 6
TRACE LogLevel = 7
)
type sqlLogHook struct { type sqlLogHook struct {
database *Database database *Database
} }
@ -29,8 +42,27 @@ func (h sqlLogHook) Fire(entry *logrus.Entry) error {
return err return err
} }
var logLevel LogLevel
switch entry.Level {
case logrus.TraceLevel:
logLevel = TRACE
case logrus.DebugLevel:
logLevel = DEBUG
case logrus.InfoLevel:
logLevel = INFO
case logrus.WarnLevel:
logLevel = WARN
case logrus.ErrorLevel:
logLevel = ERROR
case logrus.FatalLevel:
logLevel = FATAL
case logrus.PanicLevel:
logLevel = PANIC
}
_, err = db.Exec("INSERT INTO log_entry (message, level, message_data, timestamp) VALUES ($1,$2,$3,$4)", _, err = db.Exec("INSERT INTO log_entry (message, level, message_data, timestamp) VALUES ($1,$2,$3,$4)",
entry.Message, entry.Level.String(), jsonData, entry.Time.Unix()) entry.Message, logLevel, jsonData, entry.Time.Unix())
return err return err
} }
@ -40,14 +72,14 @@ func (database *Database) SetupLoggerHook() {
logrus.AddHook(hook) logrus.AddHook(hook)
} }
func (database *Database) GetLogs(since int64, level logrus.Level) *[]LogEntry { func (database *Database) GetLogs(since int64, level LogLevel) *[]LogEntry {
db := database.getDB() db := database.getDB()
var logs []LogEntry var logs []LogEntry
rows, err := db.Query("SELECT * FROM log_entry WHERE timestamp > $1 AND level=$2", rows, err := db.Query("SELECT * FROM log_entry WHERE timestamp > $1 AND level=$2",
since, level.String()) since, level)
handleErr(err) handleErr(err)
for rows.Next() { for rows.Next() {

12
storage/project.go
View File

@ -145,9 +145,9 @@ func (database *Database) GetProjectStats(id int64) *ProjectStats {
if stats.Project != nil { if stats.Project != nil {
row := db.QueryRow(`SELECT row := db.QueryRow(`SELECT
SUM(CASE WHEN status='new' THEN 1 ELSE 0 END) newCount, SUM(CASE WHEN status=1 THEN 1 ELSE 0 END) newCount,
SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) failedCount, SUM(CASE WHEN status=2 THEN 1 ELSE 0 END) failedCount,
SUM(CASE WHEN status='closed' THEN 1 ELSE 0 END) closedCount SUM(CASE WHEN status=3 THEN 1 ELSE 0 END) closedCount
FROM task WHERE project=$1 GROUP BY project`, id) FROM task WHERE project=$1 GROUP BY project`, id)
err := row.Scan(&stats.NewTaskCount, &stats.FailedTaskCount, &stats.ClosedTaskCount) err := row.Scan(&stats.NewTaskCount, &stats.FailedTaskCount, &stats.ClosedTaskCount)
@ -188,9 +188,9 @@ func (database Database) GetAllProjectsStats() *[]ProjectStats {
db := database.getDB() db := database.getDB()
rows, err := db.Query(`SELECT rows, err := db.Query(`SELECT
SUM(CASE WHEN status='new' THEN 1 ELSE 0 END) newCount, SUM(CASE WHEN status= 1 THEN 1 ELSE 0 END) newCount,
SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) failedCount, SUM(CASE WHEN status=2 THEN 1 ELSE 0 END) failedCount,
SUM(CASE WHEN status='closed' THEN 1 ELSE 0 END) closedCount, SUM(CASE WHEN status=3 THEN 1 ELSE 0 END) closedCount,
p.* p.*
FROM task RIGHT JOIN project p on task.project = p.id FROM task RIGHT JOIN project p on task.project = p.id
GROUP BY p.id ORDER BY p.name`) GROUP BY p.id ORDER BY p.name`)

62
storage/task.go
View File

@ -7,18 +7,27 @@ import (
) )
type Task struct { type Task struct {
Id int64 `json:"id"` Id int64 `json:"id"`
Priority int64 `json:"priority"` Priority int64 `json:"priority"`
Project *Project `json:"project"` Project *Project `json:"project"`
Assignee int64 `json:"assignee"` Assignee int64 `json:"assignee"`
Retries int64 `json:"retries"` Retries int64 `json:"retries"`
MaxRetries int64 `json:"max_retries"` MaxRetries int64 `json:"max_retries"`
Status string `json:"status"` Status TaskStatus `json:"status"`
Recipe string `json:"recipe"` Recipe string `json:"recipe"`
MaxAssignTime int64 `json:"max_assign_time"` MaxAssignTime int64 `json:"max_assign_time"`
AssignTime int64 `json:"assign_time"` AssignTime int64 `json:"assign_time"`
} }
type TaskStatus int
const (
NEW TaskStatus = 1
FAILED TaskStatus = 2
CLOSED TaskStatus = 3
TIMEOUT TaskStatus = 4
)
func (database *Database) SaveTask(task *Task, project int64, hash64 int64) error { func (database *Database) SaveTask(task *Task, project int64, hash64 int64) error {
db := database.getDB() db := database.getDB()
@ -58,7 +67,7 @@ func (database *Database) GetTask(worker *Worker) *Task {
SELECT task.id SELECT task.id
FROM task FROM task
INNER JOIN project p on task.project = p.id INNER JOIN project p on task.project = p.id
WHERE assignee IS NULL AND task.status='new' WHERE assignee IS NULL AND task.status=1
AND (p.public OR EXISTS ( AND (p.public OR EXISTS (
SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=p.id SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=p.id
)) ))
@ -93,7 +102,15 @@ func getTaskById(id int64, db *sql.DB) *Task {
status, recipe, max_assign_time, assign_time, project.* FROM task status, recipe, max_assign_time, assign_time, project.* FROM task
INNER JOIN project ON task.project = project.id INNER JOIN project ON task.project = project.id
WHERE task.id=$1`, id) WHERE task.id=$1`, id)
task := scanTask(row) project := &Project{}
task := &Task{}
task.Project = project
err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
&task.AssignTime, &project.Id, &project.Priority, &project.Name,
&project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public)
handleErr(err)
logrus.WithFields(logrus.Fields{ logrus.WithFields(logrus.Fields{
"id": id, "id": id,
@ -110,11 +127,11 @@ func (database Database) ReleaseTask(id int64, workerId int64, success bool) boo
var res sql.Result var res sql.Result
var err error var err error
if success { if success {
res, err = db.Exec(`UPDATE task SET (status, assignee) = ('closed', NULL) res, err = db.Exec(`UPDATE task SET (status, assignee) = (3, NULL)
WHERE id=$1 AND task.assignee=$2`, id, workerId) WHERE id=$1 AND task.assignee=$2`, id, workerId)
} else { } else {
res, err = db.Exec(`UPDATE task SET (status, assignee, retries) = res, err = db.Exec(`UPDATE task SET (status, assignee, retries) =
(CASE WHEN retries+1 >= max_retries THEN 'failed' ELSE 'new' END, NULL, retries+1) (CASE WHEN retries+1 >= max_retries THEN 2 ELSE 1 END, NULL, retries+1)
WHERE id=$1 AND assignee=$2`, id, workerId) WHERE id=$1 AND assignee=$2`, id, workerId)
} }
handleErr(err) handleErr(err)
@ -140,7 +157,7 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T
SELECT task.id SELECT task.id
FROM task FROM task
INNER JOIN project p on task.project = p.id INNER JOIN project p on task.project = p.id
WHERE assignee IS NULL AND p.id=$2 AND status='new' WHERE assignee IS NULL AND p.id=$2 AND status=1
AND (p.public OR EXISTS ( AND (p.public OR EXISTS (
SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=$2 SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=$2
)) ))
@ -167,18 +184,3 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T
return task return task
} }
func scanTask(row *sql.Row) *Task {
project := &Project{}
task := &Task{}
task.Project = project
err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
&task.AssignTime, &project.Id, &project.Priority, &project.Name,
&project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public)
handleErr(err)
return task
}

7
test/api_log_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus"
"io/ioutil" "io/ioutil"
"src/task_tracker/api" "src/task_tracker/api"
"src/task_tracker/storage"
"testing" "testing"
"time" "time"
) )
@ -134,7 +135,7 @@ func TestGetLogs(t *testing.T) {
"test": "value", "test": "value",
}).Error("error") }).Error("error")
r := getLogs(time.Now().Add(time.Second*-150).Unix(), logrus.DebugLevel) r := getLogs(time.Now().Add(time.Second*-150).Unix(), storage.DEBUG)
if r.Ok != true { if r.Ok != true {
t.Error() t.Error()
@ -162,7 +163,7 @@ func TestGetLogs(t *testing.T) {
func TestGetLogsInvalid(t *testing.T) { func TestGetLogsInvalid(t *testing.T) {
r := getLogs(-1, logrus.ErrorLevel) r := getLogs(-1, storage.ERROR)
if r.Ok != false { if r.Ok != false {
t.Error() t.Error()
@ -173,7 +174,7 @@ func TestGetLogsInvalid(t *testing.T) {
} }
} }
func getLogs(since int64, level logrus.Level) *api.GetLogResponse { func getLogs(since int64, level storage.LogLevel) *api.GetLogResponse {
r := Post(fmt.Sprintf("/logs"), api.GetLogRequest{ r := Post(fmt.Sprintf("/logs"), api.GetLogRequest{
Since: since, Since: since,

45
test/api_task_bench_test.go
View File

@ -2,11 +2,13 @@ package test
import ( import (
"src/task_tracker/api" "src/task_tracker/api"
"src/task_tracker/config"
"src/task_tracker/storage"
"strconv" "strconv"
"testing" "testing"
) )
func BenchmarkCreateTask(b *testing.B) { func BenchmarkCreateTaskRemote(b *testing.B) {
resp := createProject(api.CreateProjectRequest{ resp := createProject(api.CreateProjectRequest{
Name: "BenchmarkCreateTask" + strconv.Itoa(b.N), Name: "BenchmarkCreateTask" + strconv.Itoa(b.N),
@ -27,3 +29,44 @@ func BenchmarkCreateTask(b *testing.B) {
}, worker) }, worker)
} }
} }
func BenchmarkCreateTask(b *testing.B) {
config.SetupConfig()
db := storage.Database{}
project, _ := db.SaveProject(&storage.Project{
Priority: 1,
Id: 1,
Version: "bmcreatetask",
Public: true,
Motd: "bmcreatetask",
Name: "BenchmarkCreateTask" + strconv.Itoa(b.N),
GitRepo: "benchmark_test" + strconv.Itoa(b.N),
})
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = db.SaveTask(&storage.Task{}, project, 0)
}
}
func TestTest(t *testing.T) {
config.SetupConfig()
db := storage.Database{}
project, _ := db.SaveProject(&storage.Project{
Priority: 1,
Id: 1,
Version: "bmcreatetask",
Public: true,
Motd: "bmcreatetask",
Name: "BenchmarkCreateTask",
GitRepo: "benchmark_test",
})
for i := 0; i < 1000000; i++ {
_ = db.SaveTask(&storage.Task{}, project, 0)
}
}

4
test/api_task_test.go
View File

@ -162,10 +162,10 @@ func TestCreateGetTask(t *testing.T) {
if taskResp.Task.Id == 0 { if taskResp.Task.Id == 0 {
t.Error() t.Error()
} }
if taskResp.Task.Recipe != "{\"url\":\"test\"}" { if string(taskResp.Task.Recipe) != "{\"url\":\"test\"}" {
t.Error() t.Error()
} }
if taskResp.Task.Status != "new" { if taskResp.Task.Status != 1 {
t.Error() t.Error()
} }
if taskResp.Task.MaxRetries != 3 { if taskResp.Task.MaxRetries != 3 {

32
test/schema.sql
View File

@ -3,17 +3,6 @@ DROP TABLE IF EXISTS worker_identity, worker, project, task, log_entry,
DROP TYPE IF EXISTS status; DROP TYPE IF EXISTS status;
DROP TYPE IF EXISTS log_level; DROP TYPE IF EXISTS log_level;
CREATE TYPE status as ENUM (
'new',
'failed',
'closed',
'timeout'
);
CREATE TYPE log_level as ENUM (
'fatal', 'panic', 'error', 'warning', 'info', 'debug', 'trace'
);
CREATE TABLE worker_identity CREATE TABLE worker_identity
( (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
@ -53,25 +42,24 @@ CREATE TABLE worker_has_access_to_project
CREATE TABLE task CREATE TABLE task
( (
hash64 BIGINT DEFAULT NULL UNIQUE,
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
priority INTEGER DEFAULT 0,
project INTEGER REFERENCES project (id), project INTEGER REFERENCES project (id),
assignee INTEGER REFERENCES worker (id), assignee INTEGER REFERENCES worker (id),
retries INTEGER DEFAULT 0, max_assign_time INTEGER DEFAULT 0,
max_retries INTEGER, assign_time INTEGER DEFAULT 0,
status Status DEFAULT 'new', priority SMALLINT DEFAULT 0,
recipe TEXT, retries SMALLINT DEFAULT 0,
max_assign_time INTEGER DEFAULT 0, max_retries SMALLINT,
assign_time INTEGER DEFAULT 0, status SMALLINT DEFAULT 1,
hash64 BIGINT DEFAULT NULL UNIQUE recipe TEXT
); );
CREATE TABLE log_entry CREATE TABLE log_entry
( (
level log_level, level INTEGER,
message TEXT, message TEXT,
message_data TEXT, message_data TEXT,
timestamp INT timestamp INTEGER
); );