From d3188c512d9fad58365d4ac045e3cc916ab66b9c Mon Sep 17 00:00:00 2001 From: simon987 Date: Tue, 29 Jan 2019 21:35:31 -0500 Subject: [PATCH] bunch of probably useless micro optimisation --- api/log.go | 4 +-- api/task.go | 4 +-- schema.sql | 32 ++++++------------- storage/database.go | 3 +- storage/log.go | 46 ++++++++++++++++++++++----- storage/project.go | 12 +++---- storage/task.go | 62 +++++++++++++++++++------------------ test/api_log_test.go | 7 +++-- test/api_task_bench_test.go | 45 ++++++++++++++++++++++++++- test/api_task_test.go | 4 +-- test/schema.sql | 32 ++++++------------- 11 files changed, 153 insertions(+), 98 deletions(-) diff --git a/api/log.go b/api/log.go index 025de6b..e4d3430 100644 --- a/api/log.go +++ b/api/log.go @@ -13,8 +13,8 @@ import ( type RequestHandler func(*Request) type GetLogRequest struct { - Level logrus.Level `json:"level"` - Since int64 `json:"since"` + Level storage.LogLevel `json:"level"` + Since int64 `json:"since"` } type LogRequest struct { diff --git a/api/task.go b/api/task.go index 2cf79b0..2f56456 100644 --- a/api/task.go +++ b/api/task.go @@ -75,8 +75,8 @@ func (api *WebAPI) TaskCreate(r *Request) { if err != nil { r.Json(CreateTaskResponse{ Ok: false, - Message: err.Error(), //todo: hide sensitive error? - }, 500) + Message: err.Error(), + }, 400) } else { r.OkJson(CreateTaskResponse{ Ok: true, diff --git a/schema.sql b/schema.sql index 8770170..4f79cab 100755 --- a/schema.sql +++ b/schema.sql @@ -3,17 +3,6 @@ DROP TABLE IF EXISTS worker_identity, worker, project, task, log_entry, DROP TYPE IF EXISTS status; DROP TYPE IF EXISTS log_level; -CREATE TYPE status as ENUM ( - 'new', - 'failed', - 'closed', - 'timeout' - ); - -CREATE TYPE log_level as ENUM ( - 'fatal', 'panic', 'error', 'warning', 'info', 'debug', 'trace' - ); - CREATE TABLE worker_identity ( id SERIAL PRIMARY KEY, @@ -53,25 +42,24 @@ CREATE TABLE worker_has_access_to_project CREATE TABLE task ( + hash64 BIGINT DEFAULT NULL UNIQUE, id SERIAL PRIMARY KEY, - priority INTEGER DEFAULT 0, project INTEGER REFERENCES project (id), assignee INTEGER REFERENCES worker (id), - retries INTEGER DEFAULT 0, - max_retries INTEGER, - status Status DEFAULT 'new', - recipe TEXT, - max_assign_time INTEGER DEFAULT 0, - assign_time INTEGER DEFAULT 0, - hash64 BIGINT DEFAULT NULL UNIQUE + max_assign_time INTEGER DEFAULT 0, + assign_time INTEGER DEFAULT 0, + priority SMALLINT DEFAULT 0, + retries SMALLINT DEFAULT 0, + max_retries SMALLINT, + status SMALLINT DEFAULT 1, + recipe TEXT ); CREATE TABLE log_entry ( - level log_level, + level INTEGER, message TEXT, message_data TEXT, - timestamp INT + timestamp INTEGER ); - diff --git a/storage/database.go b/storage/database.go index 7b0ae04..24c90d7 100644 --- a/storage/database.go +++ b/storage/database.go @@ -10,7 +10,8 @@ import ( ) type Database struct { - db *sql.DB + db *sql.DB + saveTaskStmt *sql.Stmt } func (database *Database) Reset() { diff --git a/storage/log.go b/storage/log.go index 3622fba..38e0cb3 100644 --- a/storage/log.go +++ b/storage/log.go @@ -7,11 +7,24 @@ import ( ) type LogEntry struct { - Message string `json:"message"` - Timestamp int64 `json:"timestamp"` - Data string `json:"data"` - Level string `json:"level"` + Message string `json:"message"` + Timestamp int64 `json:"timestamp"` + Data string `json:"data"` + Level LogLevel `json:"level"` } + +type LogLevel int + +const ( + FATAL LogLevel = 1 + PANIC LogLevel = 2 + ERROR LogLevel = 3 + WARN LogLevel = 4 + INFO LogLevel = 5 + DEBUG LogLevel = 6 + TRACE LogLevel = 7 +) + type sqlLogHook struct { database *Database } @@ -29,8 +42,27 @@ func (h sqlLogHook) Fire(entry *logrus.Entry) error { return err } + var logLevel LogLevel + + switch entry.Level { + case logrus.TraceLevel: + logLevel = TRACE + case logrus.DebugLevel: + logLevel = DEBUG + case logrus.InfoLevel: + logLevel = INFO + case logrus.WarnLevel: + logLevel = WARN + case logrus.ErrorLevel: + logLevel = ERROR + case logrus.FatalLevel: + logLevel = FATAL + case logrus.PanicLevel: + logLevel = PANIC + } + _, err = db.Exec("INSERT INTO log_entry (message, level, message_data, timestamp) VALUES ($1,$2,$3,$4)", - entry.Message, entry.Level.String(), jsonData, entry.Time.Unix()) + entry.Message, logLevel, jsonData, entry.Time.Unix()) return err } @@ -40,14 +72,14 @@ func (database *Database) SetupLoggerHook() { logrus.AddHook(hook) } -func (database *Database) GetLogs(since int64, level logrus.Level) *[]LogEntry { +func (database *Database) GetLogs(since int64, level LogLevel) *[]LogEntry { db := database.getDB() var logs []LogEntry rows, err := db.Query("SELECT * FROM log_entry WHERE timestamp > $1 AND level=$2", - since, level.String()) + since, level) handleErr(err) for rows.Next() { diff --git a/storage/project.go b/storage/project.go index 1be93d7..72d9532 100644 --- a/storage/project.go +++ b/storage/project.go @@ -145,9 +145,9 @@ func (database *Database) GetProjectStats(id int64) *ProjectStats { if stats.Project != nil { row := db.QueryRow(`SELECT - SUM(CASE WHEN status='new' THEN 1 ELSE 0 END) newCount, - SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) failedCount, - SUM(CASE WHEN status='closed' THEN 1 ELSE 0 END) closedCount + SUM(CASE WHEN status=1 THEN 1 ELSE 0 END) newCount, + SUM(CASE WHEN status=2 THEN 1 ELSE 0 END) failedCount, + SUM(CASE WHEN status=3 THEN 1 ELSE 0 END) closedCount FROM task WHERE project=$1 GROUP BY project`, id) err := row.Scan(&stats.NewTaskCount, &stats.FailedTaskCount, &stats.ClosedTaskCount) @@ -188,9 +188,9 @@ func (database Database) GetAllProjectsStats() *[]ProjectStats { db := database.getDB() rows, err := db.Query(`SELECT - SUM(CASE WHEN status='new' THEN 1 ELSE 0 END) newCount, - SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) failedCount, - SUM(CASE WHEN status='closed' THEN 1 ELSE 0 END) closedCount, + SUM(CASE WHEN status= 1 THEN 1 ELSE 0 END) newCount, + SUM(CASE WHEN status=2 THEN 1 ELSE 0 END) failedCount, + SUM(CASE WHEN status=3 THEN 1 ELSE 0 END) closedCount, p.* FROM task RIGHT JOIN project p on task.project = p.id GROUP BY p.id ORDER BY p.name`) diff --git a/storage/task.go b/storage/task.go index 2851b3f..6d84962 100644 --- a/storage/task.go +++ b/storage/task.go @@ -7,18 +7,27 @@ import ( ) type Task struct { - Id int64 `json:"id"` - Priority int64 `json:"priority"` - Project *Project `json:"project"` - Assignee int64 `json:"assignee"` - Retries int64 `json:"retries"` - MaxRetries int64 `json:"max_retries"` - Status string `json:"status"` - Recipe string `json:"recipe"` - MaxAssignTime int64 `json:"max_assign_time"` - AssignTime int64 `json:"assign_time"` + Id int64 `json:"id"` + Priority int64 `json:"priority"` + Project *Project `json:"project"` + Assignee int64 `json:"assignee"` + Retries int64 `json:"retries"` + MaxRetries int64 `json:"max_retries"` + Status TaskStatus `json:"status"` + Recipe string `json:"recipe"` + MaxAssignTime int64 `json:"max_assign_time"` + AssignTime int64 `json:"assign_time"` } +type TaskStatus int + +const ( + NEW TaskStatus = 1 + FAILED TaskStatus = 2 + CLOSED TaskStatus = 3 + TIMEOUT TaskStatus = 4 +) + func (database *Database) SaveTask(task *Task, project int64, hash64 int64) error { db := database.getDB() @@ -58,7 +67,7 @@ func (database *Database) GetTask(worker *Worker) *Task { SELECT task.id FROM task INNER JOIN project p on task.project = p.id - WHERE assignee IS NULL AND task.status='new' + WHERE assignee IS NULL AND task.status=1 AND (p.public OR EXISTS ( SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=p.id )) @@ -93,7 +102,15 @@ func getTaskById(id int64, db *sql.DB) *Task { status, recipe, max_assign_time, assign_time, project.* FROM task INNER JOIN project ON task.project = project.id WHERE task.id=$1`, id) - task := scanTask(row) + project := &Project{} + task := &Task{} + task.Project = project + + err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee, + &task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime, + &task.AssignTime, &project.Id, &project.Priority, &project.Name, + &project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public) + handleErr(err) logrus.WithFields(logrus.Fields{ "id": id, @@ -110,11 +127,11 @@ func (database Database) ReleaseTask(id int64, workerId int64, success bool) boo var res sql.Result var err error if success { - res, err = db.Exec(`UPDATE task SET (status, assignee) = ('closed', NULL) + res, err = db.Exec(`UPDATE task SET (status, assignee) = (3, NULL) WHERE id=$1 AND task.assignee=$2`, id, workerId) } else { res, err = db.Exec(`UPDATE task SET (status, assignee, retries) = - (CASE WHEN retries+1 >= max_retries THEN 'failed' ELSE 'new' END, NULL, retries+1) + (CASE WHEN retries+1 >= max_retries THEN 2 ELSE 1 END, NULL, retries+1) WHERE id=$1 AND assignee=$2`, id, workerId) } handleErr(err) @@ -140,7 +157,7 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T SELECT task.id FROM task INNER JOIN project p on task.project = p.id - WHERE assignee IS NULL AND p.id=$2 AND status='new' + WHERE assignee IS NULL AND p.id=$2 AND status=1 AND (p.public OR EXISTS ( SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=$2 )) @@ -167,18 +184,3 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T return task } - -func scanTask(row *sql.Row) *Task { - - project := &Project{} - task := &Task{} - task.Project = project - - err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee, - &task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime, - &task.AssignTime, &project.Id, &project.Priority, &project.Name, - &project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public) - handleErr(err) - - return task -} diff --git a/test/api_log_test.go b/test/api_log_test.go index 373a85f..2680475 100644 --- a/test/api_log_test.go +++ b/test/api_log_test.go @@ -6,6 +6,7 @@ import ( "github.com/Sirupsen/logrus" "io/ioutil" "src/task_tracker/api" + "src/task_tracker/storage" "testing" "time" ) @@ -134,7 +135,7 @@ func TestGetLogs(t *testing.T) { "test": "value", }).Error("error") - r := getLogs(time.Now().Add(time.Second*-150).Unix(), logrus.DebugLevel) + r := getLogs(time.Now().Add(time.Second*-150).Unix(), storage.DEBUG) if r.Ok != true { t.Error() @@ -162,7 +163,7 @@ func TestGetLogs(t *testing.T) { func TestGetLogsInvalid(t *testing.T) { - r := getLogs(-1, logrus.ErrorLevel) + r := getLogs(-1, storage.ERROR) if r.Ok != false { t.Error() @@ -173,7 +174,7 @@ func TestGetLogsInvalid(t *testing.T) { } } -func getLogs(since int64, level logrus.Level) *api.GetLogResponse { +func getLogs(since int64, level storage.LogLevel) *api.GetLogResponse { r := Post(fmt.Sprintf("/logs"), api.GetLogRequest{ Since: since, diff --git a/test/api_task_bench_test.go b/test/api_task_bench_test.go index 4b678cc..da6d91b 100644 --- a/test/api_task_bench_test.go +++ b/test/api_task_bench_test.go @@ -2,11 +2,13 @@ package test import ( "src/task_tracker/api" + "src/task_tracker/config" + "src/task_tracker/storage" "strconv" "testing" ) -func BenchmarkCreateTask(b *testing.B) { +func BenchmarkCreateTaskRemote(b *testing.B) { resp := createProject(api.CreateProjectRequest{ Name: "BenchmarkCreateTask" + strconv.Itoa(b.N), @@ -27,3 +29,44 @@ func BenchmarkCreateTask(b *testing.B) { }, worker) } } + +func BenchmarkCreateTask(b *testing.B) { + + config.SetupConfig() + db := storage.Database{} + + project, _ := db.SaveProject(&storage.Project{ + Priority: 1, + Id: 1, + Version: "bmcreatetask", + Public: true, + Motd: "bmcreatetask", + Name: "BenchmarkCreateTask" + strconv.Itoa(b.N), + GitRepo: "benchmark_test" + strconv.Itoa(b.N), + }) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = db.SaveTask(&storage.Task{}, project, 0) + } +} + +func TestTest(t *testing.T) { + + config.SetupConfig() + db := storage.Database{} + + project, _ := db.SaveProject(&storage.Project{ + Priority: 1, + Id: 1, + Version: "bmcreatetask", + Public: true, + Motd: "bmcreatetask", + Name: "BenchmarkCreateTask", + GitRepo: "benchmark_test", + }) + + for i := 0; i < 1000000; i++ { + _ = db.SaveTask(&storage.Task{}, project, 0) + } +} diff --git a/test/api_task_test.go b/test/api_task_test.go index 309f40a..e359995 100644 --- a/test/api_task_test.go +++ b/test/api_task_test.go @@ -162,10 +162,10 @@ func TestCreateGetTask(t *testing.T) { if taskResp.Task.Id == 0 { t.Error() } - if taskResp.Task.Recipe != "{\"url\":\"test\"}" { + if string(taskResp.Task.Recipe) != "{\"url\":\"test\"}" { t.Error() } - if taskResp.Task.Status != "new" { + if taskResp.Task.Status != 1 { t.Error() } if taskResp.Task.MaxRetries != 3 { diff --git a/test/schema.sql b/test/schema.sql index 8770170..4f79cab 100755 --- a/test/schema.sql +++ b/test/schema.sql @@ -3,17 +3,6 @@ DROP TABLE IF EXISTS worker_identity, worker, project, task, log_entry, DROP TYPE IF EXISTS status; DROP TYPE IF EXISTS log_level; -CREATE TYPE status as ENUM ( - 'new', - 'failed', - 'closed', - 'timeout' - ); - -CREATE TYPE log_level as ENUM ( - 'fatal', 'panic', 'error', 'warning', 'info', 'debug', 'trace' - ); - CREATE TABLE worker_identity ( id SERIAL PRIMARY KEY, @@ -53,25 +42,24 @@ CREATE TABLE worker_has_access_to_project CREATE TABLE task ( + hash64 BIGINT DEFAULT NULL UNIQUE, id SERIAL PRIMARY KEY, - priority INTEGER DEFAULT 0, project INTEGER REFERENCES project (id), assignee INTEGER REFERENCES worker (id), - retries INTEGER DEFAULT 0, - max_retries INTEGER, - status Status DEFAULT 'new', - recipe TEXT, - max_assign_time INTEGER DEFAULT 0, - assign_time INTEGER DEFAULT 0, - hash64 BIGINT DEFAULT NULL UNIQUE + max_assign_time INTEGER DEFAULT 0, + assign_time INTEGER DEFAULT 0, + priority SMALLINT DEFAULT 0, + retries SMALLINT DEFAULT 0, + max_retries SMALLINT, + status SMALLINT DEFAULT 1, + recipe TEXT ); CREATE TABLE log_entry ( - level log_level, + level INTEGER, message TEXT, message_data TEXT, - timestamp INT + timestamp INTEGER ); -