simon987/task_tracker
1
0
Fork 0
mirror of https://github.com/simon987/task_tracker.git synced 2025-04-19 02:06:45 +00:00
Code Issues Projects Releases Wiki Activity

bunch of probably useless micro optimisation

Browse Source
This commit is contained in:
simon987 2019-01-29 21:35:31 -05:00
parent 58f20aa33d
commit d3188c512d
11 changed files with 153 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/log.go
View File

@ -13,7 +13,7 @@ import (
type RequestHandler func(*Request)
type GetLogRequest struct {
Level logrus.Level `json:"level"`
Level storage.LogLevel `json:"level"`
Since int64 `json:"since"`
}

4
api/task.go
View File

@ -75,8 +75,8 @@ func (api *WebAPI) TaskCreate(r *Request) {
if err != nil {
r.Json(CreateTaskResponse{
Ok: false,
Message: err.Error(), //todo: hide sensitive error?
}, 500)
Message: err.Error(),
}, 400)
} else {
r.OkJson(CreateTaskResponse{
Ok: true,

28
schema.sql
View File

@ -3,17 +3,6 @@ DROP TABLE IF EXISTS worker_identity, worker, project, task, log_entry,
DROP TYPE IF EXISTS status;
DROP TYPE IF EXISTS log_level;
CREATE TYPE status as ENUM (
'new',
'failed',
'closed',
'timeout'
);
CREATE TYPE log_level as ENUM (
'fatal', 'panic', 'error', 'warning', 'info', 'debug', 'trace'
);
CREATE TABLE worker_identity
(
id SERIAL PRIMARY KEY,
@ -53,25 +42,24 @@ CREATE TABLE worker_has_access_to_project
CREATE TABLE task
(
hash64 BIGINT DEFAULT NULL UNIQUE,
id SERIAL PRIMARY KEY,
priority INTEGER DEFAULT 0,
project INTEGER REFERENCES project (id),
assignee INTEGER REFERENCES worker (id),
retries INTEGER DEFAULT 0,
max_retries INTEGER,
status Status DEFAULT 'new',
recipe TEXT,
max_assign_time INTEGER DEFAULT 0,
assign_time INTEGER DEFAULT 0,
hash64 BIGINT DEFAULT NULL UNIQUE
priority SMALLINT DEFAULT 0,
retries SMALLINT DEFAULT 0,
max_retries SMALLINT,
status SMALLINT DEFAULT 1,
recipe TEXT
);
CREATE TABLE log_entry
(
level log_level,
level INTEGER,
message TEXT,
message_data TEXT,
timestamp INT
timestamp INTEGER
);

1
storage/database.go
View File

@ -11,6 +11,7 @@ import (
type Database struct {
db *sql.DB
saveTaskStmt *sql.Stmt
}
func (database *Database) Reset() {

40
storage/log.go
View File

@ -10,8 +10,21 @@ type LogEntry struct {
Message string `json:"message"`
Timestamp int64 `json:"timestamp"`
Data string `json:"data"`
Level string `json:"level"`
Level LogLevel `json:"level"`
}
type LogLevel int
const (
FATAL LogLevel = 1
PANIC LogLevel = 2
ERROR LogLevel = 3
WARN LogLevel = 4
INFO LogLevel = 5
DEBUG LogLevel = 6
TRACE LogLevel = 7
)
type sqlLogHook struct {
database *Database
}
@ -29,8 +42,27 @@ func (h sqlLogHook) Fire(entry *logrus.Entry) error {
return err
}
var logLevel LogLevel
switch entry.Level {
case logrus.TraceLevel:
logLevel = TRACE
case logrus.DebugLevel:
logLevel = DEBUG
case logrus.InfoLevel:
logLevel = INFO
case logrus.WarnLevel:
logLevel = WARN
case logrus.ErrorLevel:
logLevel = ERROR
case logrus.FatalLevel:
logLevel = FATAL
case logrus.PanicLevel:
logLevel = PANIC
}
_, err = db.Exec("INSERT INTO log_entry (message, level, message_data, timestamp) VALUES ($1,$2,$3,$4)",
entry.Message, entry.Level.String(), jsonData, entry.Time.Unix())
entry.Message, logLevel, jsonData, entry.Time.Unix())
return err
}
@ -40,14 +72,14 @@ func (database *Database) SetupLoggerHook() {
logrus.AddHook(hook)
}
func (database *Database) GetLogs(since int64, level logrus.Level) *[]LogEntry {
func (database *Database) GetLogs(since int64, level LogLevel) *[]LogEntry {
db := database.getDB()
var logs []LogEntry
rows, err := db.Query("SELECT * FROM log_entry WHERE timestamp > $1 AND level=$2",
since, level.String())
since, level)
handleErr(err)
for rows.Next() {

12
storage/project.go
View File

@ -145,9 +145,9 @@ func (database *Database) GetProjectStats(id int64) *ProjectStats {
if stats.Project != nil {
row := db.QueryRow(`SELECT
SUM(CASE WHEN status='new' THEN 1 ELSE 0 END) newCount,
SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) failedCount,
SUM(CASE WHEN status='closed' THEN 1 ELSE 0 END) closedCount
SUM(CASE WHEN status=1 THEN 1 ELSE 0 END) newCount,
SUM(CASE WHEN status=2 THEN 1 ELSE 0 END) failedCount,
SUM(CASE WHEN status=3 THEN 1 ELSE 0 END) closedCount
FROM task WHERE project=$1 GROUP BY project`, id)
err := row.Scan(&stats.NewTaskCount, &stats.FailedTaskCount, &stats.ClosedTaskCount)
@ -188,9 +188,9 @@ func (database Database) GetAllProjectsStats() *[]ProjectStats {
db := database.getDB()
rows, err := db.Query(`SELECT
SUM(CASE WHEN status='new' THEN 1 ELSE 0 END) newCount,
SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) failedCount,
SUM(CASE WHEN status='closed' THEN 1 ELSE 0 END) closedCount,
SUM(CASE WHEN status= 1 THEN 1 ELSE 0 END) newCount,
SUM(CASE WHEN status=2 THEN 1 ELSE 0 END) failedCount,
SUM(CASE WHEN status=3 THEN 1 ELSE 0 END) closedCount,
p.*
FROM task RIGHT JOIN project p on task.project = p.id
GROUP BY p.id ORDER BY p.name`)

44
storage/task.go
View File

@ -13,12 +13,21 @@ type Task struct {
Assignee int64 `json:"assignee"`
Retries int64 `json:"retries"`
MaxRetries int64 `json:"max_retries"`
Status string `json:"status"`
Status TaskStatus `json:"status"`
Recipe string `json:"recipe"`
MaxAssignTime int64 `json:"max_assign_time"`
AssignTime int64 `json:"assign_time"`
}
type TaskStatus int
const (
NEW TaskStatus = 1
FAILED TaskStatus = 2
CLOSED TaskStatus = 3
TIMEOUT TaskStatus = 4
)
func (database *Database) SaveTask(task *Task, project int64, hash64 int64) error {
db := database.getDB()
@ -58,7 +67,7 @@ func (database *Database) GetTask(worker *Worker) *Task {
SELECT task.id
FROM task
INNER JOIN project p on task.project = p.id
WHERE assignee IS NULL AND task.status='new'
WHERE assignee IS NULL AND task.status=1
AND (p.public OR EXISTS (
SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=p.id
))
@ -93,7 +102,15 @@ func getTaskById(id int64, db *sql.DB) *Task {
status, recipe, max_assign_time, assign_time, project.* FROM task
INNER JOIN project ON task.project = project.id
WHERE task.id=$1`, id)
task := scanTask(row)
project := &Project{}
task := &Task{}
task.Project = project
err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
&task.AssignTime, &project.Id, &project.Priority, &project.Name,
&project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public)
handleErr(err)
logrus.WithFields(logrus.Fields{
"id": id,
@ -110,11 +127,11 @@ func (database Database) ReleaseTask(id int64, workerId int64, success bool) boo
var res sql.Result
var err error
if success {
res, err = db.Exec(`UPDATE task SET (status, assignee) = ('closed', NULL)
res, err = db.Exec(`UPDATE task SET (status, assignee) = (3, NULL)
WHERE id=$1 AND task.assignee=$2`, id, workerId)
} else {
res, err = db.Exec(`UPDATE task SET (status, assignee, retries) =
(CASE WHEN retries+1 >= max_retries THEN 'failed' ELSE 'new' END, NULL, retries+1)
(CASE WHEN retries+1 >= max_retries THEN 2 ELSE 1 END, NULL, retries+1)
WHERE id=$1 AND assignee=$2`, id, workerId)
}
handleErr(err)
@ -140,7 +157,7 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T
SELECT task.id
FROM task
INNER JOIN project p on task.project = p.id
WHERE assignee IS NULL AND p.id=$2 AND status='new'
WHERE assignee IS NULL AND p.id=$2 AND status=1
AND (p.public OR EXISTS (
SELECT 1 FROM worker_has_access_to_project a WHERE a.worker=$1 AND a.project=$2
))
@ -167,18 +184,3 @@ func (database *Database) GetTaskFromProject(worker *Worker, projectId int64) *T
return task
}
func scanTask(row *sql.Row) *Task {
project := &Project{}
task := &Task{}
task.Project = project
err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &task.MaxAssignTime,
&task.AssignTime, &project.Id, &project.Priority, &project.Name,
&project.CloneUrl, &project.GitRepo, &project.Version, &project.Motd, &project.Public)
handleErr(err)
return task
}

7
test/api_log_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/Sirupsen/logrus"
"io/ioutil"
"src/task_tracker/api"
"src/task_tracker/storage"
"testing"
"time"
)
@ -134,7 +135,7 @@ func TestGetLogs(t *testing.T) {
"test": "value",
}).Error("error")
r := getLogs(time.Now().Add(time.Second*-150).Unix(), logrus.DebugLevel)
r := getLogs(time.Now().Add(time.Second*-150).Unix(), storage.DEBUG)
if r.Ok != true {
t.Error()
@ -162,7 +163,7 @@ func TestGetLogs(t *testing.T) {
func TestGetLogsInvalid(t *testing.T) {
r := getLogs(-1, logrus.ErrorLevel)
r := getLogs(-1, storage.ERROR)
if r.Ok != false {
t.Error()
@ -173,7 +174,7 @@ func TestGetLogsInvalid(t *testing.T) {
}
}
func getLogs(since int64, level logrus.Level) *api.GetLogResponse {
func getLogs(since int64, level storage.LogLevel) *api.GetLogResponse {
r := Post(fmt.Sprintf("/logs"), api.GetLogRequest{
Since: since,

45
test/api_task_bench_test.go
View File

@ -2,11 +2,13 @@ package test
import (
"src/task_tracker/api"
"src/task_tracker/config"
"src/task_tracker/storage"
"strconv"
"testing"
)
func BenchmarkCreateTask(b *testing.B) {
func BenchmarkCreateTaskRemote(b *testing.B) {
resp := createProject(api.CreateProjectRequest{
Name: "BenchmarkCreateTask" + strconv.Itoa(b.N),
@ -27,3 +29,44 @@ func BenchmarkCreateTask(b *testing.B) {
}, worker)
}
}
func BenchmarkCreateTask(b *testing.B) {
config.SetupConfig()
db := storage.Database{}
project, _ := db.SaveProject(&storage.Project{
Priority: 1,
Id: 1,
Version: "bmcreatetask",
Public: true,
Motd: "bmcreatetask",
Name: "BenchmarkCreateTask" + strconv.Itoa(b.N),
GitRepo: "benchmark_test" + strconv.Itoa(b.N),
})
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = db.SaveTask(&storage.Task{}, project, 0)
}
}
func TestTest(t *testing.T) {
config.SetupConfig()
db := storage.Database{}
project, _ := db.SaveProject(&storage.Project{
Priority: 1,
Id: 1,
Version: "bmcreatetask",
Public: true,
Motd: "bmcreatetask",
Name: "BenchmarkCreateTask",
GitRepo: "benchmark_test",
})
for i := 0; i < 1000000; i++ {
_ = db.SaveTask(&storage.Task{}, project, 0)
}
}

4
test/api_task_test.go
View File

@ -162,10 +162,10 @@ func TestCreateGetTask(t *testing.T) {
if taskResp.Task.Id == 0 {
t.Error()
}
if taskResp.Task.Recipe != "{\"url\":\"test\"}" {
if string(taskResp.Task.Recipe) != "{\"url\":\"test\"}" {
t.Error()
}
if taskResp.Task.Status != "new" {
if taskResp.Task.Status != 1 {
t.Error()
}
if taskResp.Task.MaxRetries != 3 {

28
test/schema.sql
View File

@ -3,17 +3,6 @@ DROP TABLE IF EXISTS worker_identity, worker, project, task, log_entry,
DROP TYPE IF EXISTS status;
DROP TYPE IF EXISTS log_level;
CREATE TYPE status as ENUM (
'new',
'failed',
'closed',
'timeout'
);
CREATE TYPE log_level as ENUM (
'fatal', 'panic', 'error', 'warning', 'info', 'debug', 'trace'
);
CREATE TABLE worker_identity
(
id SERIAL PRIMARY KEY,
@ -53,25 +42,24 @@ CREATE TABLE worker_has_access_to_project
CREATE TABLE task
(
hash64 BIGINT DEFAULT NULL UNIQUE,
id SERIAL PRIMARY KEY,
priority INTEGER DEFAULT 0,
project INTEGER REFERENCES project (id),
assignee INTEGER REFERENCES worker (id),
retries INTEGER DEFAULT 0,
max_retries INTEGER,
status Status DEFAULT 'new',
recipe TEXT,
max_assign_time INTEGER DEFAULT 0,
assign_time INTEGER DEFAULT 0,
hash64 BIGINT DEFAULT NULL UNIQUE
priority SMALLINT DEFAULT 0,
retries SMALLINT DEFAULT 0,
max_retries SMALLINT,
status SMALLINT DEFAULT 1,
recipe TEXT
);
CREATE TABLE log_entry
(
level log_level,
level INTEGER,
message TEXT,
message_data TEXT,
timestamp INT
timestamp INTEGER
);