simon987/task_tracker
1
0
Fork 0
mirror of https://github.com/simon987/task_tracker.git synced 2025-04-19 18:16:45 +00:00
Code Issues Projects Releases Wiki Activity

Handle updates via git webhooks

Browse Source
This commit is contained in:
simon987 2019-01-13 14:58:52 -05:00
parent a2b5de0e01
commit ef333b6b25
17 changed files with 514 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
api/git.go Normal file
View File

@ -0,0 +1,158 @@
package api
import (
"crypto"
"crypto/hmac"
"encoding/hex"
"encoding/json"
"fmt"
"github.com/Sirupsen/logrus"
"github.com/valyala/fasthttp"
"hash"
"src/task_tracker/config"
"src/task_tracker/storage"
"strings"
)
type GitPayload struct {
Ref string `json:"ref"`
Before string `json:"before"`
After string `json:"after"`
Repository struct {
Id int64 `json:"id"`
Owner struct {
Id int64 `json:"id"`
Username string `json:"username"`
Login string `json:"login"`
FullName string `json:"full_name"`
Email string `json:"email"`
} `json:"owner"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Fork bool `json:"fork"`
Size int64 `json:"size"`
HtmlUrl string `json:"html_url"`
SshUrl string `json:"ssh_url"`
CloneUrl string `json:"clone_url"`
DefaultBranch string `json:"default_branch"`
} `json:"repository"`
}
func (g GitPayload) String() string {
jsonBytes, _ := json.Marshal(g)
return string(jsonBytes)
}
func (api *WebAPI) ReceiveGitWebHook(r *Request) {
if !signatureValid(r) {
logrus.Error("WebHook signature does not match!")
r.Ctx.SetStatusCode(403)
_, _ = fmt.Fprintf(r.Ctx, "Signature does not match")
return
}
payload := &GitPayload{}
if r.GetJson(payload) {
logrus.WithFields(logrus.Fields{
"payload": payload,
}).Info("Received git WebHook")
}
if !isProductionBranch(payload) {
return
}
project := api.getAssociatedProject(payload)
if project == nil {
return
}
version := getVersion(payload)
project.Version = version
api.Database.UpdateProject(project)
}
func signatureValid(r *Request) (matches bool) {
signature := parseSignatureFromRequest(r.Ctx)
if signature == "" {
return false
}
body := r.Ctx.PostBody()
mac := hmac.New(getHashFuncFromConfig(), config.Cfg.WebHookSecret)
mac.Write(body)
expectedMac := hex.EncodeToString(mac.Sum(nil))
matches = strings.Compare(expectedMac, signature) == 0
logrus.WithFields(logrus.Fields{
"expected": expectedMac,
"signature": signature,
"matches": matches,
}).Trace("Validating WebHook signature")
return
}
func getHashFuncFromConfig() func() hash.Hash {
if config.Cfg.WebHookHash == "sha1" {
return crypto.SHA1.New
} else if config.Cfg.WebHookHash == "sha256" {
return crypto.SHA256.New
}
logrus.WithFields(logrus.Fields{
"hash": config.Cfg.WebHookHash,
}).Error("Invalid hash function from config")
return nil
}
func parseSignatureFromRequest(ctx *fasthttp.RequestCtx) string {
signature := string(ctx.Request.Header.Peek(config.Cfg.WebHookSigHeader))
sigParts := strings.Split(signature, "=")
signature = sigParts[len(sigParts)-1]
return signature
}
func (api *WebAPI) getAssociatedProject(payload *GitPayload) *storage.Project {
project := api.Database.GetProjectWithRepoName(payload.Repository.FullName)
logrus.WithFields(logrus.Fields{
"project": project,
}).Trace("Found project associated with WebHook")
return project
}
func isProductionBranch(payload *GitPayload) (isProd bool) {
isProd = strings.HasSuffix(payload.Ref, "master")
logrus.WithFields(logrus.Fields{
"isProd": isProd,
}).Trace("Identified if push event occured in production branch")
return
}
func getVersion(payload *GitPayload) (version string) {
version = payload.After
logrus.WithFields(logrus.Fields{
"version": version,
}).Trace("Got new version")
return
}

2
api/main.go
View File

@ -58,6 +58,8 @@ func New() *WebAPI {
api.router.GET("/task/get/:project", LogRequest(api.TaskGetFromProject))
api.router.GET("/task/get", LogRequest(api.TaskGet))
api.router.POST("/git/receivehook", LogRequest(api.ReceiveGitWebHook))
return api
}

6
api/project.go
View File

@ -8,7 +8,8 @@ import (
type CreateProjectRequest struct {
Name string `json:"name"`
GitUrl string `json:"git_url"`
CloneUrl string `json:"clone_url"`
GitRepo string `json:"git_repo"`
Version string `json:"version"`
Priority int64 `json:"priority"`
}
@ -33,7 +34,8 @@ func (api *WebAPI) ProjectCreate(r *Request) {
project := &storage.Project{
Name: createReq.Name,
Version: createReq.Version,
GitUrl: createReq.GitUrl,
CloneUrl: createReq.CloneUrl,
GitRepo: createReq.GitRepo,
Priority: createReq.Priority,
}

28
api/worker.go
View File

@ -25,10 +25,25 @@ type GetWorkerResponse struct {
func (api *WebAPI) WorkerCreate(r *Request) {
workerReq := &CreateWorkerRequest{}
if r.GetJson(workerReq) {
if !r.GetJson(workerReq) {
return
}
identity := getIdentity(r)
if canCreateWorker(r, workerReq, identity) {
if !canCreateWorker(r, workerReq, identity) {
logrus.WithFields(logrus.Fields{
"identity": identity,
"createWorkerRequest": workerReq,
}).Warn("Failed CreateWorkerRequest")
r.Json(CreateWorkerResponse{
Ok: false,
Message: "You are now allowed to create a worker",
}, 403)
return
}
id, err := api.workerCreate(workerReq, getIdentity(r))
if err != nil {
@ -39,14 +54,6 @@ func (api *WebAPI) WorkerCreate(r *Request) {
WorkerId: id,
})
}
} else {
r.Json(CreateWorkerResponse{
Ok: false,
Message: "You are now allowed to create a worker",
}, 403)
}
}
}
func (api *WebAPI) WorkerGet(r *Request) {
@ -99,6 +106,7 @@ func getIdentity(r *Request) *storage.Identity {
identity := storage.Identity{
RemoteAddr: r.Ctx.RemoteAddr().String(),
UserAgent: string(r.Ctx.UserAgent()),
}
return &identity

11
config.yml
View File

@ -1,7 +1,10 @@
server:
address: "127.0.0.1:5000"
test_address: "127.0.0.1:5001"
address: "0.0.0.0:42901"
database:
conn_str : "user=task_tracker dbname=task_tracker sslmode=disable"
test_conn_str : "user=task_tracker dbname=task_tracker_test sslmode=disable"
conn_str: "user=task_tracker dbname=task_tracker_test sslmode=disable"
git:
webhook_secret: "very_secret_secret"
webhook_hash: "sha1"
webhook_sig_header: "X-Hub-Signature"

6
config/config.go
View File

@ -7,6 +7,9 @@ import (
var Cfg struct {
ServerAddr string
DbConnStr string
WebHookSecret []byte
WebHookHash string
WebHookSigHeader string
}
func SetupConfig() {
@ -21,4 +24,7 @@ func SetupConfig() {
Cfg.ServerAddr = viper.GetString("server.address")
Cfg.DbConnStr = viper.GetString("database.conn_str")
Cfg.WebHookSecret = []byte(viper.GetString("git.webhook_secret"))
Cfg.WebHookHash = viper.GetString("git.webhook_hash")
Cfg.WebHookSigHeader = viper.GetString("git.webhook_sig_header")
}

4
schema.sql
View File

@ -11,6 +11,7 @@ CREATE TABLE workerIdentity
(
id SERIAL PRIMARY KEY,
remote_addr TEXT,
user_agent TEXT,
UNIQUE (remote_addr)
);
@ -27,7 +28,8 @@ CREATE TABLE project
id SERIAL PRIMARY KEY,
priority INTEGER DEFAULT 0,
name TEXT UNIQUE,
git_url TEXT,
clone_url TEXT,
git_repo TEXT UNIQUE,
version TEXT
);

17
setup.sh Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
export INSTALL_DIR="/home/drone/task_tracker"
mkdir ${INSTALL_DIR} 2> /dev/null
# Gogs
if [[ ! -d "${INSTALL_DIR}/gogs" ]]; then
wget "https://dl.gogs.io/0.11.79/gogs_0.11.79_linux_amd64.tar.gz"
tar -xzf "gogs_0.11.79_linux_amd64.tar.gz" -C ${INSTALL_DIR}
rm "gogs_0.11.79_linux_amd64.tar.gz"
fi
# Postgres
su - postgres -c "createuser task_tracker"
su - postgres -c "dropdb gogs"
su - postgres -c "createdb gogs"

76
storage/project.go
View File

@ -3,13 +3,15 @@ package storage
import (
"database/sql"
"github.com/Sirupsen/logrus"
"strings"
)
type Project struct {
Id int64 `json:"id"`
Priority int64 `json:"priority"`
Name string `json:"name"`
GitUrl string `json:"git_url"`
CloneUrl string `json:"clone_url"`
GitRepo string `json:"git_repo"`
Version string `json:"version"`
}
@ -24,8 +26,9 @@ func (database *Database) SaveProject(project *Project) (int64, error) {
func saveProject(project *Project, db *sql.DB) (int64, error) {
row := db.QueryRow("INSERT INTO project (name, git_url, version, priority) VALUES ($1,$2,$3, $4) RETURNING id",
project.Name, project.GitUrl, project.Version, project.Priority)
row := db.QueryRow(`INSERT INTO project (name, git_repo, clone_url, version, priority)
VALUES ($1,$2,$3,$4,$5) RETURNING id`,
project.Name, project.GitRepo, project.CloneUrl, project.Version, project.Priority)
var id int64
err := row.Scan(&id)
@ -56,14 +59,11 @@ func (database *Database) GetProject(id int64) *Project {
func getProject(id int64, db *sql.DB) *Project {
project := &Project{}
row := db.QueryRow(`SELECT * FROM project WHERE id=$1`, id)
row := db.QueryRow("SELECT id, name, git_url, version, priority FROM project WHERE id=$1",
id)
err := row.Scan(&project.Id, &project.Name, &project.GitUrl, &project.Version, &project.Priority)
project, err := scanProject(row)
if err != nil {
logrus.WithFields(logrus.Fields{
logrus.WithError(err).WithFields(logrus.Fields{
"id": id,
}).Warn("Database.getProject SELECT project NOT FOUND")
return nil
@ -76,3 +76,61 @@ func getProject(id int64, db *sql.DB) *Project {
return project
}
func scanProject(row *sql.Row) (*Project, error) {
project := &Project{}
err := row.Scan(&project.Id, &project.Priority, &project.Name, &project.CloneUrl, &project.GitRepo,
&project.Version)
return project, err
}
func (database *Database) GetProjectWithRepoName(repoName string) *Project {
db := database.getDB()
project := getProjectWithRepoName(repoName, db)
err := db.Close()
handleErr(err)
return project
}
func getProjectWithRepoName(repoName string, db *sql.DB) *Project {
row := db.QueryRow(`SELECT * FROm project WHERE LOWER(git_repo)=$1`, strings.ToLower(repoName))
project, err := scanProject(row)
if err != nil {
logrus.WithError(err).WithFields(logrus.Fields{
"repoName": repoName,
}).Error("Database.getProjectWithRepoName SELECT project NOT FOUND")
return nil
}
return project
}
func (database *Database) UpdateProject(project *Project) {
db := database.getDB()
updateProject(project, db)
err := db.Close()
handleErr(err)
}
func updateProject(project *Project, db *sql.DB) {
res, err := db.Exec(`UPDATE project
SET (priority, name, clone_url, git_repo, version) = ($1,$2,$3,$4,$5) WHERE id=$6`,
project.Priority, project.Name, project.CloneUrl, project.GitRepo, project.Version, project.Id)
handleErr(err)
rowsAffected, _ := res.RowsAffected()
logrus.WithFields(logrus.Fields{
"project": project,
"rowsAffected": rowsAffected,
}).Trace("Database.updateProject UPDATE project")
return
}

2
storage/task.go
View File

@ -165,7 +165,7 @@ func scanTask(row *sql.Row) *Task {
err := row.Scan(&task.Id, &task.Priority, &project.Id, &task.Assignee,
&task.Retries, &task.MaxRetries, &task.Status, &task.Recipe, &project.Id,
&project.Priority, &project.Name, &project.GitUrl, &project.Version)
&project.Priority, &project.Name, &project.CloneUrl, &project.GitRepo, &project.Version)
handleErr(err)
return task

12
storage/worker.go
View File

@ -8,7 +8,8 @@ import (
)
type Identity struct {
RemoteAddr string
RemoteAddr string `json:"remote_addr"`
UserAgent string `json:"user_agent"`
}
type Worker struct {
@ -76,8 +77,8 @@ func getIdentity(id int64, db *sql.DB) (*Identity, error) {
identity := &Identity{}
row := db.QueryRow("SELECT (remote_addr) FROM workeridentity WHERE id=$1", id)
err := row.Scan(&identity.RemoteAddr)
row := db.QueryRow("SELECT remote_addr, user_agent FROM workeridentity WHERE id=$1", id)
err := row.Scan(&identity.RemoteAddr, &identity.UserAgent)
if err != nil {
return nil, errors.New("identity not found")
@ -92,8 +93,8 @@ func getIdentity(id int64, db *sql.DB) (*Identity, error) {
func getOrCreateIdentity(identity *Identity, db *sql.DB) int64 {
res, err := db.Exec("INSERT INTO workeridentity (remote_addr) VALUES ($1) ON CONFLICT DO NOTHING",
identity.RemoteAddr)
res, err := db.Exec("INSERT INTO workeridentity (remote_addr, user_agent) VALUES ($1,$2) ON CONFLICT DO NOTHING",
identity.RemoteAddr, identity.UserAgent)
handleErr(err)
rowsAffected, err := res.RowsAffected()
@ -113,4 +114,3 @@ func getOrCreateIdentity(identity *Identity, db *sql.DB) int64 {
return rowId
}

104
test/api_git_test.go Normal file
View File

@ -0,0 +1,104 @@
package test
import (
"bytes"
"crypto"
"crypto/hmac"
"encoding/hex"
"fmt"
"net/http"
"src/task_tracker/api"
"src/task_tracker/config"
"testing"
)
func TestWebHookNoSignature(t *testing.T) {
r := Post("/git/receivehook", api.GitPayload{})
fmt.Println(r.StatusCode)
if r.StatusCode != 403 {
t.Error()
}
}
func TestWebHookInvalidSignature(t *testing.T) {
req, _ := http.NewRequest("POST", "http://"+config.Cfg.ServerAddr+"/git/receivehook", nil)
req.Header.Add("X-Hub-Signature", "invalid")
client := http.Client{}
r, _ := client.Do(req)
fmt.Println(r.StatusCode)
if r.StatusCode != 403 {
t.Error()
}
}
func TestWebHookDontUpdateVersion(t *testing.T) {
resp := createProject(api.CreateProjectRequest{
Name: "My version should not be updated",
Version: "old",
GitRepo: "username/not_this_one",
})
body := []byte(`{"ref": "refs/heads/master", "after": "new", "repository": {"full_name": "username/repo_name"}}`)
bodyReader := bytes.NewReader(body)
mac := hmac.New(crypto.SHA1.New, config.Cfg.WebHookSecret)
mac.Write(body)
signature := hex.EncodeToString(mac.Sum(nil))
signature = "sha1=" + signature
req, _ := http.NewRequest("POST", "http://"+config.Cfg.ServerAddr+"/git/receivehook", bodyReader)
req.Header.Add("X-Hub-Signature", signature)
client := http.Client{}
r, _ := client.Do(req)
if r.StatusCode != 200 {
t.Error()
}
getResp, _ := getProject(resp.Id)
if getResp.Project.Version != "old" {
t.Error()
}
}
func TestWebHookUpdateVersion(t *testing.T) {
resp := createProject(api.CreateProjectRequest{
Name: "My version should be updated",
Version: "old",
GitRepo: "username/repo_name",
})
body := []byte(`{"ref": "refs/heads/master", "after": "new", "repository": {"full_name": "username/repo_name"}}`)
bodyReader := bytes.NewReader(body)
mac := hmac.New(crypto.SHA1.New, config.Cfg.WebHookSecret)
mac.Write(body)
signature := hex.EncodeToString(mac.Sum(nil))
signature = "sha1=" + signature
req, _ := http.NewRequest("POST", "http://"+config.Cfg.ServerAddr+"/git/receivehook", bodyReader)
req.Header.Add("X-Hub-Signature", signature)
client := http.Client{}
r, _ := client.Do(req)
if r.StatusCode != 200 {
t.Error()
}
getResp, _ := getProject(resp.Id)
if getResp.Project.Version != "new" {
t.Error()
}
}

29
test/api_project_test.go
View File

@ -13,7 +13,8 @@ func TestCreateGetProject(t *testing.T) {
resp := createProject(api.CreateProjectRequest{
Name: "Test name",
GitUrl: "http://github.com/test/test",
CloneUrl: "http://github.com/test/test",
GitRepo: "drone/webhooktest",
Version: "Test Version",
Priority: 123,
})
@ -41,7 +42,10 @@ func TestCreateGetProject(t *testing.T) {
t.Error()
}
if getResp.Project.GitUrl != "http://github.com/test/test" {
if getResp.Project.CloneUrl != "http://github.com/test/test" {
t.Error()
}
if getResp.Project.GitRepo != "drone/webhooktest" {
t.Error()
}
if getResp.Project.Priority != 123 {
@ -57,7 +61,7 @@ func TestCreateProjectInvalid(t *testing.T) {
}
}
func TestCreateDuplicateProject(t *testing.T) {
func TestCreateDuplicateProjectName(t *testing.T) {
createProject(api.CreateProjectRequest{
Name: "duplicate name",
})
@ -74,6 +78,25 @@ func TestCreateDuplicateProject(t *testing.T) {
}
}
func TestCreateDuplicateProjectRepo(t *testing.T) {
createProject(api.CreateProjectRequest{
Name: "different name",
GitRepo: "user/same",
})
resp := createProject(api.CreateProjectRequest{
Name: "but same repo",
GitRepo: "user/same",
})
if resp.Ok != false {
t.Error()
}
if len(resp.Message) <= 0 {
t.Error()
}
}
func TestGetProjectNotFound(t *testing.T) {
getResp, r := getProject(12345)

13
test/api_task_test.go
View File

@ -15,7 +15,7 @@ func TestCreateTaskValid(t *testing.T) {
createProject(api.CreateProjectRequest{
Name: "Some Test name",
Version: "Test Version",
GitUrl: "http://github.com/test/test",
CloneUrl: "http://github.com/test/test",
})
resp := createTask(api.CreateTaskRequest{
@ -68,7 +68,8 @@ func TestCreateGetTask(t *testing.T) {
resp := createProject(api.CreateProjectRequest{
Name: "My project",
Version: "1.0",
GitUrl: "http://github.com/test/test",
CloneUrl: "http://github.com/test/test",
GitRepo: "myrepo",
Priority: 999,
})
@ -108,7 +109,7 @@ func TestCreateGetTask(t *testing.T) {
if taskResp.Task.Project.Version != "1.0" {
t.Error()
}
if taskResp.Task.Project.GitUrl != "http://github.com/test/test" {
if taskResp.Task.Project.CloneUrl != "http://github.com/test/test" {
t.Error()
}
}
@ -118,13 +119,15 @@ func createTasks(prefix string) (int64, int64) {
lowP := createProject(api.CreateProjectRequest{
Name: prefix + "low",
Version: "1.0",
GitUrl: "http://github.com/test/test",
CloneUrl: "http://github.com/test/test",
GitRepo: prefix + "low1",
Priority: 1,
})
highP := createProject(api.CreateProjectRequest{
Name: prefix + "high",
Version: "1.0",
GitUrl: "http://github.com/test/test",
CloneUrl: "http://github.com/test/test",
GitRepo: prefix + "high1",
Priority: 999,
})
createTask(api.CreateTaskRequest{

26
test/api_worker_test.go
View File

@ -15,21 +15,27 @@ func TestCreateGetWorker(t *testing.T) {
resp, r := createWorker(api.CreateWorkerRequest{})
if r.StatusCode != 200 {
t.Fail()
t.Error()
}
if resp.Ok != true {
t.Fail()
t.Error()
}
getResp, r := getWorker(resp.WorkerId.String())
if r.StatusCode != 200 {
t.Fail()
t.Error()
}
if resp.WorkerId != getResp.Worker.Id {
t.Error()
}
if resp.WorkerId != getResp.Worker.Id {
t.Fail()
if len(getResp.Worker.Identity.RemoteAddr) <= 0 {
t.Error()
}
if len(getResp.Worker.Identity.UserAgent) <= 0 {
t.Error()
}
}
@ -38,10 +44,10 @@ func TestGetWorkerNotFound(t *testing.T) {
resp, r := getWorker("8bfc0ccd-d5ce-4dc5-a235-3a7ae760d9c6")
if r.StatusCode != 404 {
t.Fail()
t.Error()
}
if resp.Ok != false {
t.Fail()
t.Error()
}
}
@ -50,13 +56,13 @@ func TestGetWorkerInvalid(t *testing.T) {
resp, r := getWorker("invalid-uuid")
if r.StatusCode != 400 {
t.Fail()
t.Error()
}
if resp.Ok != false {
t.Fail()
t.Error()
}
if len(resp.Message) <= 0 {
t.Fail()
t.Error()
}
}

5
test/config.yml
View File

@ -3,3 +3,8 @@ server:
database:
conn_str : "user=task_tracker dbname=task_tracker_test sslmode=disable"
git:
webhook_secret: "very_secret_secret"
webhook_hash: "sha1"
webhook_sig_header: "X-Hub-Signature"

1
test/schema.sql
View File

@ -1 +0,0 @@
../schema.sql

48
test/schema.sql Normal file
View File

@ -0,0 +1,48 @@
DROP TABLE IF EXISTS workerIdentity, Worker, Project, Task;
DROP TYPE IF EXISTS Status;
CREATE TYPE status as ENUM (
'new',
'failed',
'closed'
);
CREATE TABLE workerIdentity
(
id SERIAL PRIMARY KEY,
remote_addr TEXT,
user_agent TEXT,
UNIQUE (remote_addr)
);
CREATE TABLE worker
(
id TEXT PRIMARY KEY,
created INTEGER,
identity INTEGER REFERENCES workerIdentity (id)
);
CREATE TABLE project
(
id SERIAL PRIMARY KEY,
priority INTEGER DEFAULT 0,
name TEXT UNIQUE,
clone_url TEXT,
git_repo TEXT UNIQUE,
version TEXT
);
CREATE TABLE task
(
id SERIAL PRIMARY KEY,
priority INTEGER DEFAULT 0,
project INTEGER REFERENCES project (id),
assignee TEXT REFERENCES worker (id),
retries INTEGER DEFAULT 0,
max_retries INTEGER,
status Status DEFAULT 'new',
recipe TEXT
);