From fea86af41d2485faf91f9b3051b51a3341922753 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 9 Mar 2019 09:31:19 -0500 Subject: [PATCH] jenkins setup --- api/api.go | 17 ++++++++++- api/models.go | 3 ++ api/slot.go | 19 ++++++------ jenkins/Jenkinsfile | 71 +++++++++++++++++++++++++++++++++++++++++++++ jenkins/deploy.sh | 9 ++++++ test/main_test.go | 2 +- test/upload_test.go | 55 ++++++++++++----------------------- 7 files changed, 128 insertions(+), 48 deletions(-) create mode 100644 jenkins/Jenkinsfile create mode 100755 jenkins/deploy.sh diff --git a/api/api.go b/api/api.go index 12a12af..bbbc8cc 100644 --- a/api/api.go +++ b/api/api.go @@ -12,7 +12,7 @@ import ( "path/filepath" ) -var WorkDir, _ = filepath.Abs("./data/") +var WorkDir = getWorkDir() type Info struct { Name string `json:"name"` @@ -67,6 +67,10 @@ func LogRequestMiddleware(h fasthttp.RequestHandler) fasthttp.RequestHandler { func New(db *gorm.DB) *WebApi { + if _, err := os.Stat(WorkDir); err != nil && os.IsNotExist(err) { + _ = os.Mkdir(WorkDir, 0700) + } + api := &WebApi{} logrus.SetLevel(getLogLevel()) @@ -134,3 +138,14 @@ func getLogLevel() logrus.Level { return level } } + +func getWorkDir() string { + workDir := os.Getenv("WS_BUCKET_WORKDIR") + if workDir == "" { + path, _ := filepath.Abs("./data") + return path + } else { + path, _ := filepath.Abs(workDir) + return path + } +} diff --git a/api/models.go b/api/models.go index 8704286..e4f99f9 100644 --- a/api/models.go +++ b/api/models.go @@ -1,6 +1,7 @@ package api import ( + "fmt" "path/filepath" "strings" ) @@ -49,6 +50,8 @@ func (req *AllocateUploadSlotRequest) IsValid() bool { return false } if !strings.HasPrefix(pathAbs, WorkDir) { + fmt.Println(pathAbs) + fmt.Println(WorkDir) return false } diff --git a/api/slot.go b/api/slot.go index 2e8153c..7bd1b77 100644 --- a/api/slot.go +++ b/api/slot.go @@ -25,7 +25,7 @@ func (api *WebApi) AllocateUploadSlot(ctx *fasthttp.RequestCtx) { req := &AllocateUploadSlotRequest{} err := json.Unmarshal(ctx.Request.Body(), req) if err != nil { - ctx.Response.SetStatusCode(400) + ctx.Response.Header.SetStatusCode(400) Json(GenericResponse{ Ok: false, }, ctx) @@ -33,7 +33,7 @@ func (api *WebApi) AllocateUploadSlot(ctx *fasthttp.RequestCtx) { } if !req.IsValid() { - ctx.Response.SetStatusCode(400) + ctx.Response.Header.SetStatusCode(400) Json(CreateClientResponse{ Ok: false, }, ctx) @@ -54,7 +54,7 @@ func (api *WebApi) Upload(ctx *fasthttp.RequestCtx) { err := api.db.Where("token=?", token).First(&slot).Error if err != nil { ctx.Response.Header.SetStatusCode(400) - logrus.WithFields(logrus.Fields{ + logrus.WithError(err).WithFields(logrus.Fields{ "token": token, }).Warning("Upload slot not found") return @@ -67,11 +67,6 @@ func (api *WebApi) Upload(ctx *fasthttp.RequestCtx) { err = upgrader.Upgrade(ctx, func(ws *websocket.Conn) { defer ws.Close() - err := ws.WritePreparedMessage(api.MotdMessage) - if err != nil { - panic(err) - } - mt, reader, err := ws.NextReader() if err != nil { panic(err) @@ -103,6 +98,8 @@ func (api *WebApi) Upload(ctx *fasthttp.RequestCtx) { _, _ = fp.Write(buf[:toWrite]) if err == io.EOF { break + } else if err != nil { + panic(err) } totalRead += int64(read) } @@ -130,7 +127,7 @@ func (api *WebApi) ReadUploadSlot(ctx *fasthttp.RequestCtx) { if err != nil { ctx.Response.Header.SetStatusCode(404) - logrus.WithFields(logrus.Fields{ + logrus.WithError(err).WithFields(logrus.Fields{ "token": tokenStr, }).Warning("Upload slot not found") return @@ -161,6 +158,10 @@ func (api *WebApi) ReadUploadSlot(ctx *fasthttp.RequestCtx) { panic(err) } } + err = fp.Close() + if err != nil { + panic(err) + } mu.(*sync.RWMutex).RUnlock() } diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile new file mode 100644 index 0000000..7997037 --- /dev/null +++ b/jenkins/Jenkinsfile @@ -0,0 +1,71 @@ +def remote = [:] +remote.name = 'remote' +remote.host = 'localhost' +remote.user = 'simon' +remote.identityFile = '/var/lib/jenkins/.ssh/id_rsa' +remote.knownHosts = '/var/lib/jenkins/.ssh/known_hosts' +remote.allowAnyHosts = true +remote.retryCount = 3 +remote.retryWaitSec = 3 +logLevel = 'FINER' + +pipeline { + agent none + environment { + GOOS='linux' + CGO_ENABLED='1' + HOME='.' + WS_BUCKET_ADDR='127.0.0.1:3021' + WS_BUCKET_WORKDIR='./data/' + } + stages { + stage('Parallel build & test') { + failFast true + parallel { + stage('Build - api') { + agent { + docker { + image 'golang:latest' + args '--network "host"' + } + } + steps { + sh 'mkdir -p /go/src/github.com/simon987/ws_bucket' + sh 'cp -r api main.go "/go/src/github.com/simon987/ws_bucket"' + sh 'cd /go/src/github.com/simon987/ws_bucket && go get ./...' + sh 'cd /go/src/github.com/simon987/ws_bucket && go build -a -installsuffix cgo -o "${WORKSPACE}/ws_bucket" .' + stash includes: 'ws_bucket', name: 'apidist' + } + } + stage('Test - api') { + agent { + docker { + image 'golang:latest' + args '--network "host"' + } + } + steps { + sh 'mkdir -p /go/src/github.com/simon987/ws_bucket' + sh 'cp -r api test main.go "/go/src/github.com/simon987/ws_bucket"' + sh 'cd /go/src/github.com/simon987/ws_bucket/ && go get -t ./test/...' + sh 'cd /go/src/github.com/simon987/ws_bucket/test && go test .' + } + } + } + } + stage('Deploy') { + agent none + steps { + node('master') { + unstash 'apidist' + sshCommand remote: remote, command: "cd ws_bucket && rm -rf ws_bucket deploy.sh" + sshPut remote: remote, from: 'ws_bucket', into: 'ws_bucket/ws_bucket' + sshPut remote: remote, from: 'jenkins/deploy.sh', into: 'ws_bucket/' + sshCommand remote: remote, command: 'chmod +x ws_bucket/deploy.sh && ./ws_bucket/deploy.sh' + } + } + } + } +} + + diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh new file mode 100755 index 0000000..b535c3f --- /dev/null +++ b/jenkins/deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +export WSBROOT="ws_bucket" + +screen -S ws_bucket -X quit +echo "starting ws_bucket" +screen -S ws_bucket -d -m bash -c "cd ${WSBROOT} && chmod +x ws_bucket && ./ws_bucket" +sleep 1 +screen -list diff --git a/test/main_test.go b/test/main_test.go index 1dd0687..570355f 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -11,7 +11,7 @@ import ( func TestMain(m *testing.M) { //db, err := gorm.Open("postgres", "host=localhost user=ws_bucket dbname=ws_bucket password=ws_bucket sslmode=disable") - db, err := gorm.Open("sqlite3", ":memory:") + db, err := gorm.Open("sqlite3", "test.db") if err != nil { panic(err) } diff --git a/test/upload_test.go b/test/upload_test.go index 342b396..31ac317 100644 --- a/test/upload_test.go +++ b/test/upload_test.go @@ -11,30 +11,9 @@ import ( "net/http" "net/url" "testing" + "time" ) -func TestWebsocketReturnsMotd(t *testing.T) { - - id := uuid.New() - allocateUploadSlot(api.AllocateUploadSlotRequest{ - FileName: "testmotd", - MaxSize: 0, - Token: id.String(), - }) - - c := ws(id.String()) - motd := &api.WebsocketMotd{} - err := c.ReadJSON(&motd) - handleErr(err) - - if len(motd.Motd) <= 0 { - t.Error() - } - if len(motd.Info.Version) <= 0 { - t.Error() - } -} - func TestWebSocketUploadSmallFile(t *testing.T) { id := uuid.New() @@ -46,10 +25,8 @@ func TestWebSocketUploadSmallFile(t *testing.T) { }) c := ws(id.String()) - _, _, err := c.ReadMessage() - handleErr(err) - err = c.WriteMessage(websocket.BinaryMessage, []byte("testuploadsmallfile")) + err := c.WriteMessage(websocket.BinaryMessage, []byte("testuploadsmallfile")) handleErr(err) err = c.Close() @@ -73,18 +50,21 @@ func TestWebSocketUploadOverwritesFile(t *testing.T) { }) c := ws(id.String()) - _, _, err := c.ReadMessage() - handleErr(err) - err = c.WriteMessage(websocket.BinaryMessage, []byte("testuploadsmallfile")) + err := c.WriteMessage(websocket.BinaryMessage, []byte("testuploadsmallfile")) handleErr(err) err = c.Close() handleErr(err) + time.Sleep(time.Millisecond * 50) + resp := readUploadSlot(id.String()) + + if bytes.Compare(resp, []byte("testuploadsmallfile")) != 0 { + t.Error() + } + c1 := ws(id.String()) - _, _, err = c1.ReadMessage() - handleErr(err) err = c1.WriteMessage(websocket.BinaryMessage, []byte("newvalue")) handleErr(err) @@ -92,7 +72,9 @@ func TestWebSocketUploadOverwritesFile(t *testing.T) { err = c1.Close() handleErr(err) - resp := readUploadSlot(id.String()) + time.Sleep(time.Millisecond * 50) + + resp = readUploadSlot(id.String()) if bytes.Compare(resp, []byte("newvalue")) != 0 { t.Error() @@ -110,14 +92,12 @@ func TestWebSocketUploadLargeFile(t *testing.T) { }) c := ws(id.String()) - _, _, err := c.ReadMessage() - handleErr(err) chunk := make([]byte, 100000) _ = copy(chunk, "test") _ = c.WriteMessage(websocket.BinaryMessage, chunk) - err = c.Close() + err := c.Close() handleErr(err) resp := readUploadSlot(id.String()) @@ -138,14 +118,12 @@ func TestWebSocketUploadMaxSize(t *testing.T) { }) c := ws(id.String()) - _, _, err := c.ReadMessage() - handleErr(err) chunk := make([]byte, 100000) _ = copy(chunk, "test") _ = c.WriteMessage(websocket.BinaryMessage, chunk) - err = c.Close() + err := c.Close() handleErr(err) resp := readUploadSlot(id.String()) @@ -157,6 +135,7 @@ func TestWebSocketUploadMaxSize(t *testing.T) { func readUploadSlot(token string) []byte { + time.Sleep(time.Millisecond * 20) r := Get("/slot", token) data, err := ioutil.ReadAll(r.Body) @@ -175,5 +154,7 @@ func ws(slot string) *websocket.Conn { c, _, err := websocket.DefaultDialer.Dial(u.String(), header) handleErr(err) + c.EnableWriteCompression(true) + return c }