From b14106b456e451fb70daac1a28bc7d5f27473637 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 1 Jun 2019 11:26:49 -0400 Subject: [PATCH] Documentation --- API_DOCS.md | 333 ++++++++++++++++++ DOCS.md | 115 ++++++ README.md | 35 +- client/README.md | 56 +++ web/angular/src/app/api.service.ts | 2 +- .../src/app/index/index.component.html | 55 ++- 6 files changed, 550 insertions(+), 46 deletions(-) create mode 100644 API_DOCS.md create mode 100644 DOCS.md create mode 100644 client/README.md diff --git a/API_DOCS.md b/API_DOCS.md new file mode 100644 index 0000000..9a65616 --- /dev/null +++ b/API_DOCS.md @@ -0,0 +1,333 @@ + +## Worker API + +### Worker +`/worker/create` + +Request +```bash +curl -X POST 'http://localhost:3010/worker/create' -d ' +{ + "alias": "some alias" +}' +``` + +Response +```json +{ + "ok": true, + "content": { + "worker": { + "id": 1, + "created": 1559396382, + "alias": "some alias", + "secret": "ftZVO4w9Fc7bDuOISRaJL9P92ijkfvNah1Ldgc0a9f8=", + "paused": false + } + } +} +``` + +---- + +`/worker/update` + +Request +```bash +curl -X POST 'http://localhost:3010/worker/update' \ +-H 'X-Worker-ID: 1' -H 'X-Secret: ftZVO4w9Fc7bDuOISRaJL9P92ijkfvNah1Ldgc0a9f8=' -d ' +{ + "alias": "another alias" +}' +``` + +Response +```json +{ + "ok": true, + "message": "(Error message, if applicable)" +} +``` + +### Tasks + +`/task/submit` +Requires SUBMIT permissions on the project. max_assign_time is in seconds. Hash64 is +a 64-bit number - the submit will fail if another task has the same hash. If UniqueString +is specified, it will be hashed and put in place of Hash64. + +VerificationCount is the number +of times the task has to be released with the same verification hash by *different workers* +before the task is marked as closed. For example, a VerificationCount of 2 means that two +different workers have to assign and release the same task with the same verification hash before the +task can be marked as closed. + + +Request +```bash +curl -X POST 'http://localhost:3010/task/submit' \ +-H 'X-Worker-ID: 1' -H 'X-Secret: ftZVO4w9Fc7bDuOISRaJL9P92ijkfvNah1Ldgc0a9f8=' -d ' +{ + "project": 1, + "max_retries": 3, + "recipe": "test recipe", + "priority": 1, + "max_assign_time": 3600, + "hash64": 0, + "unique_string": "", + "verification_count": 0 +}' +``` + +Response +```json +{ + "ok": true, + "message": "(Error message, if applicable)" +} +``` + +---- +`/task/bulk_submit` + +Same as `/task/submit`, but instead submit an array of submit requests. Tasks must +be for the same project. Follows the project's submit rate limit. + + +---- +`/task/get/:project` + +Request +```bash +curl -X GET 'http://localhost:3010/task/get/1'\ + -H 'X-Worker-ID: 1' -H 'X-Secret: ftZVO4w9Fc7bDuOISRaJL9P92ijkfvNah1Ldgc0a9f8=' +``` + +Response +```json +{ + "ok": true, + "content": { + "task": { + "id": 7, + "priority": 1, + "assignee": 1, + "retries": 0, + "max_retries": 3, + "status": 1, + "recipe": "test recipe", + "max_assign_time": 0, + "assign_time": 1559397394, + "verification_count": 0, + "project": { + "id": 1, + "priority": 999, + "name": "My project", + "clone_url": "http://github.com/test/test", + "git_repo": "myrepo", + "version": "1.0", + "motd": "", + "public": true, + "hidden": false, + "chain": 0, + "paused": false, + "assign_rate": 2, + "submit_rate": 2 + } + } + } +} +``` + + +---- +`/task/release` + +Result can be either of: + +* *TR_OK*=0: Task was completed +* *TR_FAIL*=1: The worker failed to complete the task (tracker will mark the task as +FAILED after `max_retries` retries. +* *TR_SKIP*=2: Act as if the worker never touched this task + +Request +```bash +curl -X POST 'http://localhost:3010/task/release'\ + -H 'X-Worker-ID: 1' -H 'X-Secret: ftZVO4w9Fc7bDuOISRaJL9P92ijkfvNah1Ldgc0a9f8=' -d ' +{ + "task_id": 7, + "result": 0, + "verification": -32315129 +}' +``` + +Response + +Updated will be set to false if their was an error (see Message) or if +the task has not yet reached the required number of verifications. +```json +{ + "ok": true, + "message": "(Message, if applicable)", + "content": { + "updated": true + } +} +``` + +### Logs + +`/log/{trace|info|warn|error}` + +Logs will be publicly available through the web UI +if enabled in the config and will always appear in the log file. + +Request +```bash +curl -X POST 'http://localhost:3010/log/info'\ + -H 'X-Worker-ID: 1' -H 'X-Secret: ftZVO4w9Fc7bDuOISRaJL9P92ijkfvNah1Ldgc0a9f8=' -d ' +{ + "scope": "Arbitrary log category", + "message": "message", + "timestamp": 1559393394 +}' +``` + +Response +```json +{ + "ok": true, + "message": "(Error message, if applicable)" +} +``` + +## Projects + +A=Assign (`/task/get/:project`) +S=Submit (`/task/submit`) +R=Read project from API (`project/get/:project`) + +### Permissions +| Permission | Public | Private | Hidden | +| --- | --- | --- | --- | +| (none) | A R | R | | +| `ASSIGN` | A R | A R | A | +| `SUBMIT` | A S R | A S R | A S | +| `ROLE_READ` (Manager) | | | R | + + +---- +`/project/request_access` + +A Manager with ROLE_MANAGE_ACCESS will need to manually approve the request from the UI. + +Request +```bash +curl -X POST 'http://localhost:3010/project/request_access'\ + -H 'X-Worker-ID: 1' -H 'X-Secret: ftZVO4w9Fc7bDuOISRaJL9P92ijkfvNah1Ldgc0a9f8=' -d ' +{ + "assign": true, + "submit": true, + "project": 23 +}' +``` + +Response +```json +{ + "ok": true, + "message": "(Error message, if applicable)" +} +``` + + +---- +`/project/get/"id` + +Request + +You must be authenticated as a manager and have ROLE_READ + on the project to see hidden projects +```bash +curl -X GET 'http://localhost:3010/project/get/1' +``` + +Response +```json +{ + "ok": true, + "message": "", + "content": { + "id": 1, + "priority": 999, + "name": "My project", + "clone_url": "http://github.com/test/test", + "git_repo": "myrepo", + "version": "1.0", + "motd": "", + "public": true, + "hidden": false, + "chain": 0, + "paused": false, + "assign_rate": 2, + "submit_rate": 2 + } +} +``` + +----- +`/project/list` + +Hidden projects are returned if only if authenticated as a Manager and have the +permission to read them. + +Request +```bash +curl -X GET 'http://localhost:3010/project/list' +``` + +Response +```json +{ + "ok": true, + "message": "", + "content": { + "projects": [] + } +} +``` + +## UI / Manager API + +Currently undocumented + +``` +/worker/set_paused +/worker/stats +/project/create +/project/monitoring-between/:id +/project/monitoring/:id +/project/assignees/:id +/project/access_list/:id +/project/request_access +/project/accept_request/:id/:wid +/project/reject_request/:id/:wid +/project/secret/:id +/project/secret/:id +/project/webhook_secret/:id +/project/webhook_secret/:id +/project/reset_failed_tasks/:id +/project/hard_reset/:id +/project/reclaim_assigned_tasks/:id +/git/receivehook +/logs +/register +/login +/logout +/account +/manager/list +/manager/list_for_project/:id +/manager/promote/:id +/manager/demote/:id +/manager/set_role_for_project/:id +``` diff --git a/DOCS.md b/DOCS.md new file mode 100644 index 0000000..285d0df --- /dev/null +++ b/DOCS.md @@ -0,0 +1,115 @@ +# Documentation + +## Installation (Docker) + +Prerequisites: +* You have a postgres container using the network `tt`, listening +for connections on `172.26.0.2:5432`. +Example: +`docker run -d --name tt_pg --network tt postgres:alpine` + + +1. Initialize the database + + ```bash + psql -h 172.26.0.2 -U postgres + > CREATE USER task_tracker; + > CREATE database task_tracker; + ``` +1. Write configuration file + + `vim config.yml` + ```yaml + server: + address: "localhost:3010" + database: + conn_str: "postgres://task_tracker:task_tracker@172.26.0.2/task_tracker?sslmode=disable" + log_levels: ["error", "info", "warn"] + git: + webhook_hash: "sha256" + webhook_sig_header: "X-Gogs-Signature" + log: + level: "trace" + session: + cookie_name: "tt" + expiration: "8h" + monitoring: + snapshot_interval: "120s" + history_length: "400h" + maintenance: + reset_timed_out_tasks_interval: "5m" + ``` + +1. Create task_tracker container: + + ```bash + docker run --rm\ + -v $PWD/config.yml:/root/config.yml\ + --network tt\ + -p 0.0.0.0:12345:80\ + simon987/task_tracker + ``` + +## Installation (Linux) + +* You have a postgres daemon listening for connections on `localhost:5432`. +* You have a working installation of **go**, **nodejs** and **nginx** + +1. Initialize the database + + ```bash + sudo su postgres + createuser task_tracker + createdb task_tracker + psql task_tracker + > ALTER USER "task_tracker" WITH PASSWORD 'task_tracker'; + ``` + +1. Acquire binaries + + API + ```bash + go get -d github.com/simon987/task_tracker/... + cd $GOPATH/src/github.com/simon987/task_tracker/main + go build -o tt_api . + ``` + + UI + ```bash + git clone https://github.com/simon987/task_tracker + cd task_tracker/web/angular + npm install + ./node_modules/\@angular/cli/bin/ng build --prod --optimization + ``` + +1. Setup web server + + Move ./dist/ to /path/to/webroot/, start ./tt_api + ```nginx + index index.html; + + root /path/to/webroot; + + location / { + try_files $uri $uri/ /index.html; + } + location ~ /api(.*)$ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:3010$1?$args; + } + ``` + +## Getting started + +Register a *Manager* account from the /login page. The first account +will automatically be given tracker admin permissions. + +Follow the instructions in the index page to create a project. + +## API documentation + +See [API_DOCS.md](API_DOCS.md) + diff --git a/README.md b/README.md index 836f0fc..b619f83 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,13 @@ Fast task tracker (job queue) with authentication, statistics and web frontend +### Documentation + +* Go client: [client](https://github.com/simon987/task_tracker/tree/master/client) +* Python client: [task_tracker_drone](https://github.com/simon987/task_tracker_drone) +* API specs: [API_DOCS](API_DOCS.md) +* Installation/Usage: [DOCS](DOCS.md) + ### Features * Stateless/Fault tolerent @@ -33,34 +40,6 @@ max_assign_time | TTR (time-to-run) | Visibility timeout | Timeout \- | - | Retention Period | Expires in -### Postgres setup -```bash -sudo su postgres -createuser task_tracker -createdb task_tracker -psql task_tracker -> ALTER USER "task_tracker" WITH PASSWORD 'task_tracker'; -``` - -### Nginx Setup - -```nginx -index index.html; - -root /path/to/webroot; - -location / { - try_files $uri $uri/ /index.html; -} -location ~ /api(.*)$ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:3010$1?$args; # Change host/port if necessary -} -``` - ### Running tests ```bash cd test/ diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..52e5738 --- /dev/null +++ b/client/README.md @@ -0,0 +1,56 @@ +### Example + +```go +package test + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "github.com/simon987/task_tracker/api" + "github.com/simon987/task_tracker/client" + "io/ioutil" + "os" +) + +func main() { + + const projectId = 1 + const apiAddr = "http://localhost:3010/" + + ttClient := client.New(apiAddr) + w, _ := ttClient.MakeWorker("my alias") + ttClient.SetWorker(w) + + // Save worker credentials to file + workerJsonData, _ := json.Marshal(&w) + fp, _ := os.OpenFile("worker.json", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + _, _ = fp.Write(workerJsonData) + + // Load worker from file + var worker client.Worker + fp, _ = os.OpenFile("worker.json", os.O_RDONLY, 0600) + workerJsonData, _ = ioutil.ReadAll(fp) + _ = json.Unmarshal(workerJsonData, &worker) + + // Request access + _, _ = ttClient.RequestAccess(api.CreateWorkerAccessRequest{ + Assign:true, + Submit:true, + Project:projectId, + }) + + // Assign task + task, _ := ttClient.FetchTask(projectId) + + // Release task + _, _ = ttClient.ReleaseTask(api.ReleaseTaskRequest{ + Result: 0, + TaskId: task.Content.Task.Id, + }) + + // Get project secret + secret, _ := ttClient.GetProjectSecret(projectId) + fmt.Println(secret) +} +``` \ No newline at end of file diff --git a/web/angular/src/app/api.service.ts b/web/angular/src/app/api.service.ts index 85acbae..05b5f1e 100755 --- a/web/angular/src/app/api.service.ts +++ b/web/angular/src/app/api.service.ts @@ -6,7 +6,7 @@ import {Credentials} from './models/credentials'; @Injectable() export class ApiService { - public url: string = window.location.protocol + '//' + window.location.hostname + '/api'; + public url: string = window.location.protocol + '//' + window.location.host + '/api'; private options: { withCredentials: true, responseType: 'json' diff --git a/web/angular/src/app/index/index.component.html b/web/angular/src/app/index/index.component.html index 8ce2769..f6b58b6 100644 --- a/web/angular/src/app/index/index.component.html +++ b/web/angular/src/app/index/index.component.html @@ -2,32 +2,30 @@ task_tracker - "simple tracker that aims to blah blah" + Fast task tracker (job queue) with authentication, statistics and web frontend + +

Documentation

+

Installation instructions here, + API documentation here. +

+

Go client documentation here. +

+

Python client documentation here.

+

Get started

-

Create a project and associate it to a git repository. - Payload URL for webhooks is is:

- 
{{apiService.url}}/receivewebhook
+

Login and create a project from the New project tab

-

Workers will be made aware of version changes on the master branch - when they assign themselves to new tasks:

+

To enable Webhooks (See step 4), enter a git url and a repository name

+ +

You can chain tasks to another project if you also have permissions on that project. + Tasks will be duplicated on this project when they are marked as closed.

- 
{{
-                        'GET /task/get\n\n{\n' +
-                        '  "id": 24,\n' +
-                        '  "priority": 1,\n' +
-                        '  ...\n' +
-                        '  "project": {\n' +
-                        '    "id": 1,\n' +
-                        '    ...\n' +
-                        '    "version": "<' + ('index.version'|translate) +
-                        '>",\n  }\n}'}}
-
@@ -63,6 +61,29 @@

You will be given READ, EDIT and MANAGE_ACCESS roles from projects you create. You can also give access to other project managers from the project permissions page

+ + + +

A webhook secret is auto-generated when the project is created, you + can consult/update it from the project's dashboard. Point Github/Gogs/Gitea + payloads to this address:

+ + 
{{apiService.url}}/receivewebhook
+ +

Workers will be made aware of version changes on the master branch + when they assign themselves to new tasks:

+ + 
{{
+                        'GET /task/get\n\n{\n' +
+                        '  "id": 24,\n' +
+                        '  "priority": 1,\n' +
+                        '  ...\n' +
+                        '  "project": {\n' +
+                        '    "id": 1,\n' +
+                        '    ...\n' +
+                        '    "version": "<' + ('index.version'|translate) +
+                        '>",\n  }\n}'}}
+