Compare commits
23 Commits
master
...
e78e184375
| Author | SHA1 | Date | |
|---|---|---|---|
e78e184375
|
|||
|
02e87d7c40
|
|||
|
4450c9bb65
|
|||
|
6eb6984d6a
|
|||
| cc43a39ea3 | |||
|
962d9b5ac9
|
|||
|
e81fb5d445
|
|||
|
73945b6cb9
|
|||
| 089ea914b6 | |||
|
dd1baa4aef
|
|||
|
ea3195a93c
|
|||
|
aef1499e65
|
|||
|
c7203f58ff
|
|||
|
2e0d83cca1
|
|||
| 35882ca1a9 | |||
|
4e7f33338e
|
|||
|
ddc6c2bb4d
|
|||
| f9e29a4e30 | |||
|
c26e962898
|
|||
| 38c117d6fa | |||
|
ecb9724ee3
|
|||
|
4373e0a4a2
|
|||
| 19b71c9933 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,2 +1,3 @@
|
|||||||
.*.swp
|
.*.swp
|
||||||
*.retry
|
*.retry
|
||||||
|
vars_auth.yml
|
||||||
|
|||||||
4
install.sh
Executable file
4
install.sh
Executable file
@@ -0,0 +1,4 @@
|
|||||||
|
mkdir -p ./collections/ansible_collections/community
|
||||||
|
# git clone --depth=1 -b 1.2.1 https://github.com/ansible-collections/community.grafana.git ./collections/ansible_collections/community/grafana
|
||||||
|
git clone --depth=1 -b 1.2.1-extended https://github.com/ansible_community.grafana.git ./collections/ansible_collections/community/grafana
|
||||||
|
|
||||||
11
inv.yml
11
inv.yml
@@ -6,6 +6,12 @@ all:
|
|||||||
ssh_args: -o ControlMaster=auto -o ControlPersist=60s
|
ssh_args: -o ControlMaster=auto -o ControlPersist=60s
|
||||||
# ansible_host: 192.168.122.139
|
# ansible_host: 192.168.122.139
|
||||||
unpriv_user: thoto
|
unpriv_user: thoto
|
||||||
|
ed-c7-2:
|
||||||
|
ansible_user: root
|
||||||
|
ansible_host: ed-c7-2.virt.uller.thoto.net
|
||||||
|
# ansible_host: 192.168.123.60 # + jumphost
|
||||||
|
ssh_args: -o ControlMaster=auto -o ControlPersist=60s
|
||||||
|
unpriv_user: thoto
|
||||||
children:
|
children:
|
||||||
htcondor:
|
htcondor:
|
||||||
hosts:
|
hosts:
|
||||||
@@ -13,3 +19,8 @@ all:
|
|||||||
slurm:
|
slurm:
|
||||||
hosts:
|
hosts:
|
||||||
ed-c7-1:
|
ed-c7-1:
|
||||||
|
ed-c7-2:
|
||||||
|
cobald:
|
||||||
|
hosts:
|
||||||
|
ed-c7-1:
|
||||||
|
ed-c7-2:
|
||||||
|
|||||||
36
play.yml
36
play.yml
@@ -1,11 +1,25 @@
|
|||||||
---
|
---
|
||||||
- hosts: all
|
- hosts: all
|
||||||
|
vars_files:
|
||||||
|
- vars-auth.yml
|
||||||
|
- vars-influx.yml
|
||||||
tasks:
|
tasks:
|
||||||
|
- name: "install epel repo" # for htop etc.
|
||||||
|
yum:
|
||||||
|
name: epel-release
|
||||||
|
state: present
|
||||||
|
|
||||||
- name: "install tools"
|
- name: "install tools"
|
||||||
yum:
|
yum:
|
||||||
name: [ vim-enhanced, htop, screen, bind-utils, nmap-ncat, net-tools ]
|
name: [ vim-enhanced, htop, screen, bind-utils, nmap-ncat, net-tools ]
|
||||||
state: present
|
state: present
|
||||||
|
|
||||||
|
- name: "install ssh-key"
|
||||||
|
authorized_key:
|
||||||
|
user: thoto
|
||||||
|
key: "{{ssh_key}}"
|
||||||
|
state: present
|
||||||
|
|
||||||
- hosts: htcondor
|
- hosts: htcondor
|
||||||
pre_tasks:
|
pre_tasks:
|
||||||
- name: "install htcondor repo"
|
- name: "install htcondor repo"
|
||||||
@@ -41,16 +55,34 @@
|
|||||||
- htcondor-containered
|
- htcondor-containered
|
||||||
- htcondor
|
- htcondor
|
||||||
|
|
||||||
- hosts: slurm
|
- hosts: slurm, cobald
|
||||||
vars:
|
vars:
|
||||||
container_privileged: True
|
container_privileged: True
|
||||||
num_nodes: 3
|
slurm_num_nodes: 10
|
||||||
roles:
|
roles:
|
||||||
- name: "setup docker"
|
- name: "setup docker"
|
||||||
role: docker
|
role: docker
|
||||||
tags: docker
|
tags: docker
|
||||||
|
- name: "get facts from existing cobald instance (i.e. hostname)"
|
||||||
|
role: cobald_facts
|
||||||
|
vars:
|
||||||
|
container_name: cobald
|
||||||
|
tags: [ slurm, cobald ]
|
||||||
- name: "setup slurm test environment in docker containers"
|
- name: "setup slurm test environment in docker containers"
|
||||||
role: slurm
|
role: slurm
|
||||||
vars:
|
vars:
|
||||||
slurm_user: slurm # or root
|
slurm_user: slurm # or root
|
||||||
|
num_nodes: "{{slurm_num_nodes}}"
|
||||||
|
extra_nodes:
|
||||||
|
- "{{cobald_container_hostname}}" # from cobald_facts, read or generated
|
||||||
|
docker_network: slurm
|
||||||
|
when: '"slurm" in group_names'
|
||||||
tags: slurm
|
tags: slurm
|
||||||
|
- name: "install cobald"
|
||||||
|
role: cobald
|
||||||
|
vars:
|
||||||
|
cobald_slurm: True
|
||||||
|
container_name: cobald
|
||||||
|
# docker_network: slurm # overriden by vars/slurm.yml
|
||||||
|
when: '"cobald" in group_names'
|
||||||
|
tags: cobald
|
||||||
|
|||||||
5
roles/cobald/defaults/main.yml
Normal file
5
roles/cobald/defaults/main.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
influx_admin_user: my-user
|
||||||
|
influx_admin_pw: my-password
|
||||||
|
influx_org: my-org
|
||||||
|
influx_pubport: 28086
|
||||||
|
influx_bucket: batleth
|
||||||
7
roles/cobald/files/cobald-entrypoint.sh
Normal file
7
roles/cobald/files/cobald-entrypoint.sh
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
for i in /usr/local/lib/entrypoints.d/* ; do
|
||||||
|
[ -f $i ] && /bin/sh $i || break
|
||||||
|
done
|
||||||
|
|
||||||
|
exec "${@:-/bin/bash}"
|
||||||
589
roles/cobald/files/grafana-dashboard.json
Normal file
589
roles/cobald/files/grafana-dashboard.json
Normal file
@@ -0,0 +1,589 @@
|
|||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": "-- Grafana --",
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"gnetId": null,
|
||||||
|
"graphTooltip": 0,
|
||||||
|
"id": 1,
|
||||||
|
"iteration": 1623317629899,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"datasource": null,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 4,
|
||||||
|
"panels": [],
|
||||||
|
"title": "Row title",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": "InfluxDB",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "none"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"fill": 1,
|
||||||
|
"fillGradient": 0,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"hiddenSeries": false,
|
||||||
|
"id": 2,
|
||||||
|
"interval": "1s",
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": true,
|
||||||
|
"min": true,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": true
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"maxDataPoints": 200,
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"options": {
|
||||||
|
"alertThreshold": true
|
||||||
|
},
|
||||||
|
"percentage": false,
|
||||||
|
"pluginVersion": "7.5.7",
|
||||||
|
"pointradius": 2,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"groupBy": [
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"$__interval"
|
||||||
|
],
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"type": "fill"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"orderByTime": "ASC",
|
||||||
|
"policy": "default",
|
||||||
|
"query": "f_r = (r, accumulator) => ({\n _value: accumulator._value + (\n if r._value == \"AvailableState\" then 1\n else if r._value == \"DownState\" then -1\n else 0)\n })\n\nnodes = from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> keep(columns: [\"tardis_machine_name\", \"_time\"])\n |> sort(columns: [\"_time\"], desc: true)\n |> unique(column: \"tardis_machine_name\")\n// |> yield()\n\noffset = from(bucket: \"batleth\")\n |> range(start: 0, stop: v.timeRangeStart)\n |> filter(fn: (r) => r._field == \"state\")\n |> group(columns: [\"tardis_machine_name\", \"machine_type\"])\n |> reduce(fn: f_r, identity: {_value: 0})\n |> duplicate(column: \"_stop\", as: \"_time\")\n\nnew = from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._field == \"state\")\n// |> filter(fn: (r) => r.tardis_machine_name == \"${machine}\")\n |> group(columns: [\"tardis_machine_name\", \"machine_type\"])\n |> window(every: $__interval)\n |> reduce(fn: f_r, identity: {_value: 0})\n |> duplicate(column: \"_stop\", as: \"_time\")\n\nunion(tables: [offset, new])\n |> window(every: inf)\n |> cumulativeSum()\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> yield()\n",
|
||||||
|
"refId": "A",
|
||||||
|
"resultFormat": "time_series",
|
||||||
|
"select": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"type": "field"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [],
|
||||||
|
"type": "mean"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"tags": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeRegions": [],
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "nodes running",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "none",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"align": false,
|
||||||
|
"alignLevel": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": "InfluxDB",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"fill": 1,
|
||||||
|
"fillGradient": 0,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"hiddenSeries": false,
|
||||||
|
"id": 8,
|
||||||
|
"interval": "1s",
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"maxDataPoints": null,
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"options": {
|
||||||
|
"alertThreshold": true
|
||||||
|
},
|
||||||
|
"percentage": false,
|
||||||
|
"pluginVersion": "7.5.7",
|
||||||
|
"pointradius": 2,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"groupBy": [
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"$__interval"
|
||||||
|
],
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"type": "fill"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"orderByTime": "ASC",
|
||||||
|
"policy": "default",
|
||||||
|
"query": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"tardis_pipeline\")\n |> filter(fn: (r) => r._field == \"demand\" or r._field == \"supply\")\n |> drop(columns: [\"host\"])\n |> aggregateWindow(every: $__interval, fn: mean)\n |> yield()",
|
||||||
|
"queryType": "randomWalk",
|
||||||
|
"refId": "A",
|
||||||
|
"resultFormat": "time_series",
|
||||||
|
"select": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"type": "field"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [],
|
||||||
|
"type": "mean"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"tags": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeRegions": [],
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "pipeline demand/supply (mean)",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"$$hashKey": "object:115",
|
||||||
|
"decimals": null,
|
||||||
|
"format": "short",
|
||||||
|
"label": "cpus",
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"$$hashKey": "object:116",
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"align": false,
|
||||||
|
"alignLevel": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": "InfluxDB",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"fill": 1,
|
||||||
|
"fillGradient": 0,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 10
|
||||||
|
},
|
||||||
|
"hiddenSeries": false,
|
||||||
|
"id": 6,
|
||||||
|
"interval": "1s",
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"options": {
|
||||||
|
"alertThreshold": true
|
||||||
|
},
|
||||||
|
"percentage": false,
|
||||||
|
"pluginVersion": "7.5.7",
|
||||||
|
"pointradius": 2,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"groupBy": [
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"$__interval"
|
||||||
|
],
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"type": "fill"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"orderByTime": "ASC",
|
||||||
|
"policy": "default",
|
||||||
|
"query": "f = (r, accumulator) => ({\n _value: accumulator._value + (if r._value == \"AvailableState\" then 1 else if r._value == \"DownState\" then -1 else 0)\n })\n\nfrom(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._field == \"state\")\n |> group(columns: [\"tardis_machine_name\"])\n |> window(every: $__interval)\n |> reduce(fn: f, identity: {_value: 0})\n |> duplicate(column: \"_stop\", as: \"_time\")\n |> window(every: inf, timeColumn: \"_time\")\n |> yield()",
|
||||||
|
"queryType": "randomWalk",
|
||||||
|
"refId": "A",
|
||||||
|
"resultFormat": "time_series",
|
||||||
|
"select": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"type": "field"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [],
|
||||||
|
"type": "mean"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"tags": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeRegions": [],
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "node fluctuation",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"align": false,
|
||||||
|
"alignLevel": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": "InfluxDB",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"fill": 1,
|
||||||
|
"fillGradient": 0,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 10
|
||||||
|
},
|
||||||
|
"hiddenSeries": false,
|
||||||
|
"id": 10,
|
||||||
|
"interval": "1s",
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"maxDataPoints": null,
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"options": {
|
||||||
|
"alertThreshold": true
|
||||||
|
},
|
||||||
|
"percentage": false,
|
||||||
|
"pluginVersion": "7.5.7",
|
||||||
|
"pointradius": 2,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"groupBy": [
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"$__interval"
|
||||||
|
],
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"type": "fill"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"orderByTime": "ASC",
|
||||||
|
"policy": "default",
|
||||||
|
"query": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"tardis_pipeline\")\n |> filter(fn: (r) => r._field == \"utilisation\" or r._field == \"allocation\")\n |> keep(columns: [\"_time\", \"_measurement\", \"_field\", \"_value\", \"tardis_machine_name\"])\n |> aggregateWindow(every: $__interval, fn: mean)\n |> yield()",
|
||||||
|
"queryType": "randomWalk",
|
||||||
|
"refId": "A",
|
||||||
|
"resultFormat": "time_series",
|
||||||
|
"select": [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"params": [
|
||||||
|
"value"
|
||||||
|
],
|
||||||
|
"type": "field"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"params": [],
|
||||||
|
"type": "mean"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"tags": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeRegions": [],
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "pipeline (utilization/allocation)",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"align": false,
|
||||||
|
"alignLevel": null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "5s",
|
||||||
|
"schemaVersion": 27,
|
||||||
|
"style": "dark",
|
||||||
|
"tags": [],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"allValue": null,
|
||||||
|
"current": {
|
||||||
|
"selected": false,
|
||||||
|
"text": "cobald-xvmcqc",
|
||||||
|
"value": "cobald-xvmcqc"
|
||||||
|
},
|
||||||
|
"datasource": "InfluxDB",
|
||||||
|
"definition": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> keep(columns: [\"tardis_machine_name\", \"_time\"])\n |> sort(columns: [\"_time\"], desc: true)\n |> unique(column: \"tardis_machine_name\")",
|
||||||
|
"description": null,
|
||||||
|
"error": null,
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": false,
|
||||||
|
"label": null,
|
||||||
|
"multi": false,
|
||||||
|
"name": "machine",
|
||||||
|
"options": [],
|
||||||
|
"query": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> keep(columns: [\"tardis_machine_name\", \"_time\"])\n |> sort(columns: [\"_time\"], desc: true)\n |> unique(column: \"tardis_machine_name\")",
|
||||||
|
"refresh": 2,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"sort": 0,
|
||||||
|
"tagValuesQuery": "",
|
||||||
|
"tags": [],
|
||||||
|
"tagsQuery": "",
|
||||||
|
"type": "query",
|
||||||
|
"useTags": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-5m",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "cobald",
|
||||||
|
"uid": "urDuvE6Gk",
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
794
roles/cobald/files/influxdb-dashboard-cobald.json
Normal file
794
roles/cobald/files/influxdb-dashboard-cobald.json
Normal file
@@ -0,0 +1,794 @@
|
|||||||
|
{
|
||||||
|
"meta": {
|
||||||
|
"version": "1",
|
||||||
|
"type": "dashboard",
|
||||||
|
"name": "cobald-Template",
|
||||||
|
"description": "template created from dashboard: cobald"
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"data": {
|
||||||
|
"type": "dashboard",
|
||||||
|
"attributes": {
|
||||||
|
"name": "cobald",
|
||||||
|
"description": ""
|
||||||
|
},
|
||||||
|
"relationships": {
|
||||||
|
"label": {
|
||||||
|
"data": []
|
||||||
|
},
|
||||||
|
"cell": {
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"type": "cell",
|
||||||
|
"id": "07900a722c363000"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "cell",
|
||||||
|
"id": "07900a7236f63000"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "cell",
|
||||||
|
"id": "07900a723cf63000"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "cell",
|
||||||
|
"id": "07900a7243f63000"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "cell",
|
||||||
|
"id": "079e694f29581000"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "cell",
|
||||||
|
"id": "079e6e037c181000"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"variable": {
|
||||||
|
"data": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"included": [
|
||||||
|
{
|
||||||
|
"id": "07900a722c363000",
|
||||||
|
"type": "cell",
|
||||||
|
"attributes": {
|
||||||
|
"x": 4,
|
||||||
|
"y": 0,
|
||||||
|
"w": 4,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"relationships": {
|
||||||
|
"view": {
|
||||||
|
"data": {
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a722c363000"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "07900a7236f63000",
|
||||||
|
"type": "cell",
|
||||||
|
"attributes": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 0,
|
||||||
|
"w": 4,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"relationships": {
|
||||||
|
"view": {
|
||||||
|
"data": {
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a7236f63000"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "07900a723cf63000",
|
||||||
|
"type": "cell",
|
||||||
|
"attributes": {
|
||||||
|
"x": 4,
|
||||||
|
"y": 4,
|
||||||
|
"w": 4,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"relationships": {
|
||||||
|
"view": {
|
||||||
|
"data": {
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a723cf63000"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "07900a7243f63000",
|
||||||
|
"type": "cell",
|
||||||
|
"attributes": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 4,
|
||||||
|
"w": 4,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"relationships": {
|
||||||
|
"view": {
|
||||||
|
"data": {
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a7243f63000"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "079e694f29581000",
|
||||||
|
"type": "cell",
|
||||||
|
"attributes": {
|
||||||
|
"x": 8,
|
||||||
|
"y": 0,
|
||||||
|
"w": 4,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"relationships": {
|
||||||
|
"view": {
|
||||||
|
"data": {
|
||||||
|
"type": "view",
|
||||||
|
"id": "079e694f29581000"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "079e6e037c181000",
|
||||||
|
"type": "cell",
|
||||||
|
"attributes": {
|
||||||
|
"x": 8,
|
||||||
|
"y": 4,
|
||||||
|
"w": 4,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"relationships": {
|
||||||
|
"view": {
|
||||||
|
"data": {
|
||||||
|
"type": "view",
|
||||||
|
"id": "079e6e037c181000"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a722c363000",
|
||||||
|
"attributes": {
|
||||||
|
"name": "nodes running",
|
||||||
|
"properties": {
|
||||||
|
"shape": "chronograf-v2",
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"text": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._field == \"state\")\n |> group()\n |> window(every: 10s)\n |> reduce(fn: (r, accumulator) => ({\n _value: accumulator._value + (\n if r._value == \"AvailableState\" then 1 \n else if r._value == \"DownState\" then -1 \n else 0)\n }), identity: {_value: 0})\n |> duplicate(column: \"_stop\", as: \"_time\")\n |> window(every: inf) //, timeColumn: \"_time\")\n |> cumulativeSum()\n// |> reduce(fn: (r, accumulator) => ({r with x: r._value * 2}), identity: {x:0})\n// |> map(fn: (r) => ({r with vnew: r._value*2}))\n// |> integral(unit: 10s, timeColumn: \"_stop\")\n// |> window(every: inf) //, timeColumn: \"_stop\")\n |> yield()",
|
||||||
|
"editMode": "advanced",
|
||||||
|
"name": "",
|
||||||
|
"builderConfig": {
|
||||||
|
"buckets": [],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "_measurement",
|
||||||
|
"values": [],
|
||||||
|
"aggregateFunctionType": "filter"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"functions": [
|
||||||
|
{
|
||||||
|
"name": "mean"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"aggregateWindow": {
|
||||||
|
"period": "auto",
|
||||||
|
"fillValues": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"axes": {
|
||||||
|
"x": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "xy",
|
||||||
|
"legend": {},
|
||||||
|
"geom": "line",
|
||||||
|
"colors": [
|
||||||
|
{
|
||||||
|
"id": "9b960932-18d9-4f57-80ba-24998a06613d",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#31C0F6",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "c1742651-0d5e-4148-b9c0-92beb642417a",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#A500A5",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "756fa8cb-9d9c-4e45-9a4f-f2b106b0216a",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#FF7E27",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"note": "",
|
||||||
|
"showNoteWhenEmpty": false,
|
||||||
|
"xColumn": "_time",
|
||||||
|
"generateXAxisTicks": [],
|
||||||
|
"xTotalTicks": 0,
|
||||||
|
"xTickStart": 0,
|
||||||
|
"xTickStep": 0,
|
||||||
|
"yColumn": "_value",
|
||||||
|
"generateYAxisTicks": [],
|
||||||
|
"yTotalTicks": 0,
|
||||||
|
"yTickStart": 0,
|
||||||
|
"yTickStep": 0,
|
||||||
|
"shadeBelow": false,
|
||||||
|
"position": "overlaid",
|
||||||
|
"timeFormat": "",
|
||||||
|
"hoverDimension": "auto",
|
||||||
|
"legendColorizeRows": true,
|
||||||
|
"legendOpacity": 1,
|
||||||
|
"legendOrientationThreshold": 100000000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a7236f63000",
|
||||||
|
"attributes": {
|
||||||
|
"name": "Name this Cell",
|
||||||
|
"properties": {
|
||||||
|
"shape": "chronograf-v2",
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"text": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"tardis_machine_name\"] == \"678162c190d5\")\n |> window(every: 10s)\n |> count()\n |> duplicate(column: \"_stop\", as: \"_time\")\n |> window(every: inf)\n |> yield()",
|
||||||
|
"editMode": "advanced",
|
||||||
|
"name": "",
|
||||||
|
"builderConfig": {
|
||||||
|
"buckets": [],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "_measurement",
|
||||||
|
"values": [],
|
||||||
|
"aggregateFunctionType": "filter"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"functions": [
|
||||||
|
{
|
||||||
|
"name": "mean"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"aggregateWindow": {
|
||||||
|
"period": "auto",
|
||||||
|
"fillValues": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"axes": {
|
||||||
|
"x": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "xy",
|
||||||
|
"legend": {},
|
||||||
|
"geom": "line",
|
||||||
|
"colors": [
|
||||||
|
{
|
||||||
|
"id": "2566435b-7ee0-4222-8ac0-b7f14ab783d9",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#31C0F6",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "9263bcff-35a0-4025-bacd-68a1bef54784",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#A500A5",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "2f04bd8c-2203-4be6-bc34-c25720d24379",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#FF7E27",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"note": "",
|
||||||
|
"showNoteWhenEmpty": false,
|
||||||
|
"xColumn": "_time",
|
||||||
|
"generateXAxisTicks": [],
|
||||||
|
"xTotalTicks": 0,
|
||||||
|
"xTickStart": 0,
|
||||||
|
"xTickStep": 0,
|
||||||
|
"yColumn": "_value",
|
||||||
|
"generateYAxisTicks": [],
|
||||||
|
"yTotalTicks": 0,
|
||||||
|
"yTickStart": 0,
|
||||||
|
"yTickStep": 0,
|
||||||
|
"shadeBelow": false,
|
||||||
|
"position": "overlaid",
|
||||||
|
"timeFormat": "",
|
||||||
|
"hoverDimension": "auto",
|
||||||
|
"legendColorizeRows": true,
|
||||||
|
"legendOpacity": 1,
|
||||||
|
"legendOrientationThreshold": 100000000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a723cf63000",
|
||||||
|
"attributes": {
|
||||||
|
"name": "node fluctuation",
|
||||||
|
"properties": {
|
||||||
|
"shape": "chronograf-v2",
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"text": "f = (r, accumulator) => ({\n _value: accumulator._value + (if r._value == \"AvailableState\" then 1 else if r._value == \"DownState\" then -1 else 0)\n })\n\nfrom(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._field == \"state\")\n |> group()\n |> window(every: 10s)\n |> reduce(fn: f, identity: {_value: 0})\n |> duplicate(column: \"_stop\", as: \"_time\")\n |> window(every: inf, timeColumn: \"_time\")\n |> yield()",
|
||||||
|
"editMode": "advanced",
|
||||||
|
"name": "",
|
||||||
|
"builderConfig": {
|
||||||
|
"buckets": [],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "_measurement",
|
||||||
|
"values": [],
|
||||||
|
"aggregateFunctionType": "filter"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"functions": [
|
||||||
|
{
|
||||||
|
"name": "mean"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"aggregateWindow": {
|
||||||
|
"period": "auto",
|
||||||
|
"fillValues": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"axes": {
|
||||||
|
"x": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "xy",
|
||||||
|
"legend": {},
|
||||||
|
"geom": "line",
|
||||||
|
"colors": [
|
||||||
|
{
|
||||||
|
"id": "9b960932-18d9-4f57-80ba-24998a06613d",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#31C0F6",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "c1742651-0d5e-4148-b9c0-92beb642417a",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#A500A5",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "756fa8cb-9d9c-4e45-9a4f-f2b106b0216a",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#FF7E27",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"note": "",
|
||||||
|
"showNoteWhenEmpty": false,
|
||||||
|
"xColumn": "_time",
|
||||||
|
"generateXAxisTicks": [],
|
||||||
|
"xTotalTicks": 0,
|
||||||
|
"xTickStart": 0,
|
||||||
|
"xTickStep": 0,
|
||||||
|
"yColumn": "_value",
|
||||||
|
"generateYAxisTicks": [],
|
||||||
|
"yTotalTicks": 0,
|
||||||
|
"yTickStart": 0,
|
||||||
|
"yTickStep": 0,
|
||||||
|
"shadeBelow": false,
|
||||||
|
"position": "overlaid",
|
||||||
|
"timeFormat": "",
|
||||||
|
"hoverDimension": "auto",
|
||||||
|
"legendColorizeRows": true,
|
||||||
|
"legendOpacity": 1,
|
||||||
|
"legendOrientationThreshold": 100000000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "view",
|
||||||
|
"id": "07900a7243f63000",
|
||||||
|
"attributes": {
|
||||||
|
"name": "states",
|
||||||
|
"properties": {
|
||||||
|
"shape": "chronograf-v2",
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"text": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._field == \"state\")\n |> group(columns: [\"_value\"], mode: \"by\")\n |> duplicate(column: \"_value\", as: \"state\")\n |> window(every: 10s)\n |> count(column: \"state\")\n |> rename(columns: {\"_value\": \"_field\", \"state\": \"_value\"})\n |> group(columns: [\"_field\"])\n |> yield()",
|
||||||
|
"editMode": "advanced",
|
||||||
|
"name": "",
|
||||||
|
"builderConfig": {
|
||||||
|
"buckets": [],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "_measurement",
|
||||||
|
"values": [],
|
||||||
|
"aggregateFunctionType": "filter"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"functions": [
|
||||||
|
{
|
||||||
|
"name": "mean"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"aggregateWindow": {
|
||||||
|
"period": "auto",
|
||||||
|
"fillValues": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"axes": {
|
||||||
|
"x": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "xy",
|
||||||
|
"legend": {},
|
||||||
|
"geom": "line",
|
||||||
|
"colors": [
|
||||||
|
{
|
||||||
|
"id": "9b960932-18d9-4f57-80ba-24998a06613d",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#31C0F6",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "c1742651-0d5e-4148-b9c0-92beb642417a",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#A500A5",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "756fa8cb-9d9c-4e45-9a4f-f2b106b0216a",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#FF7E27",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"note": "",
|
||||||
|
"showNoteWhenEmpty": false,
|
||||||
|
"xColumn": "_stop",
|
||||||
|
"generateXAxisTicks": [],
|
||||||
|
"xTotalTicks": 0,
|
||||||
|
"xTickStart": 0,
|
||||||
|
"xTickStep": 0,
|
||||||
|
"yColumn": "_value",
|
||||||
|
"generateYAxisTicks": [],
|
||||||
|
"yTotalTicks": 0,
|
||||||
|
"yTickStart": 0,
|
||||||
|
"yTickStep": 0,
|
||||||
|
"shadeBelow": false,
|
||||||
|
"position": "overlaid",
|
||||||
|
"timeFormat": "",
|
||||||
|
"hoverDimension": "auto",
|
||||||
|
"legendColorizeRows": true,
|
||||||
|
"legendOpacity": 1,
|
||||||
|
"legendOrientationThreshold": 100000000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "view",
|
||||||
|
"id": "079e694f29581000",
|
||||||
|
"attributes": {
|
||||||
|
"name": "pipeline (demand / supply)",
|
||||||
|
"properties": {
|
||||||
|
"shape": "chronograf-v2",
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"text": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"tardis_pipeline\")\n |> filter(fn: (r) => r._field == \"demand\" or r._field == \"supply\")\n// |> filter(fn: (r) => r._field == \"state\")\n// |> group()\n// |> window(every: 10s)\n// |> duplicate(column: \"_stop\", as: \"_time\")\n// |> window(every: inf) //, timeColumn: \"_time\")\n// |> cumulativeSum()\n |> yield()",
|
||||||
|
"editMode": "advanced",
|
||||||
|
"name": "",
|
||||||
|
"builderConfig": {
|
||||||
|
"buckets": [],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "_measurement",
|
||||||
|
"values": [],
|
||||||
|
"aggregateFunctionType": "filter"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"functions": [
|
||||||
|
{
|
||||||
|
"name": "mean"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"aggregateWindow": {
|
||||||
|
"period": "auto",
|
||||||
|
"fillValues": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"axes": {
|
||||||
|
"x": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "xy",
|
||||||
|
"legend": {},
|
||||||
|
"geom": "line",
|
||||||
|
"colors": [
|
||||||
|
{
|
||||||
|
"id": "4ef29481-ecf3-4a09-b0f5-e34e8d3e50b5",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#31C0F6",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "719de04a-f70f-4c54-a1bb-982a9d13dbae",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#A500A5",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "8d5c1f25-3801-4cdd-ad40-b8e2e78342a0",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#FF7E27",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"note": "",
|
||||||
|
"showNoteWhenEmpty": false,
|
||||||
|
"xColumn": "_time",
|
||||||
|
"generateXAxisTicks": [],
|
||||||
|
"xTotalTicks": 0,
|
||||||
|
"xTickStart": 0,
|
||||||
|
"xTickStep": 0,
|
||||||
|
"yColumn": "_value",
|
||||||
|
"generateYAxisTicks": [],
|
||||||
|
"yTotalTicks": 0,
|
||||||
|
"yTickStart": 0,
|
||||||
|
"yTickStep": 0,
|
||||||
|
"shadeBelow": false,
|
||||||
|
"position": "overlaid",
|
||||||
|
"timeFormat": "",
|
||||||
|
"hoverDimension": "auto",
|
||||||
|
"legendColorizeRows": true,
|
||||||
|
"legendOpacity": 1,
|
||||||
|
"legendOrientationThreshold": 100000000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "view",
|
||||||
|
"id": "079e6e037c181000",
|
||||||
|
"attributes": {
|
||||||
|
"name": "pipeline (Utilization / Allocation)",
|
||||||
|
"properties": {
|
||||||
|
"shape": "chronograf-v2",
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"text": "from(bucket: \"batleth\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"tardis_pipeline\")\n |> filter(fn: (r) => r._field == \"utilisation\" or r._field == \"allocation\")\n// |> filter(fn: (r) => r._field == \"state\")\n// |> group()\n// |> window(every: 10s)\n// |> duplicate(column: \"_stop\", as: \"_time\")\n// |> window(every: inf) //, timeColumn: \"_time\")\n// |> cumulativeSum()\n |> yield()",
|
||||||
|
"editMode": "advanced",
|
||||||
|
"name": "",
|
||||||
|
"builderConfig": {
|
||||||
|
"buckets": [],
|
||||||
|
"tags": [
|
||||||
|
{
|
||||||
|
"key": "_measurement",
|
||||||
|
"values": [],
|
||||||
|
"aggregateFunctionType": "filter"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"functions": [
|
||||||
|
{
|
||||||
|
"name": "mean"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"aggregateWindow": {
|
||||||
|
"period": "auto",
|
||||||
|
"fillValues": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"axes": {
|
||||||
|
"x": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
},
|
||||||
|
"y": {
|
||||||
|
"bounds": [
|
||||||
|
"",
|
||||||
|
""
|
||||||
|
],
|
||||||
|
"label": "",
|
||||||
|
"prefix": "",
|
||||||
|
"suffix": "",
|
||||||
|
"base": "10",
|
||||||
|
"scale": "linear"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "xy",
|
||||||
|
"legend": {},
|
||||||
|
"geom": "line",
|
||||||
|
"colors": [
|
||||||
|
{
|
||||||
|
"id": "4ef29481-ecf3-4a09-b0f5-e34e8d3e50b5",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#31C0F6",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "719de04a-f70f-4c54-a1bb-982a9d13dbae",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#A500A5",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "8d5c1f25-3801-4cdd-ad40-b8e2e78342a0",
|
||||||
|
"type": "scale",
|
||||||
|
"hex": "#FF7E27",
|
||||||
|
"name": "Nineteen Eighty Four",
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"note": "",
|
||||||
|
"showNoteWhenEmpty": false,
|
||||||
|
"xColumn": "_time",
|
||||||
|
"generateXAxisTicks": [],
|
||||||
|
"xTotalTicks": 0,
|
||||||
|
"xTickStart": 0,
|
||||||
|
"xTickStep": 0,
|
||||||
|
"yColumn": "_value",
|
||||||
|
"generateYAxisTicks": [],
|
||||||
|
"yTotalTicks": 0,
|
||||||
|
"yTickStart": 0,
|
||||||
|
"yTickStep": 0,
|
||||||
|
"shadeBelow": false,
|
||||||
|
"position": "overlaid",
|
||||||
|
"timeFormat": "",
|
||||||
|
"hoverDimension": "auto",
|
||||||
|
"legendColorizeRows": true,
|
||||||
|
"legendOpacity": 1,
|
||||||
|
"legendOrientationThreshold": 100000000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"labels": []
|
||||||
|
}
|
||||||
6
roles/cobald/files/influxdb.repo
Normal file
6
roles/cobald/files/influxdb.repo
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
[influxdb]
|
||||||
|
name = InfluxDB Repository - RHEL \$releasever
|
||||||
|
baseurl = https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable
|
||||||
|
enabled = 1
|
||||||
|
gpgcheck = 1
|
||||||
|
gpgkey = https://repos.influxdata.com/influxdb.key
|
||||||
5
roles/cobald/files/init-cobaldmodules.sh
Normal file
5
roles/cobald/files/init-cobaldmodules.sh
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
[ -f /usr/local/lib/cobaldmodules/setup.py -a \
|
||||||
|
-d /usr/local/lib/cobaldmodules/cobaldmodules ] && \
|
||||||
|
pip3 install --no-deps --editable /usr/local/lib/cobaldmodules
|
||||||
8
roles/cobald/files/telegraf.Dockerfile
Normal file
8
roles/cobald/files/telegraf.Dockerfile
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
FROM centos:7
|
||||||
|
|
||||||
|
COPY influxdb.repo /etc/yum.repos.d/influxdb.repo
|
||||||
|
|
||||||
|
RUN yum -y install telegraf &&\
|
||||||
|
yum clean all && rm -rf /var/cache/yum
|
||||||
|
|
||||||
|
CMD telegraf
|
||||||
11
roles/cobald/library/TODO.md
Normal file
11
roles/cobald/library/TODO.md
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
# Tests
|
||||||
|
ANSIBLE_LIBRARY=. ansible -m influx_bucket -a "base='http://192.168.122.140:28086' org='my-org' auth_token='87-fEnSlQldFi1T_CLHsrHxH-T9VKey-qzUbVH6tmR2QzL4oZzbUPwzS1wzOoIkyfmyGbRv75yLjYfztxziivw==' name='bucky' description='test 123'" localhost -vvv
|
||||||
|
ANSIBLE_LIBRARY=. ansible -m influx_token -a "base='http://192.168.122.140:28086' org='my-org' auth_token='87-fEnSlQldFi1T_CLHsrHxH-T9VKey-qzUbVH6tmR2QzL4oZzbUPwzS1wzOoIkyfmyGbRv75yLjYfztxziivw==' key='foo' description='test 123' permissions=\"{{'[{\\\"action\\\": \\\"write\\\",\\\"resource\\\": {\\\"type\\\": \\\"buckets\\\"} }]'|from_json}}\"" localhost -vvv
|
||||||
|
ANSIBLE_LIBRARY=. ansible -m influx_dashboard -create -a "base='http://192.168.122.140:28086' org='my-org' token='2Mji-PvTzgn2oie5p36pJ-vxqWCnxczMWGrnYz2nUHj6Q6XvdIGiLPmK4DjX16KGhOjxQ5dWymDusE8qJrhFFg==' data='{{lookup(\"file\", \"../files/influxdb-dashboard-cobald.json\")}}'" localhost -vvv
|
||||||
|
Missing: lot of stuff, e.g. missing tokens, invalid data like bucket instead of buckets ...
|
||||||
|
|
||||||
|
# TODO
|
||||||
|
* tests
|
||||||
|
* state (present/absent)
|
||||||
|
* `module_utils/urls.py` (https://github.com/ansible/ansible/blob/devel/lib/ansible/module_utils/urls.py)
|
||||||
|
* see module notes
|
||||||
161
roles/cobald/library/influx_bucket.py
Executable file
161
roles/cobald/library/influx_bucket.py
Executable file
@@ -0,0 +1,161 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
import requests
|
||||||
|
from ansible.module_utils.basic import AnsibleModule
|
||||||
|
|
||||||
|
DOCUMENTATION = r'''
|
||||||
|
---
|
||||||
|
module: influx2_bucket
|
||||||
|
short_description: create bucket in influxdb2
|
||||||
|
description: create bucket in influxdb2
|
||||||
|
notes:
|
||||||
|
- just works with influxdb version 2
|
||||||
|
- does not remove buckets
|
||||||
|
- no way to configure data retention
|
||||||
|
|
||||||
|
options:
|
||||||
|
base:
|
||||||
|
description: URL for path, e.g. `https://localhost:8086`
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
org:
|
||||||
|
description: influxdb2 organisation
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
auth_token:
|
||||||
|
description: influxdb2 authentication token
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
name:
|
||||||
|
description: name of the bucket
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
force:
|
||||||
|
description: force creation even if bucket already exists
|
||||||
|
(adds a new one)
|
||||||
|
type: bool
|
||||||
|
required: False
|
||||||
|
default: False
|
||||||
|
author:
|
||||||
|
- Thorsten M. (@thoto)
|
||||||
|
'''
|
||||||
|
|
||||||
|
EXAMPLES = r'''
|
||||||
|
- name: "fetch auth token"
|
||||||
|
raw: influx auth list --user my-user --hide-headers | cut -f 3
|
||||||
|
register: influx_token_fetch
|
||||||
|
delegate_to: ed-influxdb-2
|
||||||
|
|
||||||
|
- name: "create bucket"
|
||||||
|
influx_bucket:
|
||||||
|
base: "http://localhost:8086"
|
||||||
|
org: "my-org"
|
||||||
|
token: "{{influx_token_fetch.stdout_lines[0]}}"
|
||||||
|
name: "bucky"
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def get_org_id(base, org_name, h):
|
||||||
|
ro = requests.get("{base}/api/v2/orgs".format(base=base), headers=h,
|
||||||
|
json={"org": org_name})
|
||||||
|
ro.raise_for_status()
|
||||||
|
org_id = [o["id"] for o in ro.json()["orgs"] if o["name"] == org_name]
|
||||||
|
return org_id[0]
|
||||||
|
|
||||||
|
|
||||||
|
class Bucket:
|
||||||
|
def __init__(self, base, h, org, description, name):
|
||||||
|
self.base = base
|
||||||
|
self.h = h
|
||||||
|
self.org_id = get_org_id(base, org, h)
|
||||||
|
self.description = description
|
||||||
|
self.name = name
|
||||||
|
|
||||||
|
self.result = None
|
||||||
|
self.f = None
|
||||||
|
|
||||||
|
def check(self):
|
||||||
|
ra = requests.get("{base}/api/v2/buckets".format(
|
||||||
|
base=self.base),
|
||||||
|
params={"orgID": self.org_id, "name": self.name},
|
||||||
|
headers=self.h)
|
||||||
|
if ra.status_code == 404:
|
||||||
|
# orgID + name -> 404 on empty set. Just name -> 200 but buckets=[]
|
||||||
|
x = []
|
||||||
|
else:
|
||||||
|
ra.raise_for_status()
|
||||||
|
x = [i for i in ra.json()["buckets"]
|
||||||
|
if self.name == i["name"] and i["orgID"] == self.org_id]
|
||||||
|
assert(len(x) == 1 or len(x) == 0)
|
||||||
|
|
||||||
|
update = None
|
||||||
|
if len(x) == 0: # create
|
||||||
|
self.result = None
|
||||||
|
self.f = lambda: self._create({
|
||||||
|
"orgID": self.org_id,
|
||||||
|
"description": self.description if self.description else None,
|
||||||
|
"name": self.name,
|
||||||
|
"retentionRules": []
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
self.result = x[0]
|
||||||
|
if self.description == x[0].get("description", ""):
|
||||||
|
return False # everything matches -> no change needed
|
||||||
|
else:
|
||||||
|
self.result = x[0]
|
||||||
|
update = {"id": x[0]["id"],
|
||||||
|
"description": self.description}
|
||||||
|
self.f = lambda: self._update(**update)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
if not self.f:
|
||||||
|
self.check()
|
||||||
|
self.f()
|
||||||
|
|
||||||
|
def _update(self, id, description):
|
||||||
|
ra = requests.patch(
|
||||||
|
"{base}/api/v2/buckets/{id}".format(
|
||||||
|
base=self.base, id=id),
|
||||||
|
headers=self.h, json={"description": description})
|
||||||
|
ra.raise_for_status()
|
||||||
|
return ra
|
||||||
|
|
||||||
|
def _create(self, data):
|
||||||
|
ra = requests.post("{base}/api/v2/buckets".format(base=self.base),
|
||||||
|
headers=self.h, json=data)
|
||||||
|
ra.raise_for_status()
|
||||||
|
self.result = ra.json()
|
||||||
|
return ra
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
result = dict(changed=False, message="")
|
||||||
|
module = AnsibleModule(
|
||||||
|
argument_spec=dict(
|
||||||
|
base=dict(type="str", required=True),
|
||||||
|
org=dict(type="str", required=True),
|
||||||
|
auth_token=dict(type="str", required=True),
|
||||||
|
name=dict(type="str", required=True),
|
||||||
|
description=dict(type="str", default=""),
|
||||||
|
force=dict(type="bool", default=False),
|
||||||
|
),
|
||||||
|
supports_check_mode=True
|
||||||
|
)
|
||||||
|
h = {"Authorization": "Token {token}".format(
|
||||||
|
token=module.params["auth_token"])}
|
||||||
|
b = Bucket(module.params["base"], h, org=module.params["org"],
|
||||||
|
description=module.params["description"],
|
||||||
|
name=module.params["name"])
|
||||||
|
|
||||||
|
changed = b.check()
|
||||||
|
if b.result:
|
||||||
|
result['bucket_id'] = b.result["id"]
|
||||||
|
result['changed'] = changed
|
||||||
|
|
||||||
|
if module.check_mode:
|
||||||
|
module.exit_json(**result)
|
||||||
|
|
||||||
|
if changed or module.params["force"]:
|
||||||
|
b.run()
|
||||||
|
result['bucket_id'] = b.result["id"]
|
||||||
|
module.exit_json(**result)
|
||||||
147
roles/cobald/library/influx_dashboard.py
Executable file
147
roles/cobald/library/influx_dashboard.py
Executable file
@@ -0,0 +1,147 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
from ansible.module_utils.basic import AnsibleModule
|
||||||
|
|
||||||
|
DOCUMENTATION = r'''
|
||||||
|
---
|
||||||
|
module: influx2_dashboard
|
||||||
|
short_description: create dashboard in influxdb2
|
||||||
|
description: create dashboard in influxdb2
|
||||||
|
notes:
|
||||||
|
- just works with influxdb version 2
|
||||||
|
- does not create dashboard description
|
||||||
|
- does not update dashboards
|
||||||
|
- just creates a dashboard if it does not exist.
|
||||||
|
|
||||||
|
options:
|
||||||
|
base:
|
||||||
|
description: URL for path, e.g. `https://localhost:8086`
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
org:
|
||||||
|
description: influxdb2 organisation
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
auth_token:
|
||||||
|
description: influxdb2 authentication token
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
data:
|
||||||
|
description: exported dashboard json file
|
||||||
|
type: json
|
||||||
|
required: True
|
||||||
|
force:
|
||||||
|
description: force creation even if dashboard already exists
|
||||||
|
(adds a new one)
|
||||||
|
type: bool
|
||||||
|
required: False
|
||||||
|
default: False
|
||||||
|
author:
|
||||||
|
- Thorsten M. (@thoto)
|
||||||
|
'''
|
||||||
|
|
||||||
|
EXAMPLES = r'''
|
||||||
|
- name: "fetch auth token"
|
||||||
|
raw: influx auth list --user my-user --hide-headers | cut -f 3
|
||||||
|
register: influx_token_fetch
|
||||||
|
delegate_to: ed-influxdb-2
|
||||||
|
|
||||||
|
- name: "create dashboard"
|
||||||
|
influx_dashboard:
|
||||||
|
base: "http://localhost:8086"
|
||||||
|
org: "my-org"
|
||||||
|
auth_token: "{{influx_token_fetch.stdout_lines[0]}}"
|
||||||
|
data: "{{lookup('file', 'influxdb-dashboard-cobald.json')}}"
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def parse_dashboard_template(data):
|
||||||
|
j = json.loads(data)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'name': j["content"]["data"]["attributes"]["name"],
|
||||||
|
'cells': {i["relationships"]["view"]["data"]["id"]: i["attributes"]
|
||||||
|
for i in j["content"]["included"] if i["type"] == "cell"},
|
||||||
|
'views': {i["id"]: i["attributes"]
|
||||||
|
for i in j["content"]["included"] if i["type"] == "view"},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_auth(base, org_name, token):
|
||||||
|
h = {"Authorization": "Token {token}".format(token=token)}
|
||||||
|
|
||||||
|
rm = requests.get("{base}/api/v2/me".format(base=base), headers=h)
|
||||||
|
rm.raise_for_status()
|
||||||
|
# me_name = rm.json()["name"]
|
||||||
|
me = rm.json()["id"]
|
||||||
|
|
||||||
|
ro = requests.get("{base}/api/v2/orgs".format(base=base), headers=h,
|
||||||
|
json={"userID": me})
|
||||||
|
ro.raise_for_status()
|
||||||
|
org_id = [o["id"] for o in ro.json()["orgs"] if o["name"] == org_name]
|
||||||
|
|
||||||
|
return {"org_id": org_id[0], "org_name": org_name, "uid": me, "h": h}
|
||||||
|
|
||||||
|
|
||||||
|
def check(base, auth, dashboard):
|
||||||
|
h = auth["h"]
|
||||||
|
rd = requests.get("{base}/api/v2/dashboards".format(base=base), headers=h)
|
||||||
|
rd.raise_for_status()
|
||||||
|
x = [i for i in rd.json()["dashboards"]
|
||||||
|
if i["name"] == dashboard["name"] and i["orgID"] == auth["org_id"]]
|
||||||
|
return len(x) == 0, x
|
||||||
|
|
||||||
|
|
||||||
|
def create(base, auth, dashboard):
|
||||||
|
h = auth["h"]
|
||||||
|
|
||||||
|
# create dashboard
|
||||||
|
rd = requests.post("{base}/api/v2/dashboards".format(base=base), headers=h,
|
||||||
|
json={"orgID": auth["org_id"],
|
||||||
|
"name": dashboard["name"]})
|
||||||
|
rd.raise_for_status()
|
||||||
|
dash_id = rd.json()["id"]
|
||||||
|
|
||||||
|
for k, v in dashboard["cells"].items():
|
||||||
|
# create cells in dashboard
|
||||||
|
rc = requests.post(
|
||||||
|
"{base}/api/v2/dashboards/{dash_id}/cells".format(
|
||||||
|
base=base, dash_id=dash_id),
|
||||||
|
headers=h, json=v)
|
||||||
|
rc.raise_for_status()
|
||||||
|
cell_id = rc.json()["id"]
|
||||||
|
|
||||||
|
# create view of cell in dashboard
|
||||||
|
rv = requests.patch(
|
||||||
|
"{base}/api/v2/dashboards/{dash_id}/cells/{cell_id}/view".format(
|
||||||
|
base=base, dash_id=dash_id, cell_id=cell_id),
|
||||||
|
headers=h, json=dashboard["views"][k])
|
||||||
|
rv.raise_for_status()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
result = dict(changed=False, message="")
|
||||||
|
module = AnsibleModule(
|
||||||
|
argument_spec=dict(
|
||||||
|
base=dict(type="str", required=True),
|
||||||
|
org=dict(type="str", required=True),
|
||||||
|
auth_token=dict(type="str", required=True),
|
||||||
|
data=dict(type="json", required=True),
|
||||||
|
force=dict(type="bool", default=False),
|
||||||
|
),
|
||||||
|
supports_check_mode=True
|
||||||
|
)
|
||||||
|
dashboard = parse_dashboard_template(module.params["data"])
|
||||||
|
auth = get_auth(module.params["base"], module.params["org"],
|
||||||
|
module.params["auth_token"])
|
||||||
|
changed, x = check(module.params["base"], auth, dashboard)
|
||||||
|
|
||||||
|
result['changed'] = changed
|
||||||
|
|
||||||
|
if module.check_mode:
|
||||||
|
module.exit_json(**result)
|
||||||
|
|
||||||
|
if changed or module.params["force"]:
|
||||||
|
create(module.params["base"], auth, dashboard)
|
||||||
|
module.exit_json(**result)
|
||||||
232
roles/cobald/library/influx_token.py
Executable file
232
roles/cobald/library/influx_token.py
Executable file
@@ -0,0 +1,232 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
import requests
|
||||||
|
from ansible.module_utils.basic import AnsibleModule
|
||||||
|
|
||||||
|
DOCUMENTATION = r'''
|
||||||
|
---
|
||||||
|
module: influx2_token
|
||||||
|
short_description: generate token via influxdb2 api
|
||||||
|
description: generate token via influxdb2 api
|
||||||
|
notes:
|
||||||
|
- just works with influxdb version 2
|
||||||
|
- needs token to authenticate against API (use
|
||||||
|
`influx auth list --user my-user --hide-headers | cut -f 3`
|
||||||
|
- tokens may not be removed
|
||||||
|
- permissions can not be updated. a new token is created and the old
|
||||||
|
one is not removed.
|
||||||
|
options:
|
||||||
|
base:
|
||||||
|
description: URL for path, e.g. `https://localhost:8086`
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
org:
|
||||||
|
description: influxdb2 organisation
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
auth_token:
|
||||||
|
description: influxdb2 authentication token
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
key:
|
||||||
|
description: some key used to identify token. This is put into
|
||||||
|
the tokens description
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
|
description:
|
||||||
|
description: textual description for token. key gets appended
|
||||||
|
type: str
|
||||||
|
required: False
|
||||||
|
permissions:
|
||||||
|
description: list of permissions, each dict(action, resource)
|
||||||
|
type: list
|
||||||
|
required: True
|
||||||
|
force:
|
||||||
|
description: force creation even if dashboard already exists
|
||||||
|
(adds a new one)
|
||||||
|
type: bool
|
||||||
|
required: False
|
||||||
|
default: False
|
||||||
|
author:
|
||||||
|
- Thorsten M. (@thoto)
|
||||||
|
'''
|
||||||
|
|
||||||
|
EXAMPLES = r'''
|
||||||
|
- name: "fetch auth token"
|
||||||
|
raw: influx auth list --user my-user --hide-headers | cut -f 3
|
||||||
|
register: influx_token_fetch
|
||||||
|
delegate_to: ed-influxdb-2
|
||||||
|
|
||||||
|
- name: "create dashboard"
|
||||||
|
influx_token:
|
||||||
|
base: "http://localhost:8086"
|
||||||
|
org: "my-org"
|
||||||
|
auth_token: "{{influx_token_fetch.stdout_lines[0]}}"
|
||||||
|
key: "foo123"
|
||||||
|
description: "token for foo key"
|
||||||
|
permissions:
|
||||||
|
- action: "write"
|
||||||
|
resource:
|
||||||
|
type: "buckets"
|
||||||
|
register: ed-influx-token
|
||||||
|
|
||||||
|
- debug: msg="Token: {{ed-influx-token.token}}"
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def get_org_id(base, org_name, h):
|
||||||
|
ro = requests.get("{base}/api/v2/orgs".format(base=base), headers=h,
|
||||||
|
json={"org": org_name})
|
||||||
|
ro.raise_for_status()
|
||||||
|
org_id = [o["id"] for o in ro.json()["orgs"] if o["name"] == org_name]
|
||||||
|
return org_id[0]
|
||||||
|
|
||||||
|
|
||||||
|
def marker(key):
|
||||||
|
return "__ANSIBLE_TOKEN_{key}__".format(key=key)
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_perms(perms):
|
||||||
|
for r in perms:
|
||||||
|
r["resource"] = {k: v for k, v in r["resource"].items() if v}
|
||||||
|
return perms
|
||||||
|
|
||||||
|
|
||||||
|
class Token:
|
||||||
|
def __init__(self, base, h, data):
|
||||||
|
self.base = base
|
||||||
|
self.h = h
|
||||||
|
self.marker = marker(data["key"])
|
||||||
|
|
||||||
|
self.org_id = data["org_id"]
|
||||||
|
self.perms = _filter_perms(data["perms"])
|
||||||
|
self.description = data["description"]+" "+self.marker
|
||||||
|
|
||||||
|
self.result_token = None
|
||||||
|
self.f = None
|
||||||
|
|
||||||
|
def check(self):
|
||||||
|
ra = requests.get("{base}/api/v2/authorizations".format(
|
||||||
|
base=self.base),
|
||||||
|
params={"orgID": self.org_id},
|
||||||
|
headers=self.h)
|
||||||
|
ra.raise_for_status()
|
||||||
|
|
||||||
|
update = None
|
||||||
|
for i in ra.json()["authorizations"]:
|
||||||
|
if self.marker not in i["description"] \
|
||||||
|
or i["orgID"] != self.org_id:
|
||||||
|
continue
|
||||||
|
if self._match_perms(self.perms, i["permissions"]):
|
||||||
|
self.result_token = i
|
||||||
|
if self.description == i["description"]:
|
||||||
|
return False # everything matches -> no change needed
|
||||||
|
else:
|
||||||
|
update = {"auth_id": i["id"],
|
||||||
|
"description": self.description}
|
||||||
|
# TODO: may remove token because permissions do not match?
|
||||||
|
if update:
|
||||||
|
self.f = lambda: self._update(**update)
|
||||||
|
else:
|
||||||
|
self.result_token = None
|
||||||
|
self.f = lambda: self._create({
|
||||||
|
"orgID": self.org_id,
|
||||||
|
"description": self.description,
|
||||||
|
"permissions": self.perms
|
||||||
|
})
|
||||||
|
return True
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
if not self.f:
|
||||||
|
self.check()
|
||||||
|
self.f()
|
||||||
|
|
||||||
|
def _match_perms(self, pa, pb):
|
||||||
|
def g(match, lst):
|
||||||
|
for idx, i in enumerate(lst):
|
||||||
|
if i['action'] != match['action']:
|
||||||
|
continue
|
||||||
|
for k, v in match['resource'].items():
|
||||||
|
if k not in i['resource'] or i['resource'][k] != v:
|
||||||
|
continue
|
||||||
|
else: # first best match
|
||||||
|
return idx
|
||||||
|
else:
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
|
b = [b.copy() for b in pb]
|
||||||
|
for i in pa:
|
||||||
|
try:
|
||||||
|
b.pop(g(i, b))
|
||||||
|
except ValueError:
|
||||||
|
return False # permission i not present in b
|
||||||
|
|
||||||
|
if b: # not empty
|
||||||
|
return False # some permissions in b not in a
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _update(self, auth_id, description):
|
||||||
|
ra = requests.patch(
|
||||||
|
"{base}/api/v2/authorizations/{auth_id}".format(
|
||||||
|
base=self.base, auth_id=auth_id),
|
||||||
|
headers=self.h, json={"description": description})
|
||||||
|
ra.raise_for_status()
|
||||||
|
return ra
|
||||||
|
|
||||||
|
def _create(self, data):
|
||||||
|
ra = requests.post("{base}/api/v2/authorizations".format(
|
||||||
|
base=self.base),
|
||||||
|
headers=self.h, json=data)
|
||||||
|
ra.raise_for_status()
|
||||||
|
self.result_token = ra.json()
|
||||||
|
return ra
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
result = dict(changed=False, message="")
|
||||||
|
module = AnsibleModule(
|
||||||
|
argument_spec=dict(
|
||||||
|
base=dict(type="str", required=True),
|
||||||
|
org=dict(type="str", required=True),
|
||||||
|
auth_token=dict(type="str", required=True),
|
||||||
|
key=dict(type="str", required=True),
|
||||||
|
description=dict(type="str", default=""),
|
||||||
|
permissions=dict(type="list", elements='dict', options=dict(
|
||||||
|
action=dict(type='str', choices=['read', 'write'],
|
||||||
|
required=True),
|
||||||
|
resource=dict(type='dict', options=dict(
|
||||||
|
id=dict(type='str'),
|
||||||
|
name=dict(type='str'),
|
||||||
|
org=dict(type='str'),
|
||||||
|
orgID=dict(type='str'),
|
||||||
|
type=dict(type='str', required=True, choices=[
|
||||||
|
"authorizations", "buckets", "dashboards", "orgs",
|
||||||
|
"sources", "tasks", "telegrafs", "users", "variables",
|
||||||
|
"scrapers", "secrets", "labels", "views", "documents",
|
||||||
|
"notificationRules", "notificationEndpoints", "checks",
|
||||||
|
"dbrp", "flows", "annotations", "functions"]),
|
||||||
|
), required=True),
|
||||||
|
), required=True),
|
||||||
|
force=dict(type="bool", default=False),
|
||||||
|
),
|
||||||
|
supports_check_mode=True
|
||||||
|
)
|
||||||
|
h = {"Authorization": "Token {token}".format(
|
||||||
|
token=module.params["auth_token"])}
|
||||||
|
t = Token(module.params["base"], h, {
|
||||||
|
"org_id": get_org_id(module.params["base"], module.params["org"], h),
|
||||||
|
"key": module.params["key"],
|
||||||
|
"perms": module.params["permissions"],
|
||||||
|
"description": module.params["description"]})
|
||||||
|
|
||||||
|
changed = t.check()
|
||||||
|
if t.result_token:
|
||||||
|
result['token'] = t.result_token["token"]
|
||||||
|
result['changed'] = changed
|
||||||
|
|
||||||
|
if module.check_mode:
|
||||||
|
module.exit_json(**result)
|
||||||
|
|
||||||
|
if changed or module.params["force"]:
|
||||||
|
t.run()
|
||||||
|
result['token'] = t.result_token["token"]
|
||||||
|
module.exit_json(**result)
|
||||||
54
roles/cobald/tasks/grafana.yml
Normal file
54
roles/cobald/tasks/grafana.yml
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
- name: create influx token for grafana
|
||||||
|
influx_token:
|
||||||
|
base: "http://localhost:{{influx_pubport}}"
|
||||||
|
org: "my-org"
|
||||||
|
auth_token: "{{influx_admin_token}}"
|
||||||
|
description: grafana read access
|
||||||
|
key: grafana
|
||||||
|
permissions:
|
||||||
|
- action: read
|
||||||
|
resource:
|
||||||
|
type: buckets
|
||||||
|
register: influx_grafana_token
|
||||||
|
|
||||||
|
- name: run grafana
|
||||||
|
docker_container:
|
||||||
|
name: ed-grafana
|
||||||
|
image: grafana/grafana:7.5.7
|
||||||
|
hostname: ed-grafana
|
||||||
|
domainname: cobald.local
|
||||||
|
networks:
|
||||||
|
- name: "{{docker_network}}"
|
||||||
|
published_ports:
|
||||||
|
- "3000:3000"
|
||||||
|
state: started
|
||||||
|
detach: True
|
||||||
|
cleanup: True
|
||||||
|
|
||||||
|
- wait_for:
|
||||||
|
host: localhost
|
||||||
|
port: 3000
|
||||||
|
|
||||||
|
- community.grafana.grafana_datasource:
|
||||||
|
grafana_url: http://localhost:3000
|
||||||
|
grafana_user: admin
|
||||||
|
grafana_password: admin
|
||||||
|
name: InfluxDB
|
||||||
|
ds_type: influxdb
|
||||||
|
ds_url: "{{influx_url}}"
|
||||||
|
additional_json_data:
|
||||||
|
defaultBucket: "{{influx_bucket}}"
|
||||||
|
organization: "{{influx_org}}"
|
||||||
|
version: Flux
|
||||||
|
additional_secure_json_data:
|
||||||
|
token: "{{influx_grafana_token.token}}"
|
||||||
|
register: das
|
||||||
|
|
||||||
|
- community.grafana.grafana_dashboard:
|
||||||
|
grafana_url: http://localhost:3000
|
||||||
|
grafana_user: admin
|
||||||
|
grafana_password: admin
|
||||||
|
state: present
|
||||||
|
commit_message: updated by ansible
|
||||||
|
overwrite: yes
|
||||||
|
json_data: "{{lookup('file', 'grafana-dashboard.json')|from_json}}"
|
||||||
75
roles/cobald/tasks/influxdb.yml
Normal file
75
roles/cobald/tasks/influxdb.yml
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
- name: run influxdb in docker container
|
||||||
|
docker_container:
|
||||||
|
name: ed-influxdb
|
||||||
|
image: influxdb
|
||||||
|
hostname: "{{influx_hostname}}"
|
||||||
|
domainname: "{{influx_domainname}}"
|
||||||
|
networks:
|
||||||
|
- name: "{{ docker_network }}"
|
||||||
|
published_ports:
|
||||||
|
- "{{influx_pubport}}:8086"
|
||||||
|
volumes:
|
||||||
|
- "ed-influxdb-data:/var/lib/influxdb2"
|
||||||
|
- "ed-influxdb-config:/etc/influxdb2"
|
||||||
|
- "/container/volumes/influxdb-backup/:/backup"
|
||||||
|
env:
|
||||||
|
DOCKER_INFLUXDB_INIT_MODE: setup
|
||||||
|
DOCKER_INFLUXDB_INIT_USERNAME: "{{influx_admin_user}}"
|
||||||
|
DOCKER_INFLUXDB_INIT_PASSWORD: "{{influx_admin_pw}}"
|
||||||
|
DOCKER_INFLUXDB_INIT_ORG: "{{influx_org}}"
|
||||||
|
DOCKER_INFLUXDB_INIT_BUCKET: my-bucket
|
||||||
|
state: started
|
||||||
|
detach: True
|
||||||
|
cleanup: True
|
||||||
|
networks_cli_compatible: True
|
||||||
|
|
||||||
|
- add_host:
|
||||||
|
name: ed-influxdb
|
||||||
|
ansible_connection: docker
|
||||||
|
# ansible_docker_extra_args: "-H=ssh://ed-c7-1.virt.magni.thoto.net" # FIXME
|
||||||
|
ansible_docker_extra_args: "-H=ssh://{{ansible_host}}" # FIXME
|
||||||
|
changed_when: False
|
||||||
|
|
||||||
|
- name: wait for influx to run
|
||||||
|
raw: until curl http://localhost:8086 ; do sleep 1 ; done
|
||||||
|
changed_when: False
|
||||||
|
delegate_to: ed-influxdb
|
||||||
|
|
||||||
|
- name: fetch auth token
|
||||||
|
raw: influx auth list --user my-user --hide-headers --json
|
||||||
|
register: influx_token_fetch
|
||||||
|
changed_when: False
|
||||||
|
delegate_to: ed-influxdb
|
||||||
|
|
||||||
|
- name: set admin token
|
||||||
|
set_fact:
|
||||||
|
influx_admin_token:
|
||||||
|
"{{(influx_token_fetch.stdout | from_json | first).token}}"
|
||||||
|
|
||||||
|
- name: create bucket for cobald
|
||||||
|
influx_bucket:
|
||||||
|
base: "http://localhost:{{influx_pubport}}"
|
||||||
|
org: "my-org"
|
||||||
|
auth_token: "{{influx_admin_token}}"
|
||||||
|
name: "{{influx_bucket}}"
|
||||||
|
|
||||||
|
- name: create dashboard
|
||||||
|
influx_dashboard:
|
||||||
|
base: "http://localhost:{{influx_pubport}}"
|
||||||
|
org: "my-org"
|
||||||
|
auth_token: "{{influx_admin_token}}"
|
||||||
|
data: "{{lookup('file', 'influxdb-dashboard-cobald.json')}}"
|
||||||
|
|
||||||
|
- name: create token for telegraf
|
||||||
|
influx_token:
|
||||||
|
base: "http://localhost:{{influx_pubport}}"
|
||||||
|
org: "my-org"
|
||||||
|
auth_token: "{{influx_admin_token}}"
|
||||||
|
description: cobald tardis telegraf monitoring plugin
|
||||||
|
key: telegraf_cobaldtardis
|
||||||
|
permissions:
|
||||||
|
- action: write
|
||||||
|
resource:
|
||||||
|
type: buckets
|
||||||
|
name: "{{influx_bucket}}"
|
||||||
|
register: influx_telegraf_token
|
||||||
143
roles/cobald/tasks/main.yml
Normal file
143
roles/cobald/tasks/main.yml
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
- include_vars: cobald-slurm.yml
|
||||||
|
when: cobald_slurm | default(False)
|
||||||
|
|
||||||
|
- file:
|
||||||
|
path: "/container/{{item.name}}/cobald{{item.pfx|default('')}}/"
|
||||||
|
state: directory
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
loop:
|
||||||
|
- name: docker-images
|
||||||
|
pfx: ".{{cobald_image_tag|default('latest')}}"
|
||||||
|
- name: volumes
|
||||||
|
|
||||||
|
- template:
|
||||||
|
src: cobald.Dockerfile
|
||||||
|
dest: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/Dockerfile"
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
register: cobald_cp_dockerfile
|
||||||
|
|
||||||
|
- copy:
|
||||||
|
src: "{{item}}"
|
||||||
|
dest: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/{{item}}"
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
mode: 0755
|
||||||
|
with_items:
|
||||||
|
- cobald-entrypoint.sh
|
||||||
|
- init-cobaldmodules.sh
|
||||||
|
register: cobald_cp_files
|
||||||
|
|
||||||
|
- docker_image:
|
||||||
|
name: "cobald"
|
||||||
|
tag: "{{cobald_image_tag|default('latest')}}"
|
||||||
|
# pull: False
|
||||||
|
build:
|
||||||
|
pull: False
|
||||||
|
path: "/container/docker-images/cobald.{{cobald_image_tag|default('latest')}}/"
|
||||||
|
source: build
|
||||||
|
force_source: "{{cobald_cp_dockerfile.changed or cobald_cp_files.changed}}"
|
||||||
|
|
||||||
|
- copy:
|
||||||
|
src: cobald-config/
|
||||||
|
dest: /container/volumes/cobald
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
when: False
|
||||||
|
|
||||||
|
- docker_network:
|
||||||
|
name: "{{cobald_docker_network}}" # FIXME
|
||||||
|
state: present
|
||||||
|
|
||||||
|
# docker run -v $(pwd)/cobald-config-host:/etc/cobald -v $(pwd)/cobald:/cobald --rm -it cobald bash
|
||||||
|
|
||||||
|
- yum:
|
||||||
|
name: git
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- file:
|
||||||
|
path: "{{item}}"
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: "{{unpriv_user}}"
|
||||||
|
mode: "a=rx,u=rwx"
|
||||||
|
state: directory
|
||||||
|
with_items:
|
||||||
|
- "~{{unpriv_user}}/cobald/modules"
|
||||||
|
- "~{{unpriv_user}}/cobald"
|
||||||
|
|
||||||
|
- git:
|
||||||
|
repo: https://github.com/thoto/cobald
|
||||||
|
dest: "~{{unpriv_user}}/cobald-src"
|
||||||
|
version: bugfix/mixed_construction_methods
|
||||||
|
update: no
|
||||||
|
become: yes
|
||||||
|
become_user: "{{unpriv_user}}"
|
||||||
|
register: cobald_git_pull
|
||||||
|
|
||||||
|
- git:
|
||||||
|
repo: https://github.com/MatterMiners/tardis
|
||||||
|
dest: "~{{unpriv_user}}/tardis-src"
|
||||||
|
version: master
|
||||||
|
become: yes
|
||||||
|
become_user: "{{unpriv_user}}"
|
||||||
|
register: tardis_git_pull
|
||||||
|
|
||||||
|
- name: "get unpriv_user {{unpriv_user}} uid and gid"
|
||||||
|
getent:
|
||||||
|
database: passwd
|
||||||
|
key: "{{unpriv_user}}"
|
||||||
|
|
||||||
|
- name: run pip install
|
||||||
|
docker_container:
|
||||||
|
image: "cobald:{{cobald_image_tag|default('latest')}}"
|
||||||
|
name: "cobald-src-{{item.name}}-install"
|
||||||
|
volumes:
|
||||||
|
- "~{{unpriv_user}}/{{item.name}}-src:/usr/local/src/{{item.name}}:rw"
|
||||||
|
state: started
|
||||||
|
detach: False
|
||||||
|
cleanup: True
|
||||||
|
user: "{{getent_passwd[unpriv_user][1]}}:{{getent_passwd[unpriv_user][2]}}"
|
||||||
|
entrypoint: ""
|
||||||
|
command: |
|
||||||
|
bash -c 'HOME=/tmp pip3 install --editable /usr/local/src/{{item.name}}'
|
||||||
|
with_items:
|
||||||
|
- name: cobald
|
||||||
|
run: "{{cobald_git_pull.changed}}"
|
||||||
|
- name: tardis
|
||||||
|
run: "{{tardis_git_pull.changed}}"
|
||||||
|
when: item.run
|
||||||
|
|
||||||
|
- import_tasks: telegraf.yml
|
||||||
|
|
||||||
|
- name: get cobald hostname
|
||||||
|
include_role:
|
||||||
|
name: cobald_facts
|
||||||
|
when: cobald_container_hostname is not defined
|
||||||
|
|
||||||
|
- name: run cobald container
|
||||||
|
docker_container:
|
||||||
|
name: "{{ container_name | default('cobald') }}"
|
||||||
|
image: "cobald:{{cobald_image_tag|default('latest')}}"
|
||||||
|
hostname: "{{cobald_container_hostname}}"
|
||||||
|
domainname: "{{ cobald_domainname | default('cobald.local')}}"
|
||||||
|
volumes: "{{default_mounts + cobald_mounts }}"
|
||||||
|
networks:
|
||||||
|
- name: "{{cobald_docker_network}}"
|
||||||
|
networks_cli_compatible: True
|
||||||
|
# env:
|
||||||
|
# slurmuser: "{{slurm_user}}"
|
||||||
|
# privileged: "{{ container_privileged | bool }}"
|
||||||
|
state: started
|
||||||
|
detach: True
|
||||||
|
cleanup: True
|
||||||
|
interactive: True
|
||||||
|
# command: python3 -m cobald.daemon /etc/cobald/config.yaml
|
||||||
|
vars:
|
||||||
|
default_mounts: "{{cobald_slurm_mounts | default([])}}"
|
||||||
|
cobald_mounts:
|
||||||
|
- "~{{unpriv_user}}/cobald:/etc/cobald"
|
||||||
|
# - /container/volumes/cobald:/etc/cobald:ro
|
||||||
|
- "~{{unpriv_user}}/cobald/modules:/usr/local/src/cobaldmodules"
|
||||||
|
- "~{{unpriv_user}}/cobald-src:/usr/local/src/cobald:ro"
|
||||||
|
- "~{{unpriv_user}}/tardis-src:/usr/local/src/tardis:ro"
|
||||||
73
roles/cobald/tasks/telegraf.yml
Normal file
73
roles/cobald/tasks/telegraf.yml
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
- file:
|
||||||
|
path: "/container/{{item}}/telegraf/"
|
||||||
|
state: directory
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
loop:
|
||||||
|
- docker-images
|
||||||
|
- volumes
|
||||||
|
|
||||||
|
- copy:
|
||||||
|
src: telegraf.Dockerfile
|
||||||
|
dest: /container/docker-images/telegraf/Dockerfile
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
register: cobald_cp_telegraf_dockerfile
|
||||||
|
|
||||||
|
- copy: # telegraf is found in influxdb repo
|
||||||
|
src: influxdb.repo
|
||||||
|
dest: /container/docker-images/telegraf/influxdb.repo
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
|
||||||
|
- name: docker image for telegraf
|
||||||
|
docker_image:
|
||||||
|
name: "ed-telegraf"
|
||||||
|
build:
|
||||||
|
pull: False
|
||||||
|
path: "/container/docker-images/telegraf/"
|
||||||
|
source: build
|
||||||
|
force_source: "{{cobald_cp_telegraf_dockerfile.changed}}"
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
influx_hostname: "ed-influxdb"
|
||||||
|
influx_domainname: "cobald.local"
|
||||||
|
influx_bucket: batleth
|
||||||
|
tags: influxdb
|
||||||
|
|
||||||
|
- import_tasks: influxdb.yml
|
||||||
|
tags: influxdb
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
# influx_url: "http://{{influx_hostname}}.{{influx_domainname}}:8086"
|
||||||
|
influx_url: "http://{{influx_hostname}}:8086"
|
||||||
|
tags: influxdb
|
||||||
|
|
||||||
|
- name: generate telegraf config
|
||||||
|
template:
|
||||||
|
src: telegraf.conf.j2
|
||||||
|
dest: /container/volumes/telegraf/telegraf.conf
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
vars:
|
||||||
|
influx_token: "{{influx_telegraf_token.token}}"
|
||||||
|
register: telegraf_config_gen
|
||||||
|
|
||||||
|
- name: run telegraf container
|
||||||
|
docker_container:
|
||||||
|
name: ed-telegraf
|
||||||
|
image: ed-telegraf
|
||||||
|
hostname: telegraf
|
||||||
|
domainname: cobald.local
|
||||||
|
networks:
|
||||||
|
- name: "{{docker_network | default('bridge') }}"
|
||||||
|
volumes:
|
||||||
|
- "/container/volumes/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro"
|
||||||
|
state: started
|
||||||
|
recreate: "{{ telegraf_config_gen.changed | default(False) | bool }}"
|
||||||
|
detach: True
|
||||||
|
# cleanup: True
|
||||||
|
networks_cli_compatible: True
|
||||||
|
|
||||||
|
- import_tasks: grafana.yml
|
||||||
|
tags: influxdb
|
||||||
68
roles/cobald/templates/cobald.Dockerfile
Normal file
68
roles/cobald/templates/cobald.Dockerfile
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
FROM {{ cobald_docker_base_image | default("docker.io/library/centos:7") }}
|
||||||
|
|
||||||
|
RUN yum update -y && \
|
||||||
|
yum install -y python3 git && pip3 install --upgrade pip && \
|
||||||
|
yum clean all && rm -rf /var/cache/yum
|
||||||
|
|
||||||
|
ARG REPOCOBALD=https://github.com/MatterMiners/cobald
|
||||||
|
ARG REPOTARDIS=https://github.com/MatterMiners/tardis
|
||||||
|
|
||||||
|
RUN git clone $REPOCOBALD /usr/local/src/cobald && \
|
||||||
|
git clone $REPOTARDIS /usr/local/src/tardis
|
||||||
|
|
||||||
|
RUN mkdir /etc/cobald /var/log/cobald && \
|
||||||
|
useradd -m -d /var/lib/cobald --no-log-init --system cobald && \
|
||||||
|
chown cobald:cobald /var/log/cobald
|
||||||
|
|
||||||
|
#RUN mkdir /cobald && python3 -m venv /cobald && source /cobald/bin/activate &&\
|
||||||
|
# pip3 install --upgrade pip && pip3 install cobald
|
||||||
|
|
||||||
|
RUN mkdir /usr/local/src/cobaldmodules /usr/local/lib/cobaldmodules && \
|
||||||
|
ln -s /usr/local/src/cobaldmodules/setup.py \
|
||||||
|
/usr/local/lib/cobaldmodules/setup.py && \
|
||||||
|
ln -s /usr/local/src/cobaldmodules/cobaldmodules \
|
||||||
|
/usr/local/lib/cobaldmodules/cobaldmodules && \
|
||||||
|
chown cobald:cobald /usr/local/lib/cobaldmodules
|
||||||
|
|
||||||
|
RUN pip3 install --editable /usr/local/src/cobald && \
|
||||||
|
pip3 install --editable /usr/local/src/cobald[contrib]
|
||||||
|
|
||||||
|
RUN pip3 install --editable /usr/local/src/tardis&& \
|
||||||
|
pip3 install --editable /usr/local/src/tardis[contrib]
|
||||||
|
|
||||||
|
ENV PYTHONPATH=/usr/local/src/cobaldmodules
|
||||||
|
|
||||||
|
# pip3 install --editable .
|
||||||
|
# pip3 install --editable .[contrib]
|
||||||
|
# pip3 install --upgrade --editable /etc/cobald/modules/
|
||||||
|
# su cobald -c "python3 -m cobald.daemon /etc/cobald/config.yaml"
|
||||||
|
|
||||||
|
VOLUME /usr/local/src/cobaldmodules
|
||||||
|
|
||||||
|
VOLUME /etc/cobald
|
||||||
|
|
||||||
|
RUN mkdir -p /usr/local/lib/entrypoints.d/
|
||||||
|
|
||||||
|
COPY init-cobaldmodules.sh /usr/local/lib/entrypoints.d/50-init-cobaldmodules.sh
|
||||||
|
|
||||||
|
RUN chmod 755 /usr/local/lib/entrypoints.d/50-init-cobaldmodules.sh
|
||||||
|
|
||||||
|
RUN echo -e "#!/bin/sh\npython3 -m cobald.daemon /etc/cobald/config.yaml" >> /etc/docker-init.d/70-cobald && chmod 755 /etc/docker-init.d/70-cobald
|
||||||
|
|
||||||
|
{% if cobald_docker_default_command | default(True) -%}
|
||||||
|
COPY cobald-entrypoint.sh /usr/local/sbin/cobald-entrypoint.sh
|
||||||
|
|
||||||
|
RUN chmod 755 /usr/local/sbin/cobald-entrypoint.sh
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/usr/local/sbin/cobald-entrypoint.sh" ]
|
||||||
|
|
||||||
|
RUN yum -y install iproute &&\
|
||||||
|
yum clean all && rm -rf /var/cache/yum
|
||||||
|
|
||||||
|
USER cobald
|
||||||
|
|
||||||
|
STOPSIGNAL SIGINT
|
||||||
|
|
||||||
|
# CMD "python3 -m cobald.daemon /etc/cobald/config.yaml"
|
||||||
|
CMD /etc/docker-init.d/60-cobald
|
||||||
|
{%- endif %}
|
||||||
1829
roles/cobald/templates/telegraf.conf.j2
Normal file
1829
roles/cobald/templates/telegraf.conf.j2
Normal file
File diff suppressed because it is too large
Load Diff
12
roles/cobald/vars/cobald-slurm.yml
Normal file
12
roles/cobald/vars/cobald-slurm.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
cobald_image_tag: slurm
|
||||||
|
cobald_docker_base_image: "{{slurm.base_image}}"
|
||||||
|
cobald_docker_default_command: False
|
||||||
|
cobald_docker_network: "{{slurm.network}}"
|
||||||
|
cobald_domainname: "{{slurm.domain}}"
|
||||||
|
cobald_slurm_mounts: "{{slurm.mounts}}"
|
||||||
|
#- /container/volumes/slurm/:/etc/slurm/:rw
|
||||||
|
##- "{{slurm_cfg_path | mandatory}}:/etc/slurm/:rw"
|
||||||
|
#- /container/volumes/munge/munge.key:/etc/munge/munge.key:rw
|
||||||
|
## - "{{slurm_munge_path | mandatory}}:/etc/munge/munge.key:rw"
|
||||||
|
#- slurm-shared:/shared/:rw
|
||||||
|
## - "{{slurm_shared_path | mandatory}}:{{slurm_shared_target | default('/shared')}}:rw"
|
||||||
1
roles/cobald/vars/main.yml
Normal file
1
roles/cobald/vars/main.yml
Normal file
@@ -0,0 +1 @@
|
|||||||
|
cobald_docker_network: "{{docker_network}}"
|
||||||
11
roles/cobald_facts/tasks/main.yml
Normal file
11
roles/cobald_facts/tasks/main.yml
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
- block:
|
||||||
|
- docker_container_info:
|
||||||
|
name: "{{ container_name | mandatory }}"
|
||||||
|
register: cobald_container_info
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
cobald_container_hostname: |-
|
||||||
|
{{cobald_container_info.container.Config.Hostname | default('cobald-'+
|
||||||
|
lookup('password', '/dev/null chars=ascii_lowercase length=6')) }}
|
||||||
|
when: cobald_container_hostname is not defined
|
||||||
|
|
||||||
@@ -3,3 +3,6 @@ slurm_user: slurm
|
|||||||
slurm_log_path_ctld: /var/log/slurm/slurmctld.log
|
slurm_log_path_ctld: /var/log/slurm/slurmctld.log
|
||||||
slurm_log_path_d: /var/log/slurm/slurmd.log
|
slurm_log_path_d: /var/log/slurm/slurmd.log
|
||||||
slurm_log_path_sched: /var/log/slurm/slurmsched.log
|
slurm_log_path_sched: /var/log/slurm/slurmsched.log
|
||||||
|
slurm_prefix: slurm
|
||||||
|
slurm_domain: slurm.local
|
||||||
|
docker_network: slurm
|
||||||
|
|||||||
18
roles/slurm/files/docker-init
Normal file
18
roles/slurm/files/docker-init
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
function trp_term(){
|
||||||
|
echo pkill -P $pids
|
||||||
|
for j in $pids ; do
|
||||||
|
pkill -P $j
|
||||||
|
kill -SIGTERM $j
|
||||||
|
done
|
||||||
|
}
|
||||||
|
trap trp_term SIGINT SIGTERM
|
||||||
|
pids=""
|
||||||
|
for i in /etc/docker-init.d/* ; do
|
||||||
|
[ ! -f $i ] && break
|
||||||
|
$i &
|
||||||
|
pids="$pids $!"
|
||||||
|
done
|
||||||
|
wait $pids
|
||||||
|
|
||||||
|
# TODO: call start scripts like "foo.sh start" and "foo.sh stop" to avoid pkill
|
||||||
@@ -1,32 +1,11 @@
|
|||||||
FROM docker.io/library/centos:7 as base
|
FROM slurm:base
|
||||||
|
|
||||||
RUN yum install -y epel-release && \
|
|
||||||
yum install -y slurm && \
|
|
||||||
yum clean all && rm -rf /var/cache/yum
|
|
||||||
|
|
||||||
RUN yum install -y less iproute bind-utils nmap-ncat net-tools && \
|
|
||||||
yum clean all && rm -rf /var/cache/yum
|
|
||||||
|
|
||||||
COPY entrypoint.sh /usr/local/sbin/entrypoint.sh
|
|
||||||
|
|
||||||
RUN chown root:root /usr/local/sbin/entrypoint.sh && \
|
|
||||||
chmod 755 /usr/local/sbin/entrypoint.sh
|
|
||||||
|
|
||||||
ENTRYPOINT [ "/usr/local/sbin/entrypoint.sh" ]
|
|
||||||
|
|
||||||
ARG slurmuser=slurm
|
|
||||||
ENV slurmuser=${slurmuser}
|
|
||||||
|
|
||||||
RUN useradd -d /var/lib/slurm -m --no-log-init --system $slurmuser &&\
|
|
||||||
slurm-setuser -u $slurmuser -g $slurmuser -y
|
|
||||||
|
|
||||||
RUN yum install -y slurm-slurmctld && \
|
RUN yum install -y slurm-slurmctld && \
|
||||||
yum clean all && rm -rf /var/cache/yum
|
yum clean all && rm -rf /var/cache/yum
|
||||||
|
|
||||||
|
COPY start-scripts/20-slurmctld /etc/docker-init.d/20-slurmctld
|
||||||
|
RUN chmod 755 /etc/docker-init.d/20-slurmctld
|
||||||
|
|
||||||
ENV SLURMCTLD_LOG_PATH="/var/log/slurm/slurmctld.log"
|
ENV SLURMCTLD_LOG_PATH="/var/log/slurm/slurmctld.log"
|
||||||
ENV SLURMD_LOG_PATH="/var/log/slurm/slurmd.log"
|
ENV SLURMD_LOG_PATH="/var/log/slurm/slurmd.log"
|
||||||
ENV SLURM_SCHED_LOG_PATH="/var/log/slurm/slurmsched.log"
|
ENV SLURM_SCHED_LOG_PATH="/var/log/slurm/slurmsched.log"
|
||||||
|
|
||||||
CMD bash -c 'cat <({ su -s /bin/sh -c "munged -F" munge & \
|
|
||||||
su -s /bin/sh -c "slurmctld -D" ${slurmuser} 2>/dev/null 1>/dev/null & \
|
|
||||||
tail --retry --pid $! -f ${SLURMCTLD_LOG_PATH} ${SLURM_SCHED_LOG_PATH} & })'
|
|
||||||
|
|||||||
@@ -1,32 +1,11 @@
|
|||||||
FROM docker.io/library/centos:7
|
FROM slurm:base
|
||||||
|
|
||||||
RUN yum install -y epel-release && \
|
|
||||||
yum install -y slurm && \
|
|
||||||
yum clean all && rm -rf /var/cache/yum
|
|
||||||
|
|
||||||
RUN yum install -y less iproute bind-utils nmap-ncat net-tools && \
|
|
||||||
yum clean all && rm -rf /var/cache/yum
|
|
||||||
|
|
||||||
COPY entrypoint.sh /usr/local/sbin/entrypoint.sh
|
|
||||||
|
|
||||||
RUN chown root:root /usr/local/sbin/entrypoint.sh && \
|
|
||||||
chmod 755 /usr/local/sbin/entrypoint.sh
|
|
||||||
|
|
||||||
ENTRYPOINT [ "/usr/local/sbin/entrypoint.sh" ]
|
|
||||||
|
|
||||||
ARG slurmuser=slurm
|
|
||||||
ENV slurmuser=${slurmuser}
|
|
||||||
|
|
||||||
RUN useradd -d /var/lib/slurm -m --no-log-init --system $slurmuser &&\
|
|
||||||
slurm-setuser -u $slurmuser -g $slurmuser -y
|
|
||||||
|
|
||||||
RUN yum install -y slurm-slurmd && \
|
RUN yum install -y slurm-slurmd && \
|
||||||
yum clean all && rm -rf /var/cache/yum
|
yum clean all && rm -rf /var/cache/yum
|
||||||
|
|
||||||
|
COPY start-scripts/30-slurmd /etc/docker-init.d/30-slurmd
|
||||||
|
RUN chmod 755 /etc/docker-init.d/30-slurmd
|
||||||
|
|
||||||
ENV SLURMCTLD_LOG_PATH="/var/log/slurm/slurmctld.log"
|
ENV SLURMCTLD_LOG_PATH="/var/log/slurm/slurmctld.log"
|
||||||
ENV SLURMD_LOG_PATH="/var/log/slurm/slurmd.log"
|
ENV SLURMD_LOG_PATH="/var/log/slurm/slurmd.log"
|
||||||
ENV SLURM_SCHED_LOG_PATH="/var/log/slurm/slurmsched.log"
|
ENV SLURM_SCHED_LOG_PATH="/var/log/slurm/slurmsched.log"
|
||||||
|
|
||||||
CMD bash -c 'cat <({ su -s /bin/sh -c "munged -F" munge & \
|
|
||||||
slurmd -D 2>/dev/null 1>/dev/null & \
|
|
||||||
tail --retry --pid $! -f ${SLURMD_LOG_PATH} ${SLURM_SCHED_LOG_PATH} & })'
|
|
||||||
|
|||||||
4
roles/slurm/files/start-scripts/20-slurmctld
Normal file
4
roles/slurm/files/start-scripts/20-slurmctld
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
su -s /bin/sh -c "slurmctld -D" ${slurmuser} 2>/dev/null 1>/dev/null &
|
||||||
|
tail --retry --pid $! -f ${SLURMCTLD_LOG_PATH} ${SLURM_SCHED_LOG_PATH}
|
||||||
|
|
||||||
4
roles/slurm/files/start-scripts/30-slurmd
Normal file
4
roles/slurm/files/start-scripts/30-slurmd
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
slurmd -D 2>/dev/null 1>/dev/null &
|
||||||
|
tail --retry --pid $! -f ${SLURMD_LOG_PATH} ${SLURM_SCHED_LOG_PATH}
|
||||||
|
|
||||||
@@ -1,3 +1,3 @@
|
|||||||
- name: reconfigure slurm
|
- name: reconfigure slurm
|
||||||
command:
|
shell:
|
||||||
cmd: docker container exec -it slurm-ctl scontrol reconfigure
|
cmd: "docker container exec -it {{slurm_prefix}}-ctl scontrol reconfigure || docker container restart {{slurm_prefix}}-ctl && docker container exec -it {{slurm_prefix}}-ctl scontrol reconfigure"
|
||||||
|
|||||||
@@ -1,31 +1,50 @@
|
|||||||
- file:
|
- file:
|
||||||
path: "/container/docker-images/{{item}}"
|
path: "/container/docker-images/slurm"
|
||||||
state: directory
|
state: directory
|
||||||
owner: "{{unpriv_user}}"
|
owner: "{{unpriv_user}}"
|
||||||
group: docker
|
group: docker
|
||||||
|
|
||||||
- copy:
|
- copy: # FIXME: template
|
||||||
src: "{{item}}.Dockerfile"
|
src: "{{image.name}}.Dockerfile"
|
||||||
dest: "/container/docker-images/{{item}}/Dockerfile"
|
dest: "/container/docker-images/slurm/{{image.name}}.Dockerfile"
|
||||||
owner: "{{unpriv_user}}"
|
owner: "{{unpriv_user}}"
|
||||||
group: docker
|
group: docker
|
||||||
register: slurm_cp_dockerfile
|
register: slurm_cp_dockerfile
|
||||||
|
|
||||||
- copy:
|
- name: copy entrypoint and docker-init
|
||||||
src: "entrypoint.sh"
|
copy: # FIXME: swap out
|
||||||
dest: "/container/docker-images/{{item}}/entrypoint.sh"
|
src: "{{item}}"
|
||||||
|
dest: "/container/docker-images/slurm/{{item}}"
|
||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: u=rwx,g=rx,o=rx
|
mode: u=rwx,g=rx,o=rx
|
||||||
|
loop:
|
||||||
|
- entrypoint.sh
|
||||||
|
- docker-init
|
||||||
register: slurm_cp_entrypt
|
register: slurm_cp_entrypt
|
||||||
|
|
||||||
|
- name: copy startup scripts
|
||||||
|
copy:
|
||||||
|
src: "start-scripts/"
|
||||||
|
dest: "/container/docker-images/slurm/start-scripts/"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: u=rwx,g=rx,o=rx
|
||||||
|
register: slurm_cp_stscrs
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
slurm_image_files_changed: "{{ (slurm_image_files_changed | default(False))
|
||||||
|
or slurm_cp_entrypt.changed or slurm_cp_stscrs.changed }}"
|
||||||
|
|
||||||
- docker_image:
|
- docker_image:
|
||||||
name: "slurm-{{item}}"
|
name: "slurm"
|
||||||
|
tag: "{{image.name}}"
|
||||||
# pull: False
|
# pull: False
|
||||||
build:
|
build:
|
||||||
pull: False
|
pull: False
|
||||||
path: "/container/docker-images/{{item}}"
|
path: "/container/docker-images/slurm/"
|
||||||
# target: "{{item}}" # unsupported on old docker-py versions as in el7
|
dockerfile: "{{image.name}}.Dockerfile"
|
||||||
|
# target: "{{image.name}}" # unsupported on old docker-py version as in el7
|
||||||
source: build
|
source: build
|
||||||
force_source: "{{slurm_cp_dockerfile.changed or slurm_cp_entrypt.changed}}"
|
force_source: "{{slurm_cp_dockerfile.changed or slurm_image_files_changed}}"
|
||||||
|
|
||||||
|
|||||||
@@ -3,10 +3,22 @@
|
|||||||
name: [ slurm, slurm-doc ]
|
name: [ slurm, slurm-doc ]
|
||||||
state: present
|
state: present
|
||||||
|
|
||||||
- include_tasks: dockerimage.yml
|
- include_role:
|
||||||
loop:
|
name: slurm_dockerimage
|
||||||
- slurmctld
|
loop: # FIXME: default(omit)!
|
||||||
- slurmd
|
- name: slurmctld
|
||||||
|
dockerfile: "{{ lookup('file', 'slurmctld.Dockerfile') }}"
|
||||||
|
files:
|
||||||
|
- dest: start-scripts/20-slurmctld
|
||||||
|
content: "{{ lookup('file', 'start-scripts/20-slurmctld') }}"
|
||||||
|
- name: slurmd
|
||||||
|
dockerfile: "{{ lookup('file', 'slurmd.Dockerfile') }}"
|
||||||
|
files:
|
||||||
|
- dest: start-scripts/30-slurmd
|
||||||
|
content: "{{ lookup('file', 'start-scripts/30-slurmd') }}"
|
||||||
|
loop_control:
|
||||||
|
loop_var: image
|
||||||
|
label: "{{ image.name }}"
|
||||||
|
|
||||||
- name: generate munge key
|
- name: generate munge key
|
||||||
shell:
|
shell:
|
||||||
@@ -47,12 +59,14 @@
|
|||||||
loop:
|
loop:
|
||||||
- slurm.conf
|
- slurm.conf
|
||||||
- cgroup.conf
|
- cgroup.conf
|
||||||
|
vars:
|
||||||
|
alloc_nodes: "{{ [ slurm_prefix+'-submit1' ] + extra_nodes | default([])}}"
|
||||||
notify: reconfigure slurm
|
notify: reconfigure slurm
|
||||||
tags: [ slurm-config ]
|
tags: [ slurm-config ]
|
||||||
|
|
||||||
- name: "create docker network to make service discovery work"
|
- name: "create docker network to make service discovery work"
|
||||||
docker_network:
|
docker_network:
|
||||||
name: slurm
|
name: "{{ docker_network }}"
|
||||||
state: present
|
state: present
|
||||||
|
|
||||||
- name: "create docker volume for shared access between nodes"
|
- name: "create docker volume for shared access between nodes"
|
||||||
@@ -61,45 +75,62 @@
|
|||||||
state: present
|
state: present
|
||||||
|
|
||||||
- set_fact:
|
- set_fact:
|
||||||
slurm_nodes: # default nodes: controller and submit machine
|
slurm_nodes_std: # default nodes: controller and submit machine
|
||||||
- machine: ctl
|
- machine: ctl
|
||||||
image: slurm-slurmctld
|
image: slurm:slurmctld
|
||||||
|
exposed_ports: [ "6817:6817/tcp" ]
|
||||||
- machine: submit1
|
- machine: submit1
|
||||||
image: slurm-slurmd
|
image: slurm:slurmd
|
||||||
extra_mounts:
|
extra_mounts:
|
||||||
- "/home/{{unpriv_user}}/job3/:/mnt/:rw"
|
- "/home/{{unpriv_user}}/job3/:/mnt/:rw"
|
||||||
|
slurm_nodes_exec: | # extend range to execute nodes list
|
||||||
|
{% set slurm_nodes_exec = slurm_nodes_exec | default([]) %}
|
||||||
|
{% for i in range(1, num_nodes+1) -%}
|
||||||
|
{% set _ = slurm_nodes_exec.extend([
|
||||||
|
{'machine':'exec%s'|format(i), 'image': 'slurm:slurmd'}]) -%}
|
||||||
|
{%- endfor %}
|
||||||
|
{{ slurm_nodes_exec }}
|
||||||
|
slurm_default_mounts:
|
||||||
|
- /container/volumes/slurm/:/etc/slurm/:rw
|
||||||
|
- /container/volumes/munge/munge.key:/etc/munge/munge.key:rw
|
||||||
|
- slurm-shared:/shared/:rw
|
||||||
|
slurm_network: "{{docker_network}}"
|
||||||
tags: [ slurm-config ]
|
tags: [ slurm-config ]
|
||||||
|
|
||||||
# TODO: reserve some address using docker_network_info and assign as aux
|
# TODO: reserve some address using docker_network_info and assign as aux
|
||||||
# address to enable slurmctld to get a static address in order to be
|
# address to enable slurmctld to get a static address in order to be
|
||||||
# reachable from slurm running on docker host to enable submitting jobs.
|
# reachable from slurm running on docker host to enable submitting jobs.
|
||||||
|
|
||||||
- name: run slurm docker containers
|
- name: run slurm docker containers
|
||||||
docker_container:
|
docker_container:
|
||||||
name: "slurm-{{item.machine}}"
|
name: "{{ slurm_prefix }}-{{ item.machine }}"
|
||||||
hostname: "slurm-{{item.machine}}"
|
hostname: "{{ slurm_prefix }}-{{ item.machine }}"
|
||||||
domainname: "slurm.local"
|
domainname: "{{ slurm_domain }}"
|
||||||
volumes: "{{default_mounts + ( item.extra_mounts | default([]) ) }}"
|
volumes: "{{ slurm_default_mounts + ( item.extra_mounts | default([]) ) }}"
|
||||||
|
ports: "{{ item.exposed_ports | default([]) }}"
|
||||||
networks:
|
networks:
|
||||||
- name: "slurm"
|
- name: "{{ slurm_network }}"
|
||||||
env:
|
env:
|
||||||
slurmuser: "{{slurm_user}}"
|
slurmuser: "{{ slurm_user }}"
|
||||||
image: "{{item.image}}"
|
image: "{{ item.image }}"
|
||||||
state: started
|
state: started
|
||||||
detach: True
|
detach: True
|
||||||
cleanup: True
|
cleanup: True
|
||||||
privileged: "{{ container_privileged | bool }}"
|
privileged: "{{ container_privileged | bool }}"
|
||||||
networks_cli_compatible: True
|
networks_cli_compatible: True
|
||||||
|
interactive: True
|
||||||
vars:
|
vars:
|
||||||
default_mounts:
|
slurm_nodes_all: "{{ slurm_nodes_exec + slurm_nodes_std }}"
|
||||||
- /container/volumes/slurm/:/etc/slurm/:rw
|
loop: "{{ slurm_nodes_all }}"
|
||||||
- /container/volumes/munge/munge.key:/etc/munge/munge.key:rw
|
loop_control:
|
||||||
- slurm-shared:/shared/:rw
|
label: "{{slurm_prefix}}-{{ item.machine }}"
|
||||||
slurm_nodes_all: | # add execute nodes
|
|
||||||
{% for i in range(1, 4) -%}
|
|
||||||
{% set _ = slurm_nodes.extend([
|
|
||||||
{'machine':'exec%s'|format(i), 'image': 'slurm-slurmd'}]) -%}
|
|
||||||
{%- endfor %}
|
|
||||||
{{ slurm_nodes }}
|
|
||||||
loop: "{{slurm_nodes_all}}"
|
|
||||||
tags: [ slurm-config ]
|
tags: [ slurm-config ]
|
||||||
|
|
||||||
|
- name: set facts to be used by other modules
|
||||||
|
set_fact:
|
||||||
|
slurm:
|
||||||
|
user: "{{slurm_user}}"
|
||||||
|
domain: "{{slurm_domain}}"
|
||||||
|
base_image: "slurm:base"
|
||||||
|
mounts: "{{slurm_default_mounts}}"
|
||||||
|
network: "{{docker_network}}"
|
||||||
|
|||||||
@@ -164,5 +164,7 @@ SlurmSchedLogFile={{slurm_log_path_sched}}
|
|||||||
#
|
#
|
||||||
# COMPUTE NODES
|
# COMPUTE NODES
|
||||||
NodeName=slurm-exec[1-{{num_nodes}}] CPUs=2 CoresPerSocket=2 State=UNKNOWN
|
NodeName=slurm-exec[1-{{num_nodes}}] CPUs=2 CoresPerSocket=2 State=UNKNOWN
|
||||||
NodeName=slurm-submit1 CPUs=1 State=UNKNOWN
|
{% for i in alloc_nodes -%}
|
||||||
PartitionName=debug Nodes=slurm-exec[1-{{num_nodes}}] AllocNodes=slurm-submit1 Default=YES MaxTime=INFINITE State=UP
|
NodeName={{i}} State=UNKNOWN
|
||||||
|
{% endfor %}
|
||||||
|
PartitionName=debug Nodes=slurm-exec[1-{{num_nodes}}] AllocNodes={{alloc_nodes | join(',')}} Default=YES MaxTime=INFINITE State=UP
|
||||||
|
|||||||
@@ -5,5 +5,3 @@ if [ -f "/etc/munge/munge.key" ] ; then
|
|||||||
chown munge:munge /etc/munge/munge.key
|
chown munge:munge /etc/munge/munge.key
|
||||||
chmod 600 /etc/munge/munge.key
|
chmod 600 /etc/munge/munge.key
|
||||||
fi
|
fi
|
||||||
|
|
||||||
exec "$@"
|
|
||||||
8
roles/slurm_dockerimage/files/entrypoint.sh
Normal file
8
roles/slurm_dockerimage/files/entrypoint.sh
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
for i in /usr/local/lib/entrypoints.d/* ; do
|
||||||
|
[ -f $i ] && /bin/sh $i || break
|
||||||
|
done
|
||||||
|
|
||||||
|
exec "${@:-/bin/bash}"
|
||||||
31
roles/slurm_dockerimage/files/slurm-base.Dockerfile
Normal file
31
roles/slurm_dockerimage/files/slurm-base.Dockerfile
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
FROM docker.io/library/centos:7
|
||||||
|
|
||||||
|
RUN yum install -y epel-release && \
|
||||||
|
yum install -y slurm && \
|
||||||
|
yum clean all && rm -rf /var/cache/yum
|
||||||
|
|
||||||
|
RUN yum install -y less iproute bind-utils nmap-ncat net-tools && \
|
||||||
|
yum clean all && rm -rf /var/cache/yum
|
||||||
|
|
||||||
|
RUN mkdir -p /usr/local/lib/entrypoints.d/
|
||||||
|
|
||||||
|
COPY --chown=root:root entry-munge.sh /usr/local/lib/entrypoints.d/10-munge.sh
|
||||||
|
COPY --chown=root:root entrypoint.sh /usr/local/sbin/entrypoint.sh
|
||||||
|
|
||||||
|
RUN chmod 755 /usr/local/lib/entrypoints.d/10-munge.sh && \
|
||||||
|
chmod 755 /usr/local/sbin/entrypoint.sh
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/usr/local/sbin/entrypoint.sh" ]
|
||||||
|
|
||||||
|
ARG slurmuser=slurm
|
||||||
|
ENV slurmuser=${slurmuser}
|
||||||
|
|
||||||
|
RUN useradd -d /var/lib/slurm -m --no-log-init --system $slurmuser &&\
|
||||||
|
slurm-setuser -u $slurmuser -g $slurmuser -y
|
||||||
|
|
||||||
|
COPY docker-init /usr/local/sbin/docker-init
|
||||||
|
RUN mkdir /etc/docker-init.d && chmod 755 /usr/local/sbin/docker-init
|
||||||
|
COPY start-scripts/10-munge /etc/docker-init.d/10-munge
|
||||||
|
RUN chmod 755 /etc/docker-init.d/10-munge
|
||||||
|
|
||||||
|
CMD /usr/local/sbin/docker-init
|
||||||
2
roles/slurm_dockerimage/files/start-scripts/10-munge
Normal file
2
roles/slurm_dockerimage/files/start-scripts/10-munge
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
exec su -s /bin/sh -c "munged -F" munge
|
||||||
40
roles/slurm_dockerimage/tasks/dockerimage.yml
Normal file
40
roles/slurm_dockerimage/tasks/dockerimage.yml
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
- name: create directories for docker image build
|
||||||
|
file:
|
||||||
|
path: "/container/docker-images/slurm-{{image.name}}/{{item}}"
|
||||||
|
state: directory
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
loop: "{{ [''] + (image.files | map(attribute='dest') | map('dirname') |
|
||||||
|
unique | select | list) }}"
|
||||||
|
|
||||||
|
- name: copy Dockerfile
|
||||||
|
copy:
|
||||||
|
src: "{{image.name}}.Dockerfile"
|
||||||
|
dest: "/container/docker-images/slurm-{{image.name}}/Dockerfile"
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
register: slurm_cp_dockerfile
|
||||||
|
|
||||||
|
- name: copy requisite files
|
||||||
|
copy:
|
||||||
|
content: "{{ item.content }}"
|
||||||
|
dest: "/container/docker-images/slurm-{{image.name}}/{{item.dest}}"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: u=rwx,g=rx,o=rx
|
||||||
|
loop: "{{ image.files | default([]) }}"
|
||||||
|
loop_control:
|
||||||
|
label: "{{ item.dest }}"
|
||||||
|
register: slurm_cp_files
|
||||||
|
|
||||||
|
- docker_image:
|
||||||
|
name: "slurm"
|
||||||
|
tag: "{{image.name}}"
|
||||||
|
# pull: False
|
||||||
|
build:
|
||||||
|
pull: False
|
||||||
|
path: "/container/docker-images/slurm-{{image.name}}/"
|
||||||
|
source: build
|
||||||
|
force_source: "{{slurm_cp_dockerfile.changed or
|
||||||
|
slurm_cp_files.changed or
|
||||||
|
slurm_baseimg_build_chg }}"
|
||||||
46
roles/slurm_dockerimage/tasks/main.yml
Normal file
46
roles/slurm_dockerimage/tasks/main.yml
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
- file:
|
||||||
|
path: "/container/docker-images/slurm-base/start-scripts"
|
||||||
|
state: directory
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
|
||||||
|
# - name: copy Dockerfile, entrypoint, docker-init and munge startup
|
||||||
|
- name: copy slurm base image requisite files
|
||||||
|
copy: # FIXME: swap out
|
||||||
|
src: "{{item.file}}"
|
||||||
|
dest: "/container/docker-images/slurm-base/{{item.file}}"
|
||||||
|
owner: "{{unpriv_user}}"
|
||||||
|
group: docker
|
||||||
|
mode: "{{ item.perms | default('u=rwx,g=rx,o=rx') }}"
|
||||||
|
loop:
|
||||||
|
- file: slurm-base.Dockerfile
|
||||||
|
perms: u=rw,g=r,o=r
|
||||||
|
- file: entrypoint.sh
|
||||||
|
- file: entry-munge.sh
|
||||||
|
- file: docker-init
|
||||||
|
- file: start-scripts/10-munge
|
||||||
|
when: not (slurm_baseimg_build_chg | default(False))
|
||||||
|
register: slurm_baseimg_copy
|
||||||
|
|
||||||
|
- name: build base image
|
||||||
|
docker_image:
|
||||||
|
name: "slurm"
|
||||||
|
tag: "base"
|
||||||
|
# pull: False
|
||||||
|
build:
|
||||||
|
pull: False
|
||||||
|
path: "/container/docker-images/slurm-base/"
|
||||||
|
dockerfile: "slurm-base.Dockerfile"
|
||||||
|
# target: "{{image.name}}" # unsupported on old docker-py version as in el7
|
||||||
|
source: build
|
||||||
|
force_source: "{{slurm_baseimg_copy.changed}}"
|
||||||
|
# when: run only once but keep changed state
|
||||||
|
when: not (slurm_baseimg_build_chg | default(False))
|
||||||
|
register: slurm_baseimg_build
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
slurm_baseimg_build_chg:
|
||||||
|
"{{(slurm_baseimg_build_chg | default(False)) or
|
||||||
|
slurm_baseimg_build.changed}}"
|
||||||
|
|
||||||
|
- include_tasks: dockerimage.yml
|
||||||
Reference in New Issue
Block a user