grafana-watcher: add datasource handling

grafana-watcher/examples/grafana-bundle.yaml

@@ -43,6 +43,11 @@ spec:
       containers:
       - name: grafana
         image: grafana/grafana:3.1.1
+        env:
+        - name: GF_AUTH_BASIC_ENABLED
+          value: "true"
+        - name: GF_AUTH_ANONYMOUS_ENABLED
+          value: "true"
         volumeMounts:
         - name: grafana-storage
           mountPath: /var/grafana-storage

grafana-watcher/examples/updated-configmap1.yaml

@@ -7,3 +7,5 @@ data:
     {"dashboard":{"__inputs":[{"name":"DS_PROMETHEUS","label":"Prometheus","description":"","type":"datasource","pluginId":"prometheus","pluginName":"Prometheus"}],"__requires":[{"type":"panel","id":"singlestat","name":"Singlestat","version":""},{"type":"panel","id":"text","name":"Text","version":""},{"type":"panel","id":"graph","name":"Graph","version":""},{"type":"grafana","id":"grafana","name":"Grafana","version":"3.1.0"},{"type":"datasource","id":"prometheus","name":"Prometheus","version":"1.0.0"}],"id":null,"title":"Prometheus Stats","tags":[],"style":"dark","timezone":"browser","editable":true,"hideControls":true,"sharedCrosshair":false,"rows":[{"collapse":false,"editable":true,"height":178,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"datasource":"${DS_PROMETHEUS}","decimals":1,"editable":true,"error":false,"format":"s","id":5,"interval":null,"links":[],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"targets":[{"expr":"(time() - process_start_time_seconds{job=\"prometheus\"})","intervalFactor":2,"refId":"A","step":4}],"thresholds":"","title":"Uptime","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current","mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"rangeMaps":[{"from":"null","to":"null","text":"N/A"}],"mappingType":1,"gauge":{"show":false,"minValue":0,"maxValue":100,"thresholdMarkers":true,"thresholdLabels":false}},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"${DS_PROMETHEUS}","editable":true,"error":false,"format":"none","id":6,"interval":null,"links":[],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":true},"targets":[{"expr":"prometheus_local_storage_memory_series","intervalFactor":2,"refId":"A","step":4}],"thresholds":"1,5","title":"Local Storage Memory Series","type":"singlestat","valueFontSize":"70%","valueMaps":[],"valueName":"current","mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"rangeMaps":[{"from":"null","to":"null","text":"N/A"}],"mappingType":1,"gauge":{"show":false,"minValue":0,"maxValue":100,"thresholdMarkers":true,"thresholdLabels":false}},{"cacheTimeout":null,"colorBackground":false,"colorValue":true,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"${DS_PROMETHEUS}","editable":true,"error":false,"format":"none","id":7,"interval":null,"links":[],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":true},"targets":[{"expr":"prometheus_local_storage_indexing_queue_length","intervalFactor":2,"refId":"A","step":4}],"thresholds":"500,4000","title":"Interal Storage Queue Length","type":"singlestat","valueFontSize":"70%","valueMaps":[{"op":"=","text":"Empty","value":"0"}],"valueName":"current","mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"rangeMaps":[{"from":"null","to":"null","text":"N/A"}],"mappingType":1,"gauge":{"show":false,"minValue":0,"maxValue":100,"thresholdMarkers":true,"thresholdLabels":false}},{"content":"<img src=\"http://prometheus.io/assets/prometheus_logo_grey.svg\" alt=\"Prometheus logo\" style=\"height: 40px;\">\n<span style=\"font-family: 'Open Sans', 'Helvetica Neue', Helvetica; font-size: 25px;vertical-align: text-top;color: #bbbfc2;margin-left: 10px;\">Prometheus</span>\n\n<p style=\"margin-top: 10px;\">You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the <a href=\"http://www.grafana.org/\">Grafana</a> and <a href=\"http://prometheus.io/\">Prometheus</a> projects.</p>","editable":true,"error":false,"id":9,"links":[],"mode":"html","span":3,"style":{},"title":"","transparent":true,"type":"text"}],"title":"New row"},{"collapse":false,"editable":true,"height":227,"panels":[{"aliasColors":{"prometheus":"#C15C17","{instance=\"localhost:9090\",job=\"prometheus\"}":"#C15C17"},"bars":false,"datasource":"${DS_PROMETHEUS}","editable":true,"error":false,"fill":1,"grid":{"threshold1":null,"threshold1Color":"rgba(216, 200, 27, 0.27)","threshold2":null,"threshold2Color":"rgba(234, 112, 112, 0.22)"},"id":3,"legend":{"avg":falLine truncated
   grafana-net-162-dashboard.json: |
     {"dashboard":{"__inputs":[{"name":"DS_PROMETHEUS","label":"Prometheus","description":"","type":"datasource","pluginId":"prometheus","pluginName":"Prometheus"}],"__requires":[{"type":"panel","id":"singlestat","name":"Singlestat","version":""},{"type":"panel","id":"graph","name":"Graph","version":""},{"type":"grafana","id":"grafana","name":"Grafana","version":"3.1.0"},{"type":"datasource","id":"prometheus","name":"Prometheus","version":"1.0.0"}],"id":null,"title":"Kubernetes cluster monitoring (via Prometheus)","tags":["kubernetes"],"style":"dark","timezone":"browser","editable":true,"hideControls":true,"sharedCrosshair":true,"rows":[{"collapse":false,"editable":true,"height":"250px","panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"${DS_PROMETHEUS}","editable":true,"error":false,"format":"percent","gauge":{"maxValue":100,"minValue":0,"show":true,"thresholdLabels":false,"thresholdMarkers":true},"id":4,"interval":null,"isNew":true,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":4,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"targets":[{"expr":"(sum(node_memory_MemTotal) - sum(node_memory_MemFree+node_memory_Buffers+node_memory_Cached) ) / sum(node_memory_MemTotal) * 100","interval":"10s","intervalFactor":1,"refId":"A","step":10}],"thresholds":"65, 90","title":"Cluster memory usage","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"${DS_PROMETHEUS}","decimals":2,"editable":true,"error":false,"format":"percent","gauge":{"maxValue":100,"minValue":0,"show":true,"thresholdLabels":false,"thresholdMarkers":true},"id":6,"interval":null,"isNew":true,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":4,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"targets":[{"expr":"sum(sum by (io_kubernetes_container_name)( rate(container_cpu_usage_seconds_total{image!=\"\"}[1m] ) )) / count(node_cpu{mode=\"system\"}) * 100","interval":"10s","intervalFactor":1,"refId":"A","step":10}],"thresholds":"65, 90","title":"Cluster CPU usage","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"${DS_PROMETHEUS}","decimals":2,"editable":true,"error":false,"format":"percent","gauge":{"maxValue":100,"minValue":0,"show":true,"thresholdLabels":false,"thresholdMarkers":true},"id":7,"interval":null,"isNew":true,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":4,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"targets":[{"expr":"(sum(node_filesystem_size{device=\"rootfs\"}) - sum(node_filesystem_free{device=\"rootfs\"}) ) / sum(node_filesystem_size{device=\"rootfs\"}) * 100","interval":"10s","intervalFactor":1,"metric":"","refId":"A","step":10}],"thresholds":"65, 90","title":"Cluster Filesystem usage","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"}],"title":"Row"},{"collapse":false,"editable":true,"height":"250px","panels":[{"aliasColors":{},"bars":false,"datasource":"${DS_PROMETHEUS}","decimals":3,"editable":true,"error":false,"fill":0,"grid":{"threshold1":null,"threshold1Color":"rgba(216, 200, 27, 0.27)","threshold2":null,"threshold2Color":"rgba(234, 112, 112, 0.22)"},"id":3,"isNew":true,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"span":12,"stack":false,Line truncated
+  prometheus-datasource.json: |
+    {"name":"prometheus","type":"prometheus","url":"http://prometheus:9090","access":"proxy","basicAuth":false}

grafana-watcher/grafana/datasource.go

@@ -0,0 +1,76 @@
+package grafana
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+)
+
+type DatasourcesInterface interface {
+	All() ([]GrafanaDatasource, error)
+	Create(datasourceJson io.Reader) error
+	Delete(id int) error
+}
+
+// DatasourcesClient is an implementation of the DatasourcesInterface. The
+// datasources HTTP API of Grafana requires admin access.
+type DatasourcesClient struct {
+	BaseUrl    string
+	HTTPClient *http.Client
+}
+
+type GrafanaDatasource struct {
+	Id   int    `json:"id"`
+	Name string `json:"name"`
+}
+
+func NewDatasourcesClient(baseUrl string, c *http.Client) DatasourcesInterface {
+	return &DatasourcesClient{
+		BaseUrl:    baseUrl,
+		HTTPClient: c,
+	}
+}
+
+func (c *DatasourcesClient) All() ([]GrafanaDatasource, error) {
+	allUrl := c.BaseUrl + "/api/datasources"
+	resp, err := c.HTTPClient.Get(allUrl)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	datasources := make([]GrafanaDatasource, 0)
+
+	err = json.NewDecoder(resp.Body).Decode(&datasources)
+	if err != nil {
+		return nil, err
+	}
+
+	return datasources, nil
+}
+
+func (c *DatasourcesClient) Delete(id int) error {
+	deleteUrl := c.BaseUrl + "/api/datasources/" + strconv.Itoa(id)
+	req, err := http.NewRequest("DELETE", deleteUrl, nil)
+	if err != nil {
+		return err
+	}
+
+	_, err = c.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *DatasourcesClient) Create(datasourceJson io.Reader) error {
+	createUrl := c.BaseUrl + "/api/datasources"
+	_, err := c.HTTPClient.Post(createUrl, "application/json", datasourceJson)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

grafana-watcher/grafana/grafana.go

@@ -6,6 +6,7 @@ import (
 
 type Interface interface {
 	Dashboards() DashboardsInterface
+	Datasources() DatasourcesInterface
 }
 
 type Clientset struct {
@@ -23,3 +24,7 @@ func New(baseUrl string) Interface {
 func (c *Clientset) Dashboards() DashboardsInterface {
 	return NewDashboardsClient(c.BaseUrl, c.HTTPClient)
 }
+
+func (c *Clientset) Datasources() DatasourcesInterface {
+	return NewDatasourcesClient(c.BaseUrl, c.HTTPClient)
+}

grafana-watcher/main.go

@@ -86,10 +86,10 @@ func main() {
 
 	g := grafana.New(*grafanaUrl)
 	du := updater.NewGrafanaDashboardUpdater(g.Dashboards(), filepath.Join(*watchDir, "*-dashboard.json"))
-	//su := updater.NewGrafanaSourceUpdater(g.Sources(), filepath.Join(*watchDir, "*-source.json"))
+	su := updater.NewGrafanaDatasourceUpdater(g.Datasources(), filepath.Join(*watchDir, "*-datasource.json"))
 	w := newVolumeWatcher(*watchDir)
 	w.AddEventHandler(du)
-	//w.AddEventHandler(su)
+	w.AddEventHandler(su)
 
 	w.Run()
 }

grafana-watcher/updater/datasource.go

@@ -0,0 +1,82 @@
+package updater
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/coreos/kube-prometheus/grafana-watcher/grafana"
+)
+
+type GrafanaDatasourceUpdater struct {
+	client grafana.DatasourcesInterface
+	glob   string
+}
+
+func NewGrafanaDatasourceUpdater(c grafana.DatasourcesInterface, g string) Updater {
+	return &GrafanaDatasourceUpdater{
+		client: c,
+		glob:   g,
+	}
+}
+
+func (u *GrafanaDatasourceUpdater) OnModify() error {
+	err := u.deleteAllDatasources()
+	if err != nil {
+		return err
+	}
+	err = u.createDatasourcesFromFiles()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (u *GrafanaDatasourceUpdater) deleteAllDatasources() error {
+	log.Println("Retrieving existing datasources")
+	datasources, err := u.client.All()
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Deleting %d datasources\n", len(datasources))
+	for _, d := range datasources {
+		log.Println("Deleting datasource:", d.Id)
+
+		err := u.client.Delete(d.Id)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (u *GrafanaDatasourceUpdater) createDatasourcesFromFiles() error {
+	filePaths, err := filepath.Glob(u.glob)
+	if err != nil {
+		return err
+	}
+
+	for _, fp := range filePaths {
+		u.createDatasourceFromFile(fp)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (u *GrafanaDatasourceUpdater) createDatasourceFromFile(filePath string) error {
+	log.Println("Creating datasource from:", filePath)
+
+	f, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return u.client.Create(f)
+}