Merge pull request #2335 from mxinden/kube-prometheus-e2e-tests

kube-prometheus: Add e2e tests

Makefile

@@ -42,6 +42,9 @@ test: $(JB_BINARY)
 	$(JB_BINARY) install
 	./test.sh
 
+test-e2e:
+	go test -timeout 55m -v ./tests/e2e
+
 test-in-docker: ../../hack/jsonnet-docker-image
 	@echo ">> Compiling assets and generating Kubernetes manifests"
 	docker run \

tests/e2e/main_test.go

@@ -0,0 +1,118 @@
+// Copyright 2019 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+var promClient *prometheusClient
+
+func TestMain(m *testing.M) {
+	os.Exit(testMain(m))
+}
+
+// testMain circumvents the issue, that one can not call `defer` in TestMain, as
+// `os.Exit` does not honor `defer` statements. For more details see:
+// http://blog.englund.nu/golang,/testing/2017/03/12/using-defer-in-testmain.html
+func testMain(m *testing.M) int {
+	kubeConfigPath, ok := os.LookupEnv("KUBECONFIG")
+	if !ok {
+		log.Fatal("failed to retrieve KUBECONFIG env var")
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "creating kubeClient failed"))
+	}
+
+	promClient = newPrometheusClient(kubeClient)
+
+	return m.Run()
+}
+
+func TestQueryPrometheus(t *testing.T) {
+	t.Parallel()
+	queries := []struct {
+		query   string
+		expectN int
+	}{
+		{
+			// 	query:   `up{job="node-exporter"} == 1`,
+			// 	expectN: 1,
+			// }, {
+			// 	query:   `up{job="kubelet"} == 1`,
+			// 	expectN: 1,
+			// }, {
+			query:   `up{job="apiserver"} == 1`,
+			expectN: 1,
+			// }, {
+			// 	query:   `up{job="kube-state-metrics"} == 1`,
+			// 	expectN: 1,
+		}, {
+			query:   `up{job="prometheus-k8s"} == 1`,
+			expectN: 1,
+		}, {
+			query:   `up{job="prometheus-operator"} == 1`,
+			expectN: 1,
+		}, {
+			query:   `up{job="alertmanager-main"} == 1`,
+			expectN: 2,
+		},
+	}
+
+	// Wait for pod to respond at queries at all. Then start verifying their results.
+	err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) {
+		_, err := promClient.query("up")
+		return err == nil, nil
+	})
+	if err != nil {
+		t.Fatal(errors.Wrap(err, "wait for prometheus-k8s"))
+	}
+
+	err = wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) {
+		defer t.Log("---------------------------\n")
+
+		for _, q := range queries {
+			n, err := promClient.query(q.query)
+			if err != nil {
+				return false, err
+			}
+			if n < q.expectN {
+				// Don't return an error as targets may only become visible after a while.
+				t.Logf("expected at least %d results for %q but got %d", q.expectN, q.query, n)
+				return false, nil
+			}
+			t.Logf("query %q succeeded", q.query)
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}

tests/e2e/prometheus_client.go

@@ -0,0 +1,52 @@
+// Copyright 2019 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/Jeffail/gabs"
+)
+
+type prometheusClient struct {
+	kubeClient kubernetes.Interface
+}
+
+func newPrometheusClient(kubeClient kubernetes.Interface) *prometheusClient {
+	return &prometheusClient{kubeClient}
+}
+
+// Query makes a request against the Prometheus /api/v1/query endpoint.
+func (c *prometheusClient) query(query string) (int, error) {
+	req := c.kubeClient.CoreV1().RESTClient().Get().
+		Namespace("monitoring").
+		Resource("pods").
+		SubResource("proxy").
+		Name("prometheus-k8s-0:9090").
+		Suffix("/api/v1/query").Param("query", query)
+
+	b, err := req.DoRaw()
+	if err != nil {
+		return 0, err
+	}
+
+	res, err := gabs.ParseJSON(b)
+	if err != nil {
+		return 0, err
+	}
+
+	n, err := res.ArrayCountP("data.result")
+	return n, err
+}

tests/e2e/travis-e2e.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# exit immediately when a command fails
+set -e
+# only exit with zero if all commands of the pipeline exit successfully
+set -o pipefail
+# error on unset variables
+set -u
+# print each command before executing it
+set -x
+
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+"${SCRIPT_DIR}"/../../../../scripts/create-minikube.sh
+
+# waiting for kube-dns to be ready
+JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n kube-system get pods -lk8s-app=kube-dns -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for kube-dns to be available"; kubectl get pods --all-namespaces; done
+
+(
+    cd "${SCRIPT_DIR}"/../.. || exit
+    kubectl apply -f manifests
+    KUBECONFIG=~/.kube/config make test-e2e
+)
+
+"${SCRIPT_DIR}"/../../../../scripts/delete-minikube.sh