说明：使用grafana官方plugins(grafana-kuberneres-app)链接prometheus展示kubernetes的信息

注意：所有与监控有关的信息都放在namespace=monitor上github:

１、创建monitor的namespace和安装prometheus

namespace.yaml

apiVersion: v1kind: Namespacemetadata: name: monitor

prom-rbac.yaml

apiVersion: rbac.authorization.k8s.io/v1beta1kind: ClusterRolemetadata:  name: prometheus  namespace: monitorrules:- apiGroups: [""]  resources:  - nodes  - nodes/proxy  - services  - endpoints  - pods  verbs: ["get", "list", "watch"]- apiGroups:  - extensions  resources:  - ingresses  verbs: ["get", "list", "watch"]- nonResourceURLs: ["/metrics"]  verbs: ["get"]---apiVersion: v1kind: ServiceAccountmetadata:  name: prometheus  namespace: monitor---apiVersion: rbac.authorization.k8s.io/v1beta1kind: ClusterRoleBindingmetadata:  name: prometheusroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: prometheussubjects:- kind: ServiceAccount  name: prometheus  namespace: monitor

cat prom-config-kubernetes.yaml

apiVersion: v1kind: ConfigMapmetadata: name: prometheus-config namespace: monitordata: prometheus.yml: |  global:  scrape_configs:   - job_name: 'kubernetes-kubelet'     scheme: https     tls_config:       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt       insecure_skip_verify: true     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token     kubernetes_sd_configs:     - role: node     relabel_configs:     - action: labelmap       regex: __meta_kubernetes_node_label_(.+)     - target_label: __address__       replacement: kubernetes.default.svc.cluster.local:443     - source_labels: [__meta_kubernetes_node_name]       regex: (.+)       target_label: __metrics_path__       replacement: /api/v1/nodes/${1}/proxy/metrics   - job_name: 'kubernetes-cadvisor'     scheme: https     tls_config:       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt       insecure_skip_verify: true     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token     kubernetes_sd_configs:     - role: node     relabel_configs:     - action: labelmap       regex: __meta_kubernetes_node_label_(.+)     - target_label: __address__       replacement: kubernetes.default.svc.cluster.local:443     - source_labels: [__meta_kubernetes_node_name]       regex: (.+)       target_label: __metrics_path__       replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor   - job_name: 'kubernetes-kube-state'     kubernetes_sd_configs:     - role: pod     relabel_configs:     - action: labelmap       regex: __meta_kubernetes_pod_label_(.+)     - source_labels: [__meta_kubernetes_namespace]       action: replace       target_label: kubernetes_namespace     - source_labels: [__meta_kubernetes_pod_name]       action: replace       target_label: kubernetes_pod_name     - source_labels: [__meta_kubernetes_pod_label_grafanak8sapp]       regex: .*true.*       action: keep     - source_labels: ['__meta_kubernetes_pod_label_daemon', '__meta_kubernetes_pod_node_name']       regex: 'node-exporter;(.*)'       action: replace       target_label: nodename

cat prom-install.yaml

apiVersion: v1kind: ConfigMapmetadata: name: prometheus-config-test namespace: monitordata: prometheus.yml: |  global:  scrape_configs:   - job_name: 'kubernetes-kubelet'     scheme: https     tls_config:       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt       insecure_skip_verify: true     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token     kubernetes_sd_configs:     - role: node     relabel_configs:     - action: labelmap       regex: __meta_kubernetes_node_label_(.+)     - target_label: __address__       replacement: kubernetes.default.svc.cluster.local:443     - source_labels: [__meta_kubernetes_node_name]       regex: (.+)       target_label: __metrics_path__       replacement: /api/v1/nodes/${1}/proxy/metrics   - job_name: 'kubernetes-cadvisor'     scheme: https     tls_config:       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt       insecure_skip_verify: true     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token     kubernetes_sd_configs:     - role: node     relabel_configs:     - action: labelmap       regex: __meta_kubernetes_node_label_(.+)     - target_label: __address__       replacement: kubernetes.default.svc.cluster.local:443     - source_labels: [__meta_kubernetes_node_name]       regex: (.+)       target_label: __metrics_path__       replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor   - job_name: 'kubernetes-kube-state'     kubernetes_sd_configs:     - role: pod     relabel_configs:     - action: labelmap       regex: __meta_kubernetes_pod_label_(.+)     - source_labels: [__meta_kubernetes_namespace]       action: replace       target_label: kubernetes_namespace     - source_labels: [__meta_kubernetes_pod_name]       action: replace       target_label: kubernetes_pod_name     - source_labels: [__meta_kubernetes_pod_label_grafanak8sapp]       regex: .*true.*       action: keep     - source_labels: ['__meta_kubernetes_pod_label_daemon', '__meta_kubernetes_pod_node_name']       regex: 'node-exporter;(.*)'       action: replace       target_label: nodename   - job_name: 'tomcat-pods'     kubernetes_sd_configs:     - role: pod     relabel_configs:     - source_labels: [__meta_kubernetes_namespace]       regex: tomcat       action: keep     - source_labels: [__meta_kubernetes_pod_container_port_number]       regex: 1234       action: keep     - source_labels: [__meta_kubernetes_pod_name]       action: replace       target_label: kubernetes_pod_name[root@k8s-master1 monitor]# cat prom/prom-install.yaml apiVersion: extensions/v1beta1kind: Deploymentmetadata: labels:  name: prometheus name: prometheus namespace: monitorspec:  replicas: 1  template:   metadata:    labels:      app: prometheus-server   spec:    serviceAccountName: prometheus    containers:    - name: prometheus      image: hub.llzg.com.cn/monitor/prometheus:latest      imagePullPolicy: Always      ports:        - containerPort: 9090          protocol: TCP      volumeMounts:        - mountPath: "/etc/prometheus"          name: config-prometheus    imagePullSecrets:    - name: authllzg    volumes:    - name: config-prometheus      configMap:       name: prometheus-config

cat prom-svc.yaml

kind: ServiceapiVersion: v1metadata:  name: prometheus-ingress-service  namespace: monitorspec:  selector:     app: prometheus-server  ports:    - protocol: TCP      port: 9090      name: prom---apiVersion: extensions/v1beta1kind: Ingressmetadata:  name: prom-web-ui  namespace: monitorspec:  rules:  - host: prom.name.com.cn    http:      paths:      - path: /        backend:          serviceName: prometheus-ingress-service          servicePort: prom

2、安装state-metrics

cat state-metrics-rbac.yaml

apiVersion: v1kind: ServiceAccountmetadata:  name: kube-state-metrics  namespace: monitor---apiVersion: rbac.authorization.k8s.io/v1kind: Rolemetadata:  namespace: monitor  name: kube-state-metrics-resizerrules:- apiGroups: [""]  resources:  - pods  verbs: ["get"]- apiGroups: ["extensions"]  resources:  - deployments  resourceNames: ["kube-state-metrics"]  verbs: ["get", "update"]---apiVersion: rbac.authorization.k8s.io/v1kind: RoleBindingmetadata:  name: kube-state-metrics  namespace: monitorroleRef:  apiGroup: rbac.authorization.k8s.io  kind: Role  name: kube-state-metrics-resizersubjects:- kind: ServiceAccount  name: kube-state-metrics  namespace: monitor---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRolemetadata:  name: kube-state-metrics  namespace: monitorrules:- apiGroups: [""]  resources:  - nodes  - pods  - services  - resourcequotas  - replicationcontrollers  - limitranges  - persistentvolumeclaims  - persistentvolumes  - namespaces  - endpoints  verbs: ["list", "watch"]- apiGroups: ["extensions"]  resources:  - daemonsets  - deployments  - replicasets  verbs: ["list", "watch"]- apiGroups: ["apps"]  resources:  - statefulsets  verbs: ["list", "watch"]- apiGroups: ["batch"]  resources:  - cronjobs  - jobs  verbs: ["list", "watch"]- apiGroups: ["autoscaling"]  resources:  - horizontalpodautoscalers  verbs: ["list", "watch"]---apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBindingmetadata:  name: kube-state-metrics  namespace: monitorroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: kube-state-metricssubjects:- kind: ServiceAccount  name: kube-state-metrics  namespace: monitor---{  "apiVersion": "apps/v1beta1",  "kind": "Deployment",  "metadata": {    "name": "kube-state-metrics",    "namespace": "monitor"  },  "spec": {    "selector": {      "matchLabels": {        "k8s-app": "kube-state-metrics",        "grafanak8sapp": "true"      }    },    "replicas": 1,    "template": {      "metadata": {        "labels": {          "k8s-app": "kube-state-metrics",          "grafanak8sapp": "true"        }      },      "spec": {        "serviceAccountName": "kube-state-metrics",        "containers": [          {            "name": "kube-state-metrics",            "image": "hub.llzg.com.cn/monitor/kube-state-metrics:v1.1.0",            "ports": [              {                "name": "http-metrics",                "containerPort": 8080              }            ],            "readinessProbe": {              "httpGet": {                "path": "/healthz",                "port": 8080              },              "initialDelaySeconds": 5,              "timeoutSeconds": 5            }          }        ],        "imagePullSecrets": [          {            "name": "authllzg"              }            ]      }    }  }}

3、安装node-exporter

cat node-exporter.yaml

{  "kind": "DaemonSet",  "apiVersion": "extensions/v1beta1",  "metadata": {    "name": "node-exporter",    "namespace": "monitor"  },  "spec": {    "selector": {      "matchLabels": {        "daemon": "node-exporter",        "grafanak8sapp": "true"      }    },    "template": {      "metadata": {        "name": "node-exporter",        "labels": {          "daemon": "node-exporter",          "grafanak8sapp": "true"        }      },      "spec": {        "volumes": [          {            "name": "proc",            "hostPath": {              "path": "/proc"            }          },          {            "name": "sys",            "hostPath": {              "path": "/sys"            }          }        ],        "containers": [          {            "name": "node-exporter",            "image": "hub.llzg.com.cn/monitor/node-exporter:v0.15.0",            "args": [              "--path.procfs=/proc_host",              "--path.sysfs=/host_sys"            ],            "ports": [              {                "name": "node-exporter",                "hostPort": 9100,                "containerPort": 9100              }            ],            "volumeMounts": [              {                "name": "sys",                "readOnly": true,                "mountPath": "/host_sys"              },              {                "name": "proc",                "readOnly": true,                "mountPath": "/proc_host"              }            ],            "imagePullPolicy": "IfNotPresent"          }        ],        "restartPolicy": "Always",        "hostNetwork": true,        "hostPID": true      }    }  }}

4、安装grafana(添加grafana-kubernetes-app插件)

cat grafana.yaml

apiVersion: extensions/v1beta1kind: Deploymentmetadata: labels:  name: grafana-server name: grafana namespace: monitorspec:  replicas: 1  template:   metadata:    labels:      app: grafana-server   spec:    serviceAccountName: prometheus    containers:    - name: grafana      image: hub.llzg.com.cn/monitor/grafana:latest      ports:        - containerPort: 3000          protocol: TCP      volumeMounts:      - mountPath: "/var/lib/grafana"        readOnly: false        name: grafana-nfs      env:      - name: GF_INSTALL_PLUGINS        value: "grafana-kubernetes-app"    imagePullSecrets:    - name: authllzg    volumes:    - name: grafana-nfs      nfs:       server: 192.168.10.28       path: "/data/nfs/monitor/grafana"---kind: ServiceapiVersion: v1metadata:  name: grafana-ingress-service  namespace: monitorspec:  selector:    app: grafana-server  ports:    - protocol: TCP      port: 3000      name: grafana---apiVersion: extensions/v1beta1kind: Ingressmetadata:  name: grafana-ingress  namespace: monitor  annotations:    kubernetes.io/ingress.class: traefikspec:  rules:  - host: grafana.llzg.com.cn    http:      paths:      - path: /        backend:          serviceName: grafana-ingress-service          servicePort: grafana

５、grafana+prometheus监控显示kubernetes

(1)grafana添加prometheus(2)加载grafana-kubernetes-app插件configuration---->plugins--------->kubernetes(3)配置grafana-kubernetes-app插件

４、OK，去看图吧