1、部署 node-exporter

[root@master-1 node-exporter]# cat node-exporter-cfg.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: node-exporter-web-config
  namespace: kube-monitor
data:
  web.yml: |
    basic_auth_users:
      prome: $2y$11$WJYP1tC.w5nyysv7SN6sc.2UAzfoQaaqvorJ54MbCsTinZy9ePNtq

[root@master-1 node-exporter]# cat node-exporter-cfg.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: node-exporter-web-config
  namespace: kube-monitor
data:
  web.yml: |
    basic_auth_users:
      prome: $2y$11$WJYP1tC.w5nyysv7SN6sc.2UAzfoQaaqvorJ54MbCsTinZy9ePNtq
[root@master-1 node-exporter]# cat node-exporter.yaml 
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  labels:
    app: node-exporter
  namespace: kube-monitor
spec:
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      annotations:
        prometheus.io/should_be_scraped: "true"
        prometheus.io/scrape_port: "48100"
        prometheus.io/metric_path: "metrics"
      labels:
        app: node-exporter
    spec:
      tolerations:
      - key: node-role.kubernetes.io/control-plane
        effect: NoSchedule
      containers:
      - image: docker.io/prom/node-exporter:v1.9.0 
        args:
        - --web.listen-address=:48100
        - --web.config.file=/usr/local/node_exporter/web.yml
        - --path.sysfs=/host/sys
        - --path.rootfs=/host/root
        - --no-collector.wifi
        - --no-collector.hwmon
        - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)
        - --collector.netclass.ignored-devices=^(veth.*)$
        - --collector.netdev.device-exclude=^(veth.*)$
        imagePullPolicy: IfNotPresent
        name: node-exporter
        ports:
        - containerPort: 48100
          hostPort: 48100
          name: scrape
        #securityContext:
        #  privileged: true
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
        - mountPath: /host/sys
          mountPropagation: HostToContainer
          name: sys
          readOnly: true
        - mountPath: /host/root
          mountPropagation: HostToContainer
          name: root
          readOnly: true
        - mountPath: /usr/local/node_exporter/
          name: node-exporter-web-config
      volumes:
      - hostPath:
          path: /sys
        name: sys
      - hostPath:
          path: /
        name: root
      - name: node-exporter-web-config
        configMap:
          name: node-exporter-web-config
      hostNetwork: true
      hostPID: true
      hostIPC: true

2、部署 kube-state-metrics

[root@master-1 kube-state-metrics]# cat cluster-role.yaml 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: 2.9.2
  name: kube-state-metrics
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - secrets
  - nodes
  - pods
  - services
  - serviceaccounts
  - resourcequotas
  - replicationcontrollers
  - limitranges
  - persistentvolumeclaims
  - persistentvolumes
  - namespaces
  - endpoints
  verbs:
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - statefulsets
  - daemonsets
  - deployments
  - replicasets
  verbs:
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - cronjobs
  - jobs
  verbs:
  - list
  - watch
- apiGroups:
  - autoscaling
  resources:
  - horizontalpodautoscalers
  verbs:
  - list
  - watch
- apiGroups:
  - authentication.k8s.io
  resources:
  - tokenreviews
  verbs:
  - create
- apiGroups:
  - authorization.k8s.io
  resources:
  - subjectaccessreviews
  verbs:
  - create
- apiGroups:
  - policy
  resources:
  - poddisruptionbudgets
  verbs:
  - list
  - watch
- apiGroups:
  - certificates.k8s.io
  resources:
  - certificatesigningrequests
  verbs:
  - list
  - watch
- apiGroups:
  - discovery.k8s.io
  resources:
  - endpointslices
  verbs:
  - list
  - watch
- apiGroups:
  - storage.k8s.io
  resources:
  - storageclasses
  - volumeattachments
  verbs:
  - list
  - watch
- apiGroups:
  - admissionregistration.k8s.io
  resources:
  - mutatingwebhookconfigurations
  - validatingwebhookconfigurations
  verbs:
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - networkpolicies
  - ingressclasses
  - ingresses
  verbs:
  - list
  - watch
- apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - list
  - watch
- apiGroups:
  - rbac.authorization.k8s.io
  resources:
  - clusterrolebindings
  - clusterroles
  - rolebindings
  - roles
  verbs:
  - list
  - watch

[root@master-1 kube-state-metrics]# cat cluster-role-binding.yaml 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: 2.9.2
  name: kube-state-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-state-metrics
subjects:
- kind: ServiceAccount
  name: kube-state-metrics
  namespace: kube-system

[root@master-1 kube-state-metrics]# cat service.yaml 
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: 2.9.2
  name: kube-state-metrics
  namespace: kube-system
spec:
  clusterIP: None
  ports:
  - name: http-metrics
    port: 8080
    targetPort: http-metrics
  - name: telemetry
    port: 8081
    targetPort: telemetry
  selector:
    app.kubernetes.io/name: kube-state-metrics

[root@master-1 kube-state-metrics]# cat service-account.yaml 
apiVersion: v1
automountServiceAccountToken: false
kind: ServiceAccount
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: 2.9.2
  name: kube-state-metrics
  namespace: kube-system

[root@master-1 kube-state-metrics]# cat deployment.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: 2.9.2
  name: kube-state-metrics
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: kube-state-metrics
  template:
    metadata:
      labels:
        app.kubernetes.io/component: exporter
        app.kubernetes.io/name: kube-state-metrics
        app.kubernetes.io/version: 2.9.2
    spec:
      automountServiceAccountToken: true
      containers:
      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.9.2
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
          initialDelaySeconds: 5
          timeoutSeconds: 5
        name: kube-state-metrics
        ports:
        - containerPort: 8080
          name: http-metrics
        - containerPort: 8081
          name: telemetry
        readinessProbe:
          httpGet:
            path: /
            port: 8081
          initialDelaySeconds: 5
          timeoutSeconds: 5
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 65534
          seccompProfile:
            type: RuntimeDefault
      nodeSelector:
        kubernetes.io/os: linux
      serviceAccountName: kube-state-metrics

3、部署 cadvisor

[root@master-1 cadvisor]# cat clusterrole.yaml 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    app: kube-monitor-cadvisor
  name: kube-monitor-cadvisor
rules:
- apiGroups:
  - policy
  resourceNames:
  - cadvisor
  resources:
  - podsecuritypolicies
  verbs:
  - use

[root@master-1 cadvisor]# cat clusterrole.yaml 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    app: kube-monitor-cadvisor
  name: kube-monitor-cadvisor
rules:
- apiGroups:
  - policy
  resourceNames:
  - cadvisor
  resources:
  - podsecuritypolicies
  verbs:
  - use
[root@master-1 cadvisor]# cat clusterrolebinding.yaml 
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    app: kube-monitor-cadvisor
  name: kube-monitor-cadvisor
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-monitor-cadvisor
subjects:
- kind: ServiceAccount
  name: kube-monitor-cadvisor
  namespace: kube-monitor

[root@master-1 cadvisor]# cat sa.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: kube-monitor-cadvisor
  name: kube-monitor-cadvisor
  namespace: kube-monitor

[root@master-1 cadvisor]# cat svc.yaml 
apiVersion: v1
kind: Service
metadata:
  name: kube-monitor-cadvisor
  labels:
    app: kube-monitor-cadvisor
  namespace: kube-monitor
spec:
  selector:
    app: kube-monitor-cadvisor
  ports:
  - name: cadvisor
    port: 8080
    protocol: TCP
    targetPort: 8080

[root@master-1 cadvisor]# cat daemonset.yaml 
apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    seccomp.security.alpha.kubernetes.io/pod: docker/default
  labels:
    app: kube-monitor-cadvisor
  name: kube-monitor-cadvisor
  namespace: kube-monitor
spec:
  selector:
    matchLabels:
      app: kube-monitor-cadvisor
      name: kube-monitor-cadvisor
  template:
    metadata:
      labels:
        app: kube-monitor-cadvisor
        name: kube-monitor-cadvisor
    spec:
      automountServiceAccountToken: false
      containers:
      - args:
        - --housekeeping_interval=10s
        - --max_housekeeping_interval=15s
        - --event_storage_event_limit=default=0
        - --event_storage_age_limit=default=0
        - --enable_metrics=app,cpu,disk,diskIO,memory,network,process
        - --docker_only
        - --store_container_labels=false
        - --whitelisted_container_labels=io.kubernetes.container.name,io.kubernetes.pod.name,io.kubernetes.pod.namespace
        image: gcr.io/cadvisor/cadvisor:v0.49.1 
        name: cadvisor
        ports:
        - containerPort: 8080
          name: http
          protocol: TCP
        resources:
          limits:
            cpu: 1
            memory: 2000Mi
          requests:
            cpu: 500m
            memory: 500Mi
        volumeMounts:
        - mountPath: /rootfs
          name: rootfs
          readOnly: true
        - mountPath: /var/run
          name: var-run
          readOnly: true
        - mountPath: /sys
          name: sys
          readOnly: true
        - mountPath: /var/lib/docker
          name: docker
          readOnly: true
        - mountPath: /dev/disk
          name: disk
          readOnly: true
        securityContext: 
          privileged: true
      priorityClassName: system-node-critical
      serviceAccountName: kube-monitor-cadvisor
      terminationGracePeriodSeconds: 30
      tolerations:
      - key: node-role.kubernetes.io/control-plane
        effect: NoSchedule
      volumes:
      - hostPath:
          path: /
        name: rootfs
      - hostPath:
          path: /var/run
        name: var-run
      - hostPath:
          path: /sys
        name: sys
      - hostPath:
          path: /var/lib/docker
        name: docker
      - hostPath:
          path: /dev/disk
        name: disk

4、部署 grafana

[root@master-1 grafana]# cat grafana.yaml 
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana-server
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      task: monitoring
      k8s-app: grafana
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: grafana
    spec:
      nodeName: node-1 # 部署到那个节点
      containers:
      - name: grafana
        image: docker.io/grafana/grafana:latest
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 3000
          protocol: TCP
        volumeMounts:
        - mountPath: /etc/ssl/certs
          name: ca-certificates
          readOnly: true
        - mountPath: /var
          name: grafana-storage
        - mountPath: /var/lib/grafana/
          name: lib
        #- name: localtime
         #mountPath: /etc/localtime
        env:
        - name: INFLUXDB_HOST
          value: monitoring-influxdb
        - name: GF_SERVER_HTTP_PORT
          value: "3000"
          # The following env variables are required to make Grafana accessible via
          # the kubernetes api-server proxy. On production clusters, we recommend
          # removing these env variables, setup auth for grafana, and expose the grafana
          # service using a LoadBalancer or a public IP.
        - name: GF_AUTH_BASIC_ENABLED
          value: "false"
        - name: GF_AUTH_ANONYMOUS_ENABLED
          value: "true"
        - name: GF_AUTH_ANONYMOUS_ORG_ROLE
          value: Admin
        - name: GF_SERVER_ROOT_URL
          # If you're only using the API Server proxy, set this value instead:
          # value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy
          value: /
      volumes:
      #- name: localtime
        #hostPath:
          #path: /etc/localtime
      - name: ca-certificates
        hostPath:
          path: /etc/ssl/certs
          type: DirectoryOrCreate
      - name: grafana-storage
        emptyDir: {}
      - name: lib
        hostPath:
         path: /var/lib/grafana/
         type: DirectoryOrCreate
---
apiVersion: v1
kind: Service
metadata:
  labels:
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-grafana
  name: grafana-svc
  namespace:  kube-system
spec:
  # In a production setup, we recommend accessing Grafana through an external Loadbalancer
  # or through a public IP.
  # type: LoadBalancer
  # You could also use NodePort to expose the service at a randomly-generated port
  # type: NodePort
  ports:
  - port: 80
    targetPort: 3000
    nodePort: 31091
  selector:
    k8s-app: grafana
  type: NodePort

mkdir -p /var/lib/grafana/plugins/

chmod 777 /var/lib/grafana/plugins/

chmod 777 -R /var/lib/grafana/

5、部署 prometheus

[root@master-1 prometheus]# cat prom-svc.yaml 
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: kube-monitor
  labels:
    app: prometheus
spec:
  type: NodePort
  ports:
  - port: 9090
    name: tcp-web
    targetPort: 9090
  selector:
    app: prometheus

[root@master-1 prometheus]# cat prom-svc.yaml 
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: kube-monitor
  labels:
    app: prometheus
spec:
  type: NodePort
  ports:
  - port: 9090
    name: tcp-web
    targetPort: 9090
  selector:
    app: prometheus
[root@master-1 prometheus]# cat prom-cfg.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-configuration
  namespace: kube-monitor
data:
  prometheus.yml: |
    global:
      scrape_interval:     15s
      evaluation_interval: 15s

    rule_files:
      - "/etc/prometheus/*.rules"

    scrape_configs:
      - job_name: 'prometheus'
        basic_auth:
          username: prome
          password: G73iUpiZRvviDARk
        static_configs:
        - targets: ['localhost:9090']

      - job_name: 'kubernetes-apiservers'
        kubernetes_sd_configs:
        - role: endpoints
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https
        - target_label: __address__
          replacement: kubernetes.default.svc:443

      - job_name: 'kubernetes-nodes'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

      - job_name: 'kubernetes-cadvisor'
        scheme: https
        metrics_path: /metrics/cadvisor
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node
        relabel_configs:
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: metrics/cadvisor
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - source_labels: [instance]
          separator: ;
          regex: (.+)
          target_label: node
          replacement: $1
          action: replace

      - job_name: 'kube-state-metrics'
        kubernetes_sd_configs:
        - role: endpoints
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: kube-system;kube-state-metrics;http-metrics
        - target_label: __address__
          replacement: kube-state-metrics.kube-system.svc.cluster.local:8080


      - job_name: 'kubernetes-service-endpoints'
        kubernetes_sd_configs:
        - role: endpoints
        relabel_configs:
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
          action: replace
          target_label: __scheme__
          regex: (https?)
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_service_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: kubernetes_name
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: kubernetes_pod_name
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: pod


      - job_name: 'kubernetes-pods'
        basic_auth:
          username: prome
          password: gt4A19JHlsiIjQeE
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_should_be_scraped]
          action: keep
          regex: true
      # - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_metric_path]
      #   action: replace
      #   target_label: __metrics_path__
      #   regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_scrape_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: kubernetes_pod_name

      - job_name: 'kubernetes-services'
        metrics_path: /probe
        params:
          module: [http_2xx]
        kubernetes_sd_configs:
        - role: service
        relabel_configs:
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
          action: keep
          regex: true
        - source_labels: [__address__]
          target_label: __param_target
        - source_labels: [__param_target]
          target_label: instance
        - source_labels: [__address__]
          target_label: __param_target
          regex: 'jenkins-server\.kube-jenkins\.svc\:8080'
          replacement: 'jenkins-server.kube-jenkins.svc:8080/login'
        - target_label: __address__
          replacement: blackbox-exporter.kube-monitor.svc.cluster.local:9115
        - action: labelmap
          regex: __meta_kubernetes_service_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_service_name]
          target_label: kubernetes_name

      - job_name: 'kube-monitor-cadvisor'
        kubernetes_sd_configs:
        - role: endpoints
          namespaces:
            names:
            - kube-monitor
          selectors:
            - role: endpoints
              label: "app=kube-monitor-cadvisor"
        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_node_name]
          target_label: node
        - source_labels: [__metrics_path__]
          target_label: __metrics_path__
          replacement: /metrics
        metric_relabel_configs:
        - source_labels: [container_label_io_kubernetes_pod_name]
          action: replace
          target_label: pod
        - source_labels: [container_label_io_kubernetes_container_name]
          action: replace
          target_label: container
        - source_labels: [container_label_io_kubernetes_pod_namespace]
          action: replace
          target_label: namespace
        - regex: container_label_io_kubernetes_pod_name
          action: labeldrop
        - regex: container_label_io_kubernetes_container_name
          action: labeldrop
        - regex: container_label_io_kubernetes_pod_namespace
          action: labeldrop

      - job_name: 'my-milvus'
        scrape_interval: 30s
        scrape_timeout: 10s
        scheme: http
        metrics_path: /metrics
        kubernetes_sd_configs:
        - role: endpoints
          namespaces:
            names:
            - my-milvus
        relabel_configs:
        - source_labels: [__address__]
          action: replace
          regex: ([^:]+)(?::\d+)?
          replacement: $1:9091
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_endpoints_label_(.+)
        - source_labels: [__meta_kubernetes_endpoints_label_component]
          action: replace
          target_label: app_kubernetes_io_component
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: pod
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: kubernetes_service_name
        - regex: component
          action: labeldrop


      - job_name: 'docker-etcd'
        metrics_path: /metrics
        scheme: http
        static_configs:
        - targets:
          - 10.0.41.84:3379
          - 10.0.41.85:3379
          - 10.0.41.86:3379

      - job_name: 'minio-cluster'
        bearer_token: eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJwcm9tZXRoZXVzIiwic3ViIjoiYWRtaW4iLCJleHAiOjQ5MDMzMTg5OTB9.Ilw95qxcGWpm6CEHn8qX2yLyWjvCF2tnrm1QcTnDqC297u_htOh_UiNs8wSw4bmU65m1FZpqS9UuT4RQc2UZ3w
        metrics_path: /minio/v2/metrics/cluster
        scheme: http
        static_configs:
        - targets:
          - my-milvus-minio.my-milvus.svc.cluster.local:9000

      - job_name: 'minio-node'
        bearer_token: eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJwcm9tZXRoZXVzIiwic3ViIjoiYWRtaW4iLCJleHAiOjQ5MDMzMTg5OTB9.Ilw95qxcGWpm6CEHn8qX2yLyWjvCF2tnrm1QcTnDqC297u_htOh_UiNs8wSw4bmU65m1FZpqS9UuT4RQc2UZ3w
        metrics_path: /minio/v2/metrics/node
        scheme: http
        static_configs:
        - targets:
          - 10.0.41.81:9000
          - 10.0.41.84:9000
          - 10.0.41.85:9000
          - 10.0.41.86:9000

      - job_name: 'minio-bucket'
        bearer_token: eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJwcm9tZXRoZXVzIiwic3ViIjoiYWRtaW4iLCJleHAiOjQ5MDMzMTg5OTB9.Ilw95qxcGWpm6CEHn8qX2yLyWjvCF2tnrm1QcTnDqC297u_htOh_UiNs8wSw4bmU65m1FZpqS9UuT4RQc2UZ3w
        metrics_path: /minio/v2/metrics/bucket
        static_configs:
        - targets:
          - my-milvus-minio.my-milvus.svc.cluster.local:9000
        scheme: http

      - job_name: 'minio-resource'
        bearer_token: eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJwcm9tZXRoZXVzIiwic3ViIjoiYWRtaW4iLCJleHAiOjQ5MDMzMTg5OTB9.Ilw95qxcGWpm6CEHn8qX2yLyWjvCF2tnrm1QcTnDqC297u_htOh_UiNs8wSw4bmU65m1FZpqS9UuT4RQc2UZ3w
        metrics_path: /minio/v2/metrics/resource
        static_configs:
        - targets:
          - my-milvus-minio.my-milvus.svc.cluster.local:9000
        scheme: http

      - job_name: 'blackbox'
        params:
          module: [tcp_connect]
        static_configs:
          - targets:
            - 10.0.41.84:2181
            - 10.0.41.85:2181
            - 10.0.41.86:2181
            - 10.0.41.84:9092
            - 10.0.41.85:9092
            - 10.0.41.86:9092
            - 10.0.41.81:9000
            - 10.0.41.84:9000
            - 10.0.41.85:9000
            - 10.0.41.86:9000
            - 10.0.41.84:3379
            - 10.0.41.85:3379
            - 10.0.41.86:3379
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: blackbox-exporter:9115
      - job_name: 'blackbox_exporter'
        static_configs:
          - targets: ['blackbox-exporter:9115']

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-web-config
  namespace: kube-monitor
data:
  web.yml: |
    basic_auth_users:
      prome: $2y$11$ftK8kUAG.Fi1bAKlJcSFK.x3U.rq9IuSpQpcLApg58QWY52xa/n3y

[root@master-1 prometheus]# cat prom-deployment.yaml 
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus-server
  namespace: kube-monitor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus-server
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - nodes/metrics
  - services
  - endpoints
  - pods
  verbs: ["get", "list", "watch"]
- apiGroups:
  - extensions
  - networking.k8s.io
  resources:
  - ingresses
  verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus-server
subjects:
- kind: ServiceAccount
  name: prometheus-server
  namespace: kube-monitor
roleRef:
  kind: ClusterRole
  name: prometheus-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: kube-monitor
spec:
  selector:
    matchLabels:
      app: prometheus
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      serviceAccountName: prometheus-server
      containers:
      - name: prometheus
        image: docker.io/prom/prometheus:v2.45.6 
        args:
        - "--config.file=/etc/prometheus/prometheus.yml"
        - "--storage.tsdb.path=/prometheus"
        - "--web.config.file=/etc/prometheus/web.yml"
        - "--storage.tsdb.retention.time=14d"
        - "--web.console.libraries=/usr/share/prometheus/console_libraries"
        - "--web.console.templates=/usr/share/prometheus/consoles"
        - "--web.enable-lifecycle"
        - "--storage.tsdb.wal-compression"
        - "--log.level=info"
        ports:
        - containerPort: 9090
          name : tcp-web
        livenessProbe:
          httpGet:
            httpHeaders:
              - name: Authorization
                value: Basic cHJvbWU6RzczaVVwaVpSdnZpREFSaw==
            path: /status
            port: 9090
          initialDelaySeconds: 15
          timeoutSeconds: 3
          failureThreshold: 12
        readinessProbe:
          httpGet:
            httpHeaders:
              - name: Authorization
                value: Basic cHJvbWU6RzczaVVwaVpSdnZpREFSaw==
            path: /status
            port: 9090
          initialDelaySeconds: 15
          timeoutSeconds: 3
          failureThreshold: 12
        volumeMounts:
          - name: prometheus-config-volume
            mountPath: /etc/prometheus/prometheus.yml
            subPath: prometheus.yml
          - name: prometheus-config-rules-volume
            mountPath: /etc/prometheus/kube_state.rules
            subPath: kube_state.rules
          - name: prometheus-storage-volume
            mountPath: /prometheus
          - mountPath: /etc/prometheus/web.yml
            name: prometheus-web-config
            subPath: web.yml
        resources:
          limits:
            cpu: 2
            memory: 4Gi
          requests:
            cpu: 1
            memory: 2Gi
      volumes:
      - name: prometheus-config-volume
        configMap:
          name: prometheus-configuration
      - name: prometheus-config-rules-volume
        configMap:
          name: prometheus-config-rules
      - name: prometheus-web-config
        configMap:
          name: prometheus-web-config
      - name: prometheus-storage-volume
        hostPath:
          path: /data/prometheus
          type: Directory
      nodeSelector:
        kubernetes.io/hostname: node-1

[root@master-1 prometheus]# cat rules.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config-rules
  namespace: kube-monitor
data:
  kube_state.rules: |
    groups:
    - name: kube-state-metrics
      rules:
      - alert: KubeStateMetricsListErrors
        annotations:
          description: kube-state-metrics is experiencing errors at an elevated rate in
            list operations. This is likely causing it to not be able to expose metrics
            about Kubernetes objects correctly or at all.
          summary: kube-state-metrics is experiencing errors in list operations.
        expr: |
          (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical
      - alert: KubeStateMetricsWatchErrors
        annotations:
          description: kube-state-metrics is experiencing errors at an elevated rate in
            watch operations. This is likely causing it to not be able to expose metrics
            about Kubernetes objects correctly or at all.
          summary: kube-state-metrics is experiencing errors in watch operations.
        expr: |
          (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical


      - alert: The average memory usage of the node exceeds 85%
        expr: (1 - avg_over_time(node_memory_MemFree_bytes[30m]) / avg_over_time(node_memory_MemTotal_bytes[30m]))*100 > 85
        for: 15m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '⚠警告 - 冯明洋-milvus集群 - 节点/{{ $labels.instance }} 内存平均使用率为 {{ $value | printf "%.2f" }}%'
          description: "服务器 {{ $labels.instance }} of job {{ $labels.job }} 内存平均使用率超过80%."

      - alert: Deployment_Status_Condition
        expr: kube_deployment_status_condition{condition="Available",status="true"} == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.deployment }} 当前状态异常."
          description: "The current status conditions of a deployment is available."

      - alert: Deployment_Status_Replicas_Unavailable
        expr: kube_deployment_status_replicas_unavailable > 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.deployment }} 不可用副本数为 {{ $value }}."
          description: "The number of unavailable replicas per deployment."

      - alert: Pod_Container_Status_Running
        expr: kube_pod_container_status_running == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 运行状态异常."
          description: "whether the container is currently in running state."

      - alert: Pod_Status_Phase_Running
        expr: kube_pod_status_phase{phase="Running"} == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.pod }} 运行状态异常."
          description: "The pods current phase."

      - alert: Pod_Container_Status_Last_Terminated_Reason_OOMKilled
        expr: kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} > 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "⚠警告 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 上次终止的原因为OOMKilled."
          description: "Describes the last reason the container was in terminated state."

      - alert: Pod_Container_Status_Restarted
        expr: changes(kube_pod_container_status_restarts_total [5m]) > 0
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 已重启."
          description: "The container has been restarted."

      - alert: Pod_Container_Status_Ready
        expr: kube_pod_container_status_ready == 0
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 运行状态异常."
          description: "The pod status unavailable."

      - alert: Container_Memory_Working_Set_Proportion
        expr: |
          (sum by(namespace, container, pod) (avg_over_time(container_memory_working_set_bytes{container!="",container!="POD"}[30m]))
            /
          sum by(namespace, container, pod) (container_spec_memory_limit_bytes{container!="",container!="POD"}))
            *
          on (namespace, pod) group_left(node, pod_ip) kube_pod_info * 100
          >= 80 !=+Inf
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: '⚠警告 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 内存平均使用率为 {{ $value | printf "%.2f" }}%.'
          description: "The Memory_Working_Set_Proportion of Container."

      - alert: Container_Cpu_Usage_Seconds_Total_Proportion
        expr: |
          (sum(rate(container_cpu_usage_seconds_total{container !="",container!="POD"}[30m])) by (namespace, container, pod)
            /
          (sum(container_spec_cpu_quota{container !="",container!="POD"}/100000) by (namespace, container, pod))) * 100
          >= 80
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: '⚠警告 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} CPU平均使用率为 {{ $value | printf "%.2f" }}%.'
          description: "The Cpu Usage_Seconds_Total of Container."

      - alert: Daemonset_Status_Number_Unavailable
        expr: kube_daemonset_status_number_unavailable > 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: '❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.daemonset }} 不可用副本数为 {{ $value }}.'
          description: "The number of nodes that should be running the daemon pod and have none of the daemon pod running and available."

      - alert: Daemonset_Status_Number_Misscheduled
        expr: kube_daemonset_status_number_misscheduled > 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: '❌严重 - milvus-冯明洋集群- {{ $labels.namespace }}/{{ $labels.daemonset }} 未正常调度副本数为 {{ $value }}.'
          description: "The number of nodes running a daemon pod but are not supposed to."

      - alert: Node_Status_Condition_Ready
        expr: kube_node_status_condition{condition="Ready", status="true"} != 1
        for: 3m
        labels:
          severity: critical
          service: k8s
        annotations:
          summary: '❌严重 - milvus-冯明洋集群- 节点/{{ $labels.node }} 状态为不可用.'
          description: "The condition of a cluster node."
 
      
      - alert: The load average utilization of the node exceeds 50%
        expr: sum(node_load15{}) by (instance) / count(node_cpu_seconds_total{mode="user"}) by (instance) * 100 > 50
        for: 15m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群- 节点/{{ $labels.instance }} 当前负载平均使用率为 {{ $value | printf "%.2f" }}%'
          description: "服务器 {{ $labels.instance }} of job {{ $labels.job }} 负载平均使用率超过50%."
      
      - alert: The node's network bandwidth usage exceeds 70%.
        expr: rate(node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr.*|lo.*|cni.*'}[30m]) / node_network_speed_bytes{device!~'tap.*|veth.*|br.*|docker.*|virbr.*|lo.*|cni.*'} * 100 >= 70
        for: 15m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群- 节点/{{ $labels.instance }} 网络带宽使用率为 {{ $value | printf "%.2f" }}%'
          description: "服务器 {{ $labels.instance }} of job {{ $labels.job }} 网络带宽使用率超过70%."
      
      - alert: The CPU usage of the node exceeds 50%
        expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[15m])) by (instance))*100 >= 50
        for: 5m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群- 节点/{{ $labels.instance }} CPU使用率为 {{ $value | printf "%.2f" }}%'
          description: "服务器 {{ $labels.instance }} of job {{ $labels.job }} CPU使用率超过50%."

      - alert: The Kubernetes etcd server up status
        expr: changes(up{job=~".*-etcd"}[5m]) != 0
        for: 3m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群- etcd/{{ $labels.instance }} 运行状态异常'
          description: "服务器 {{ $labels.instance }} of job {{ $labels.job }} 运行状态异常."

      - alert: The Milvus server Components up status
        expr: changes(up{job=~"blackbox"}[5m]) != 0
        for: 3m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群- Milvus/{{ $labels.instance }} 运行状态异常'
          description: "milvus集群 {{ $labels.instance }} of job {{ $labels.job }} 运行状态异常."

      - alert: The Minio Cluster health status
        expr: changes(minio_cluster_health_status[5m]) != 0
        for: 3m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群 minio集群健康状态异常'
          description: "minio集群 {{ $labels.instance }} of job {{ $labels.job }} 健康状态异常."

      - alert: The Minio Cluster drive offline total
        expr: avg_over_time(minio_cluster_drive_offline_total[5m]) > 0
        for: 3m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群 minio集群离线磁盘数量为 {{ $value }}'
          description: "minio集群磁盘 {{ $labels.instance }} of job {{ $labels.job }} 离线数量为 {{ $value }}."

      - alert: The Minio Cluster nodes offline total
        expr: avg_over_time(minio_cluster_nodes_offline_total{job="minio-cluster"}[5m]) > 0
        for: 3m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群 minio集群离线节点数量为 {{ $value }}'
          description: "minio集群节点 {{ $labels.instance }} of job {{ $labels.job }} 离线数量为 {{ $value }}"

      - alert: The Minio Cluster capacity usage exceeds 90%
        expr: (1- avg_over_time(minio_cluster_capacity_raw_free_bytes[30m]) / minio_cluster_capacity_raw_total_bytes) * 100 > 90
        for: 3m
        labels:
          severity: warning
          service: k8s
        annotations:
          summary: '严重 - milvus-冯明洋集群 minio集群空间使用率为 {{ $value | printf "%.2f" }}%'
          description: "minio集群 {{ $labels.instance }} of job {{ $labels.job }} 使用率超过90%"

 mkdir -p /data/prometheus/

chmod 777 /data/prometheus/

6、部署 alert

[root@master-1 alert]# cat configmap.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-monitor-alerts-nginx-config
  namespace: kube-monitor
data:
  alerts.conf: |
    server {
        listen 80;
        server_name localhost;
        charset utf-8;

        location / {
            root   /usr/share/nginx/alerts;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-monitor-get-alerts
  namespace: kube-monitor
data:
  get_alerts.sh: |
  
    livez_check() {
      livez_response=$(curl -k --silent https://localhost:6443/livez?verbose | grep -Ev 'ok|passed')
      if [ -n "${livez_response}" ]; then
        echo -e "error\n${livez_response}"
      else
        echo "ok"
      fi
    }
    livez_check

    pods_status() {
      status=$(kubectl get pod --all-namespaces -o wide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -Ev 'Completed')
      result=$(echo "$status" | grep -Ev 'NAMESPACE')
      if [ -n "$result" ]; then
        echo -e "冯明洋 Milvus 集群\nerror:\n$result"
      else
        echo "ok"
      fi
    }
    pods_status

    nodes_status() {
      status=$(kubectl get node -o wide | grep -v NAME)
      result=$(echo "$status" | awk '{if ($2 == "NotReady") {print "冯明洋 Milvus 集群 \nerror: \n"$1, $2, $3, $6}}')
      if [ -n "$result" ]; then
        echo -e "$result"
      else
        echo "ok"
      fi
    }
    nodes_status

    get_k8s_nodes_alerts(){
      alerts=$(curl -s -H 'Authorization: Basic cHJvbWU6RzczaVVwaVpSdnZpREFSaw==' http://10.0.41.81:32628/api/v1/alerts | jq ".data.alerts[] | select(.state==\"firing\") | .annotations.summary")
      if [ -n "$alerts" ]; then
        echo -e "error\n$alerts" | sed 's/"//g;s/:49100//g' | sort
      else
        echo "ok"
      fi
    }

    if [ -f /root/zujian/zujian_status.txt ]; then
      cp /root/zujian/zujian_status.txt /alerts/zujian_status.txt
    else
      echo "zujian_status.txt not found on host" > /alerts/zujian_status.txt
    fi

    livez_check > /alerts/livez.txt
    pods_status > /alerts/pods_status.txt
    nodes_status > /alerts/nodes_status.txt
    get_k8s_nodes_alerts k8s >/alerts/k8s-nodes.txt

[root@master-1 alert]# cat deployment.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kube-monitor-alerts
  namespace: kube-monitor
spec:
  selector:
    matchLabels:
      app: kube-monitor-alerts
  replicas: 1
  template:
    metadata:
      labels:
        app: kube-monitor-alerts
    spec:
      serviceAccountName: prometheus-server
      tolerations:
      - key: node-role.kubernetes.io/control-plane
        effect: NoSchedule
      containers:
      - image: docker.io/nginx:1.24.0
        name: kube-monitor-alerts
        ports:
          - containerPort: 80
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
          - mountPath: /etc/nginx/conf.d
            name: kube-monitor-alerts-nginx-config
          - name: alerts-data
            mountPath: /usr/share/nginx/alerts
      - name: get-alert
        image: 10.0.31.201/k8s/livez-check:1.3
        command: ['sh', '-c', 'while true;do sleep 6;sh /scripts/get_alerts.sh;done']
        env:
          - name: MONITOR_PATH
            value: /alert
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
          - name: alerts-data
            mountPath: /alerts
          - name: kube-monitor-get-alerts
            mountPath: /scripts
          - name: zujian-volume
            mountPath: /root/zujian/zujian_status.txt
      initContainers:
      - name: init-html
        image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/busybox:latest
        command: ['sh', '-c', 'echo "ok" | tee /alerts/livez.txt /alerts/pods_status.txt /alerts/k8s-nodes.txt']
        volumeMounts:
          - name: alerts-data
            mountPath: /alerts
      volumes:
        - name: kube-monitor-alerts-nginx-config
          configMap:
            name: kube-monitor-alerts-nginx-config
        - name: kube-monitor-get-alerts
          configMap:
            name: kube-monitor-get-alerts
        - name: alerts-data
          emptyDir: {}
        - name: zujian-volume
          hostPath:
            path: /root/zujian/zujian_status.txt
            type: File
      nodeSelector:
        kubernetes.io/hostname: master-1
---
apiVersion: v1
kind: Service
metadata:
  name: kube-monitor-alerts
  namespace: kube-monitor
  labels:
    app: kube-monitor-alerts
spec:
  type: NodePort
  ports:
    - port: 80
      protocol: TCP
  selector:
    app: kube-monitor-alerts

mkdir  /root/zujian

vim zujian.sh
#!/bin/bash
  
zujian_status() {
echo "------------------------------------"
NODES=("10.0.41.81" "10.0.41.84" "10.0.41.85" "10.0.41.86")

MINIO_PORT=9000

HEALTH_PATH="/minio/health/ready"

for node in "${NODES[@]}"; do
    STATUS=$(curl -I http://"$node":"$MINIO_PORT""$HEALTH_PATH" 2>/dev/null | grep -i 'HTTP')

    if [[ "$STATUS" == *"200 OK"* ]]; then
        echo "$node-minio-healthy"
    else
        echo "$node-minio-Nothealthy"
    fi
done
echo "------------------------------------"
NODES=("10.0.41.84" "10.0.41.85" "10.0.41.86")

ETCD_PORT=3379
HEALTH_PATH="/health"

for node in "${NODES[@]}"; do
    RESPONSE=$(curl -s http://"$node":"$ETCD_PORT""$HEALTH_PATH")

    if echo "$RESPONSE" | grep -q '"health":"true"'; then
        echo "$node-etcd-healthy"
    else
        echo "$node-etcd-Nothealthy"
    fi
done
echo "------------------------------------"
ZOOKEEPER_NODES=("10.0.41.84" "10.0.41.85" "10.0.41.86")

ZOOKEEPER_PORT=2181

for node in "${ZOOKEEPER_NODES[@]}"; do
    RESPONSE=$(echo "ruok" | nc "$node" "$ZOOKEEPER_PORT" 2>/dev/null)

    if [[ "$RESPONSE" == *"imok"* ]]; then
        echo "$node-zookeeper-healthy"
    else
        echo "$node-zookeeper-Nothealthy"
    fi
done
echo "------------------------------------"
}
#zujian_status

    zujian_status > /root/zujian/zujian_status.txt

crontab -e
* * * * * /root/zujian/zujian.sh > /root/zujian/zujian_status.txt 2>&1 && sleep 5 && /root/zujian/zujian.sh > /root/zujian/zujian_status.txt 2>&1 && sleep 5 && /root/zujian/zujian.sh > /root/zujian/zujian_status.txt 2>&1 && sleep 5 && /root/zujian/zujian.sh > /root/zujian/zujian_status.txt 2>&1 && sleep 5 && /root/zujian/zujian.sh > /root/zujian/zujian_status.txt 2>&1

7、部署 blackbox

[root@master-1 blackbox]# cat blackbox-exporter.yml 
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/name: blackbox-exporter
  name: blackbox-exporter
  namespace: kube-monitor
spec:
  ports:
  - name: http
    port: 9115
    targetPort: http
  selector:
    app.kubernetes.io/name: blackbox-exporter
---
apiVersion: v1
data:
  config.yml: |-
    "modules":
      "http_2xx":
        "http":
          "preferred_ip_protocol": "ip4"
        "prober": "http"
      "http_post_2xx":
        "http":
          "method": "POST"
          "preferred_ip_protocol": "ip4"
        "prober": "http"
      "irc_banner":
        "prober": "tcp"
        "tcp":
          "preferred_ip_protocol": "ip4"
          "query_response":
          - "send": "NICK prober"
          - "send": "USER prober prober prober :prober"
          - "expect": "PING :([^ ]+)"
            "send": "PONG ${1}"
          - "expect": "^:[^ ]+ 001"
      "pop3s_banner":
        "prober": "tcp"
        "tcp":
          "preferred_ip_protocol": "ip4"
          "query_response":
          - "expect": "^+OK"
          "tls": true
          "tls_config":
            "insecure_skip_verify": false
      "ssh_banner":
        "prober": "tcp"
        "tcp":
          "preferred_ip_protocol": "ip4"
          "query_response":
          - "expect": "^SSH-2.0-"
      "tcp_connect":
        "prober": "tcp"
        "tcp":
          "preferred_ip_protocol": "ip4"
kind: ConfigMap
metadata:
  labels:
    app.kubernetes.io/name: blackbox-exporter
  name: blackbox-exporter-configuration
  namespace: kube-monitor
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/name: blackbox-exporter
  name: blackbox-exporter
  namespace: kube-monitor
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: blackbox-exporter
  template:
    metadata:
      labels:
        app.kubernetes.io/name: blackbox-exporter
    spec:
      containers:
      - args:
        - --config.file=/etc/blackbox_exporter/config.yml
        - --web.listen-address=:9115
        image: 10.0.31.201/k8s/prom/blackbox-exporter:v0.26.0
        name: blackbox-exporter
        ports:
        - containerPort: 9115
          name: http
        resources:
          limits:
            cpu: 20m
            memory: 40Mi
          requests:
            cpu: 10m
            memory: 20Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsGroup: 65534
          runAsNonRoot: true
          runAsUser: 65534
        volumeMounts:
        - mountPath: /etc/blackbox_exporter/
          name: config
          readOnly: true
      nodeSelector:
        kubernetes.io/os: linux
      volumes:
      - configMap:
          name: blackbox-exporter-configuration
        name: config