Helm/Prometheus

helm repo add prometheus https://prometheus-community.github.io/helm-charts
helm repo update && helm repo list
kubectl config get-contexts

Config

export KUBECONFIG="${HOME}/.kube/dev-kubeconfig.yaml"
export KUBECONFIG="${HOME}/.kube/gcp-kubeconfig.yaml"
export KUBECONFIG="${HOME}/.kube/config"

Install

kubectl create namespace prometheus
helm show values prometheus/prometheus|less
cat <<YML | helm install -n prometheus prometheus prometheus/prometheus -f -
server:
  ingress:
    enabled: true
    hosts:
    - prometheus.k8s.local
  tls:
  - secretName: prometheus.k8s.local.tls
      hosts:
      - prometheus.k8s.local
YML

ConfigMap

cat <<-'YML' | kubectl -n prometheus apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-server-conf
  namespace: prometheus
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s

    scrape_configs:
      - job_name: 'kubernetes-apiservers'

        kubernetes_sd_configs:
        - role: endpoints

        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https

      - job_name: 'kubernetes-nodes'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: node

        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}/proxy/metrics

      - job_name: 'metrics-server'
        kubernetes_sd_configs:
        - role: endpoints
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: kube-system;metrics-server;https
YML

kubectl -n prometheus get cm prometheus-server-conf -o yaml

Uninstall

helm uninstall -n prometheus prometheus
helm delete    -n prometheus prometheus

Playground

PROMETHEUS_PUSH_POD_NAME="$(kubectl -n prometheus get pod -l 'app.kubernetes.io/name=prometheus-pushgateway' -o json|jq -r '.items[0].metadata.name')" PROMETHEUS_STATE_POD_NAME="$(kubectl -n prometheus get pod -l 'app.kubernetes.io/name=kube-state-metrics' -o json|jq -r '.items[0].metadata.name')" PROMETHEUS_POD_NAME="$(kubectl -n prometheus get pod -l 'app.kubernetes.io/name=prometheus' -o json|jq -r '.items[0].metadata.name')"
helm upgrade --install -f values.yaml prometheus prometheus/prometheus helm upgrade --install prometheus prometheus/prometheus helm show values prometheus/prometheus|less	kubectl -n prometheus exec -it ${PROMETHEUS_POD_NAME} -- cat /etc/config/prometheus.yml kubectl -n prometheus exec -it ${PROMETHEUS_POD_NAME} -- ls -lah /etc/config/ kubectl -n prometheus exec -it ${PROMETHEUS_POD_NAME} -- /bin/sh
kubectl -n prometheus logs -f ${PROMETHEUS_POD_NAME} kubectl -n prometheus rollout restart deploy prometheus-server kubectl -n prometheus logs -f prometheus-server-hash	kubectl config --kubeconfig=${HOME}/.kube/aws-kubeconfig.yaml view --flatten kubectl config --kubeconfig=${HOME}/.kube/gcp-kubeconfig.yaml view --flatten kubectl config --kubeconfig=${HOME}/.kube/config view --flatten
kubectl -n prometheus get cm prometheus-server-conf -o yaml kubectl -n prometheus edit deploy prometheus-server -o yaml kubectl -n prometheus get deploy prometheus-server -o yaml	prometheus-prometheus-pushgateway.prometheus.svc.cluster.local:9091 prometheus-alertmanager.prometheus.svc.cluster.local prometheus-server.prometheus.svc.cluster.local
kubectl -n prometheus delete all --all kubectl -n prometheus delete ing --all kubectl -n prometheus delete sts --all	kubectl -n prometheus delete svc --all kubectl -n prometheus delete pvc --all kubectl -n prometheus delete pv --all

References

Helm » Pass YAML/JSON using stdin Helm » Prometheus » GitHub Helm » Prometheus Stack Helm » Cert Manager Helm » Prometheus Helm
Prometheus » Spacelift » Blog Prometheus » Client Libraries Prometheus » K8s » Grafana Prometheus » Alerting Rules Prometheus » Configuration Prometheus » Data Model Prometheus » Grafana Prometheus » Quering Prometheus » Storage Prometheus	Prometheus » Exporters
K8s » Configure Service Accounts for Pods K8s » Restart Pods With Kubectl K8s » Interactive Pod K8s » Restart Pods Docker » Makefile Kubernetes Minikube Kubectl CURL K8s	K8s » CSI Hostpath Driver