From f81a8efa0610b1872953ed3399b025752898f989 Mon Sep 17 00:00:00 2001 From: Sergei Bondar Date: Wed, 17 Jan 2024 01:43:49 +0400 Subject: [PATCH] update neon-proxy.sh monitor install --- README.MD | 8 ++ config.ini.sample | 7 ++ helmfile_config/helmfile.yaml | 86 +++++++++++++++++++ helmfile_config/values.yaml | 35 ++++++++ monitoring/grafana/values.yaml | 4 +- monitoring/loki/values.yaml | 122 ++------------------------ monitoring/loki/values_old.yaml | 115 +++++++++++++++++++++++++ neon-proxy.sh | 146 +++++++++++++++++++++++--------- 8 files changed, 367 insertions(+), 156 deletions(-) create mode 100644 helmfile_config/helmfile.yaml create mode 100644 helmfile_config/values.yaml create mode 100644 monitoring/loki/values_old.yaml diff --git a/README.MD b/README.MD index 5586c41..ae16911 100644 --- a/README.MD +++ b/README.MD @@ -112,3 +112,11 @@ If you need monitoring part PYTH_MAPPING_ACCOUNT PP_SOLANA_URL +## Helmfile setup + + mkdir helmfile && cd helmfile + wget https://github.com/helmfile/helmfile/releases/download/v0.160.0/helmfile_0.160.0_linux_amd64.tar.gz + tar -xzvf helmfile_0.160.0_linux_amd64.tar.gz + chmod +x helmfile + sudo mv helmfile /usr/bin/helmfile + helm plugin install https://github.com/databus23/helm-diff diff --git a/config.ini.sample b/config.ini.sample index 66280e2..f17c0bf 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -146,3 +146,10 @@ GRAFANA_ADMIN_USER="admin" GRAFANA_ADMIN_PASSWD="grafana" GRAFANA_INGRESS_ENABLED="true" GRAFANA_INGRESS_PATH="/grafana" + + +################ +## Loki +################ +LOKI_STORAGE_SIZE="10Gi" +LOKI_STORAGE_CLASS="standard" \ No newline at end of file diff --git a/helmfile_config/helmfile.yaml b/helmfile_config/helmfile.yaml new file mode 100644 index 0000000..6c90bd8 --- /dev/null +++ b/helmfile_config/helmfile.yaml @@ -0,0 +1,86 @@ +repositories: + - name: metrics-server + url: https://kubernetes-sigs.github.io/metrics-server/ + - name: grafana + url: https://grafana.github.io/helm-charts + - name: prometheus-community + url: https://prometheus-community.github.io/helm-charts + + +environments: + default: + values: + - ./values.yaml + - namespace: + monitoring: 'neon-proxy' + +--- +releases: + - name: metrics-server + chart: metrics-server/metrics-server + namespace: kube-system + atomic: true + installed: {{ .Values.metrics_server.installed }} + set: + - name: args[0] + value: --kubelet-insecure-tls + + - name: kube-state-metrics + chart: prometheus-community/kube-state-metrics + namespace: {{ .Values.namespace.monitoring }} + atomic: true + version: 5.15.3 + installed: {{ .Values.prometheus.installed }} + + - name: prometheus + chart: prometheus-community/prometheus + namespace: {{ .Values.namespace.monitoring }} + atomic: true + version: 25.8.2 + installed: {{ .Values.prometheus.installed }} + values: + - ../monitoring/prometheus/values.yaml + set: + - name: extraScrapeConfigs + file: '../monitoring/prometheus/extraScrapeConfigs.yaml' +{{- range $key, $value := .Values.prometheus }} + - name: {{ $key }} + value: {{ $value }} +{{- end }} + + - name: loki + chart: grafana/loki + namespace: {{ .Values.namespace.monitoring }} + atomic: true + version: 5.41.6 + installed: {{ .Values.loki.installed }} + values: + - ../monitoring/loki/values.yaml + set: +{{- range $key, $value := .Values.loki }} + - name: {{ $key }} + value: {{ $value }} +{{- end }} + + - name: promtail + chart: grafana/promtail + namespace: {{ .Values.namespace.monitoring }} + atomic: true + version: 6.15.3 + installed: {{ .Values.promtail.installed }} + + - name: grafana + chart: grafana/grafana + namespace: {{ .Values.namespace.monitoring }} + atomic: true + version: 7.1.0 + installed: {{ .Values.grafana.installed }} + values: + - ../monitoring/grafana/values.yaml + set: + - name: namespace + value: {{ .Values.namespace.monitoring }} +{{- range $key, $value := .Values.grafana }} + - name: {{ $key }} + value: {{ $value }} +{{- end }} \ No newline at end of file diff --git a/helmfile_config/values.yaml b/helmfile_config/values.yaml new file mode 100644 index 0000000..632bc24 --- /dev/null +++ b/helmfile_config/values.yaml @@ -0,0 +1,35 @@ +namespace: + monitoring: 'neon-proxy' + system: 'neon-proxy' + +metrics_server: + installed: false + +prometheus: + installed: false + server.persistentVolume.storageClass: 'standard' + server.persistentVolume.size: '3Gi' + alertmanager.persistence.storageClass: 'standard' + alertmanager.persistence.size: '3Gi' + server.ingress.host: '' + server.ingress.className: 'nginx' + server.ingress.path: '/prometheus' + +loki: + installed: true + singleBinary.persistence.storageClass: 'standard' + singleBinary.persistence.size: '3Gi' + +promtail: + installed: false + +grafana: + installed: false + persistence.storageClassName: 'standard' + persistence.size: '3Gi' + ingress.enabled: 'true' + ingress.host: '' + ingress.className: 'nginx' + ingress.path: '/grafana' + adminUser: 'admin' + adminPassword: 'grafana' \ No newline at end of file diff --git a/monitoring/grafana/values.yaml b/monitoring/grafana/values.yaml index a1a5994..a8c1bd0 100644 --- a/monitoring/grafana/values.yaml +++ b/monitoring/grafana/values.yaml @@ -22,12 +22,12 @@ datasources: datasources: - name: Prometheus type: prometheus - url: http://prometheus-server.neon-proxy.svc.cluster.local + url: http://prometheus-server.{{ .Values.namespace }}.svc.cluster.local access: proxy isDefault: true - name: Loki type: loki - url: http://loki.neon-proxy.svc.cluster.local:3100 + url: http://loki.{{ .Values.namespace }}.svc.cluster.local:3100 access: proxy isDefault: false diff --git a/monitoring/loki/values.yaml b/monitoring/loki/values.yaml index 328edae..596610d 100644 --- a/monitoring/loki/values.yaml +++ b/monitoring/loki/values.yaml @@ -1,115 +1,9 @@ -test_pod: - enabled: true - image: bats/bats:1.8.2 - pullPolicy: IfNotPresent - +#https://github.com/grafana/loki/blob/main/production/helm/loki/values.yaml loki: - enabled: true - isDefault: true - url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }} - readinessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - livenessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - datasource: - jsonData: "{}" - uid: "" - - -promtail: - enabled: true - config: - snippets: - pipelineStages: - - cri: {} - - docker: {} - logLevel: info - serverPort: 3101 - clients: - - url: http://{{ .Release.Name }}:3100/loki/api/v1/push - -fluent-bit: - enabled: false - -grafana: - enabled: false - sidecar: - datasources: - label: "" - labelValue: "" - enabled: true - maxLines: 1000 - image: - tag: 8.3.5 - -prometheus: - enabled: false - isDefault: false - url: http://{{ include "prometheus.fullname" .}}:{{ .Values.prometheus.server.service.servicePort }}{{ .Values.prometheus.server.prefixURL }} - datasource: - jsonData: "{}" - -filebeat: - enabled: false - filebeatConfig: - filebeat.yml: | - # logging.level: debug - filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" - output.logstash: - hosts: ["logstash-loki:5044"] - -logstash: - enabled: false - image: grafana/logstash-output-loki - imageTag: 1.0.1 - filters: - main: |- - filter { - if [kubernetes] { - mutate { - add_field => { - "container_name" => "%{[kubernetes][container][name]}" - "namespace" => "%{[kubernetes][namespace]}" - "pod" => "%{[kubernetes][pod][name]}" - } - replace => { "host" => "%{[kubernetes][node][name]}"} - } - } - mutate { - remove_field => ["tags"] - } - } - outputs: - main: |- - output { - loki { - url => "http://loki:3100/loki/api/v1/push" - #username => "test" - #password => "test" - } - # stdout { codec => rubydebug } - } - -# proxy is currently only used by loki test pod -# Note: If http_proxy/https_proxy are set, then no_proxy should include the -# loki service name, so that tests are able to communicate with the loki -# service. -proxy: - http_proxy: "" - https_proxy: "" - no_proxy: "" + auth_enabled: false + commonConfig: + replication_factor: 1 + storage: + type: 'filesystem' +singleBinary: + replicas: 1 \ No newline at end of file diff --git a/monitoring/loki/values_old.yaml b/monitoring/loki/values_old.yaml new file mode 100644 index 0000000..328edae --- /dev/null +++ b/monitoring/loki/values_old.yaml @@ -0,0 +1,115 @@ +test_pod: + enabled: true + image: bats/bats:1.8.2 + pullPolicy: IfNotPresent + +loki: + enabled: true + isDefault: true + url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }} + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + datasource: + jsonData: "{}" + uid: "" + + +promtail: + enabled: true + config: + snippets: + pipelineStages: + - cri: {} + - docker: {} + logLevel: info + serverPort: 3101 + clients: + - url: http://{{ .Release.Name }}:3100/loki/api/v1/push + +fluent-bit: + enabled: false + +grafana: + enabled: false + sidecar: + datasources: + label: "" + labelValue: "" + enabled: true + maxLines: 1000 + image: + tag: 8.3.5 + +prometheus: + enabled: false + isDefault: false + url: http://{{ include "prometheus.fullname" .}}:{{ .Values.prometheus.server.service.servicePort }}{{ .Values.prometheus.server.prefixURL }} + datasource: + jsonData: "{}" + +filebeat: + enabled: false + filebeatConfig: + filebeat.yml: | + # logging.level: debug + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" + output.logstash: + hosts: ["logstash-loki:5044"] + +logstash: + enabled: false + image: grafana/logstash-output-loki + imageTag: 1.0.1 + filters: + main: |- + filter { + if [kubernetes] { + mutate { + add_field => { + "container_name" => "%{[kubernetes][container][name]}" + "namespace" => "%{[kubernetes][namespace]}" + "pod" => "%{[kubernetes][pod][name]}" + } + replace => { "host" => "%{[kubernetes][node][name]}"} + } + } + mutate { + remove_field => ["tags"] + } + } + outputs: + main: |- + output { + loki { + url => "http://loki:3100/loki/api/v1/push" + #username => "test" + #password => "test" + } + # stdout { codec => rubydebug } + } + +# proxy is currently only used by loki test pod +# Note: If http_proxy/https_proxy are set, then no_proxy should include the +# loki service name, so that tests are able to communicate with the loki +# service. +proxy: + http_proxy: "" + https_proxy: "" + no_proxy: "" diff --git a/neon-proxy.sh b/neon-proxy.sh index eb67b7c..b95b858 100755 --- a/neon-proxy.sh +++ b/neon-proxy.sh @@ -14,6 +14,10 @@ VAR_FILE="config.ini" echo "ERROR: Helm not installed" exit 1 } +[ "$(which helmfile)" ] || { + echo "ERROR: Helmfile not installed" + exit 1 +} [ "$(which jq)" ] || { echo "ERROR: jq not installed" @@ -417,50 +421,112 @@ kubectl -n ${VAULT_NAMESPACE} exec vault-0 -- /bin/sh -c "echo '$INDEXER_ENV' | } ## 4. Monitoring -[[ $MONITORING_ENABLED != "true" ]] || { - [[ $PROMETHEUS_ENABLED != "true" ]] || { - echo "Installing prometheus" - helm upgrade --install kube-state-metrics prometheus-community/kube-state-metrics --namespace=$MONITORING_NAMESPACE 1>/dev/null - kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml 1>/dev/null - helm upgrade --install prometheus prometheus-community/prometheus \ - -f monitoring/prometheus/values.yaml \ - --namespace=$MONITORING_NAMESPACE \ - --history-max 3 \ - --set server.persistentVolume.storageClass=$PROMETHEUS_STORAGE_CLASS \ - --set server.persistentVolume.size=$PROMETHEUS_STORAGE_SIZE \ - --set alertmanager.persistence.storageClass=$PROMETHEUS_STORAGE_CLASS \ - --set alertmanager.persistence.size=$PROMETHEUS_STORAGE_SIZE \ - --set server.ingress.host=$PROXY_HOST \ - --set server.ingress.className=$INGRESS_CLASS \ - --set server.ingress.path=$PROMETHEUS_INGRESS_PATH \ - --set-file extraScrapeConfigs=monitoring/prometheus/extraScrapeConfigs.yaml 1>/dev/null - } - [[ $LOKI_ENABLED != "true" ]] || { - echo "Installing Loki..." - helm upgrade --install loki grafana/loki-stack \ - -f monitoring/loki/values.yaml \ - --namespace=$MONITORING_NAMESPACE \ - --history-max 3 1>/dev/null - } - [[ $GRAFANA_ENABLED != "true" ]] || { - echo "Installing Grafana..." - helm upgrade --install grafana grafana/grafana \ - -f monitoring/grafana/values.yaml \ - --namespace=$MONITORING_NAMESPACE \ - --history-max 3 \ - --set persistence.storageClassName=$GRAFANA_STORAGE_CLASS \ - --set persistence.size=$GRAFANA_STORAGE_SIZE \ - --set adminUser=$GRAFANA_ADMIN_USER \ - --set ingress.enabled=$GRAFANA_INGRESS_ENABLED \ - --set ingress.host=$PROXY_HOST \ - --set ingress.className=$INGRESS_CLASS \ - --set ingress.path=$GRAFANA_INGRESS_PATH \ - --set adminPassword=$GRAFANA_ADMIN_PASSWD 1>/dev/null - } +[[ $MONITORING_ENABLED != "true" ]] || { + +#Install helm-diff plugin +if helm plugin list | grep -q "diff"; then + echo "Helm diff plugin is already installed." +else + # Install helm-diff plugin + helm plugin install https://github.com/databus23/helm-diff + echo "Helm diff plugin has been installed." +fi + + +# Prepare values helmfile +values_yaml=$(cat </dev/null +# kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml 1>/dev/null +# helm upgrade --install prometheus prometheus-community/prometheus \ +# -f monitoring/prometheus/values.yaml \ +# --namespace=$MONITORING_NAMESPACE \ +# --history-max 3 \ +# --set server.persistentVolume.storageClass=$PROMETHEUS_STORAGE_CLASS \ +# --set server.persistentVolume.size=$PROMETHEUS_STORAGE_SIZE \ +# --set alertmanager.persistence.storageClass=$PROMETHEUS_STORAGE_CLASS \ +# --set alertmanager.persistence.size=$PROMETHEUS_STORAGE_SIZE \ +# --set server.ingress.host=$PROXY_HOST \ +# --set server.ingress.className=$INGRESS_CLASS \ +# --set server.ingress.path=$PROMETHEUS_INGRESS_PATH \ +# --set-file extraScrapeConfigs=monitoring/prometheus/extraScrapeConfigs.yaml 1>/dev/null +# } + +# [[ $LOKI_ENABLED != "true" ]] || { +# echo "Installing Loki..." +# helm upgrade --install loki grafana/loki-stack \ +# -f monitoring/loki/values.yaml \ +# --namespace=$MONITORING_NAMESPACE \ +# --history-max 3 1>/dev/null +# } + +# [[ $GRAFANA_ENABLED != "true" ]] || { +# echo "Installing Grafana..." +# helm upgrade --install grafana grafana/grafana \ +# -f monitoring/grafana/values.yaml \ +# --namespace=$MONITORING_NAMESPACE \ +# --history-max 3 \ +# --set persistence.storageClassName=$GRAFANA_STORAGE_CLASS \ +# --set persistence.size=$GRAFANA_STORAGE_SIZE \ +# --set adminUser=$GRAFANA_ADMIN_USER \ +# --set ingress.enabled=$GRAFANA_INGRESS_ENABLED \ +# --set ingress.host=$PROXY_HOST \ +# --set ingress.className=$INGRESS_CLASS \ +# --set ingress.path=$GRAFANA_INGRESS_PATH \ +# --set adminPassword=$GRAFANA_ADMIN_PASSWD 1>/dev/null +# } +# } + [ $VAULT_TYPE == "dev" ] || [ ! $FIRST_RUN ] || echo -e "\n###################\nWARNING: Please copy and keep $VAULT_KEYS_FILE in safe place!\n###################\n"