From 46903d618d1e3870968a1e08a391237fa31a23b3 Mon Sep 17 00:00:00 2001 From: Thomas Rouaux <33913696+trouaux@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:21:43 +0100 Subject: [PATCH 1/7] add ruler + ruler-svc templates --- thanos/charts/templates/ruler-svc.yaml | 33 +++++ thanos/charts/templates/ruler.yaml | 191 +++++++++++++++++++++++++ thanos/charts/values.yaml | 47 ++++++ 3 files changed, 271 insertions(+) create mode 100644 thanos/charts/templates/ruler-svc.yaml create mode 100644 thanos/charts/templates/ruler.yaml diff --git a/thanos/charts/templates/ruler-svc.yaml b/thanos/charts/templates/ruler-svc.yaml new file mode 100644 index 00000000..3872f6a6 --- /dev/null +++ b/thanos/charts/templates/ruler-svc.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +# SPDX-License-Identifier: Apache-2.0 + +{{ if .Values.ruler.enabled }} +apiVersion: v1 +kind: Service +metadata: + {{- if .Values.thanos.query.annotations }} + annotations: + {{ toYaml .Values.thanos.query.annotations | nindent 8 }} + {{- end }} + labels: + {{- include "plugin.labels" . | nindent 4 }} + {{- include "thanos.labels" . | nindent 4 }} + {{- if .Values.ruler.serviceLabels }} + {{ toYaml .Values.ruler.serviceLabels | nindent 4 }} + {{- end }} + name: {{ include "release.name" . }}-ruler +spec: + ports: + - name: grpc + port: 10901 + protocol: TCP + targetPort: grpc + - name: http + port: 10902 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/managed-by: {{ include "release.name" . }} + app.kubernetes.io/name: ruler + +{{- end }} diff --git a/thanos/charts/templates/ruler.yaml b/thanos/charts/templates/ruler.yaml new file mode 100644 index 00000000..3e1d036f --- /dev/null +++ b/thanos/charts/templates/ruler.yaml @@ -0,0 +1,191 @@ +{{- if .Values.ruler.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "release.name" . }}-ruler + labels: + app.kubernetes.io/name: {{ include "release.name" . }}-ruler + helm.sh/chart: {{ include "prometheus-thanos.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "plugin.labels" . | nindent 4 }} + {{- include "thanos.labels" . | nindent 4 }} + {{- if .Values.ruler.deploymentLabels }} + {{ toYaml .Values.ruler.deploymentLabels | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.ruler.replicaCount }} + updateStrategy: + type: {{ .Values.ruler.updateStrategy }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "release.name" . }}-ruler + app.kubernetes.io/instance: {{ .Release.Name }} + serviceName: {{ include "release.name" . }}-ruler + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "release.name" . }}-ruler + app.kubernetes.io/instance: {{ .Release.Name }} + prometheus-thanos-peer: "true" + {{- with .Values.ruler.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10902" + spec: + serviceAccount: {{ include "release.name" . }}-ruler + # {{- with .Values.ruler.imagePullSecrets }} + # imagePullSecrets: + # {{- toYaml . | nindent 8 }} + # {{- end }} + containers: + - name: {{ .Chart.Name }}-ruler + imagePullPolicy: {{ .Values.ruler.image.pullPolicy }} + image: "{{ .Values.thanos.image.repository }}:{{ .Values.thanos.image.tag | default .Chart.AppVersion }}" + args: + - rule + - --data-dir=/data + - --log.level={{ .Values.ruler.logLevel }} + - --eval-interval={{ .Values.ruler.evalInterval }} + - --rule-file={{ .Values.ruler.ruleFile }} + - --alertmanagers.url={{ .Values.ruler.alertmanagerUrl }} + - --query={{ include "release.name" . }}-querier:{{ .Values.service.querier.http.port }} + {{- range .Values.ruler.queries }} + - --query={{ . }} + {{- end }} + {{- if .Values.ruler.objStoreType }} + - | + --objstore.config=type: {{ .Values.ruler.objStoreType }} + config: + {{- toYaml .Values.ruler.objStoreConfig | nindent 14 }} + {{ else if .Values.ruler.objStoreConfigFile }} + - --objstore.config-file={{ .Values.ruler.objStoreConfigFile }} + {{- end }} + {{- range $key, $value := .Values.ruler.additionalFlags }} + - "--{{ $key }}{{if $value }}={{ $value }}{{end}}" + {{- end }} + - --label=ruler_cluster="{{ .Values.ruler.clusterName }}" + - --label=replica="$(POD_NAME)" + {{- if .Values.tracing.enabled }} + - | + --tracing.config=type: {{ .Values.tracing.type }} + config: + {{- toYaml .Values.tracing.config | nindent 14 }} + {{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - name: http + containerPort: 10902 + protocol: TCP + - name: grpc + containerPort: 10901 + protocol: TCP + resources: + {{- toYaml .Values.ruler.resources | nindent 12 }} + # securityContext: + # {{- toYaml .Values.ruler.containerSecurityContext | nindent 12 }} + volumeMounts: + - mountPath: /etc/thanos-ruler + name: config + - mountPath: /etc/thanos-ruler/external + name: external-config-volume + - mountPath: /data + name: storage-volume + # {{- with .Values.ruler.volumeMounts }} + # {{- toYaml . | nindent 10 }} + # {{- end }} + # {{- if .Values.ruler.sidecar.enabled }} + # - env: + # - name: LABEL + # value: {{ .Values.ruler.sidecar.watchLabel }} + # - name: FOLDER + # value: /etc/config + # - name: NAMESPACE + # value: ALL + # - name: REQ_URL + # value: http://127.0.0.1:10902/-/reload + # - name: REQ_METHOD + # value: POST + # imagePullPolicy: {{ .Values.ruler.sidecar.image.pullPolicy }} + # image: "{{ .Values.ruler.sidecar.image.repository }}:{{ .Values.ruler.sidecar.image.tag }}" + # name: k8s-configmap-watcher + # resources: + # limits: + # cpu: 20m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + # terminationMessagePath: /dev/termination-log + # terminationMessagePolicy: File + # volumeMounts: + # - mountPath: /etc/config + # name: external-config-volume + # {{- end }} + # {{- with .Values.ruler.nodeSelector }} + # nodeSelector: + # {{- toYaml . | nindent 8 }} + # {{- end }} + # {{- with .Values.ruler.affinity }} + # affinity: + # {{- toYaml . | nindent 8 }} + # {{- end }} + # {{- with .Values.ruler.tolerations }} + # tolerations: + # {{- toYaml . | nindent 6 }} + # {{- end }} + # {{- with .Values.ruler.podSecurityContext }} + # securityContext: + # {{- toYaml . | nindent 8 }} + # {{- end }} + volumes: + - name: external-config-volume + emptyDir: {} + - configMap: + name: {{ include "release.name" . }}-ruler + name: config + {{- with .Values.ruler.volumes }} + {{- toYaml . | nindent 6 }} + {{- end }} + # {{- if not .Values.ruler.persistentVolume.enabled }} + # - name: storage-volume + # emptyDir: {} + # {{- else if .Values.ruler.persistentVolume.existingClaim }} + # - name: storage-volume + # persistentVolumeClaim: + # claimName: {{ .Values.ruler.persistentVolume.existingClaim }} + # {{- end }} + # {{- if .Values.ruler.podNumericalPriorityEnabled }} + # priority: {{ .Values.ruler.podPriority }} + # {{- else if .Values.ruler.podPriorityClassName }} + # priorityClassName: {{ .Values.ruler.podPriorityClassName }} + # {{- end }} + # {{- if and .Values.ruler.persistentVolume.enabled (not .Values.ruler.persistentVolume.existingClaim) }} + # volumeClaimTemplates: + # - metadata: + # name: storage-volume + # {{- if .Values.ruler.persistentVolume.annotations }} + # annotations: + # {{- toYaml .Values.ruler.persistentVolume.annotations | nindent 10 }} + # {{- end }} + # spec: + # accessModes: + # {{- toYaml .Values.ruler.persistentVolume.accessModes | nindent 10 }} + # resources: + # requests: + # storage: "{{ .Values.ruler.persistentVolume.size }}" + # {{- if .Values.ruler.persistentVolume.storageClass }} + # {{- if (eq "-" .Values.ruler.persistentVolume.storageClass) }} + # storageClassName: "" + # {{- else }} + # storageClassName: "{{ .Values.ruler.persistentVolume.storageClass }}" + # {{- end }} + # {{- end }} + # {{- end }} +{{- end }} \ No newline at end of file diff --git a/thanos/charts/values.yaml b/thanos/charts/values.yaml index 493166ef..c4dcd7eb 100644 --- a/thanos/charts/values.yaml +++ b/thanos/charts/values.yaml @@ -2,11 +2,58 @@ # SPDX-License-Identifier: Apache-2.0 global: + clusterType: + region: ## Labels to apply to all resources ## commonLabels: {} +# Create RBAC resources. +rbac: + create: true + +# ServiceAccount to use for the Thanos. +# Note that a ServiceAccount with name `default` cannot be created. +# Instead the generated name will be used. +serviceAccount: + create: true + +ruler: + enabled: true + + annotations: + + deploymentLabels: {} + + serviceLabels: + + paused: + + logLevel: + + externalPrefix: + + evaluationInterval: + + externalClusterLabels: + enabled: + + alertmanagers: + hosts: + authentication: + enabled: + + queryEndpoints: + + resources: + + externalUrl: "https://" + + retention: true + image: + pullPolicy: "IfNotPresent" + thanos: image: repository: quay.io/thanos/thanos From 654183cc82bf81ef181ad747a22035f8aca0efbe Mon Sep 17 00:00:00 2001 From: Thomas Rouaux <33913696+trouaux@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:33:11 +0100 Subject: [PATCH 2/7] remove label helm.sh --- thanos/charts/templates/ruler.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/thanos/charts/templates/ruler.yaml b/thanos/charts/templates/ruler.yaml index 3e1d036f..f8699ee5 100644 --- a/thanos/charts/templates/ruler.yaml +++ b/thanos/charts/templates/ruler.yaml @@ -5,7 +5,6 @@ metadata: name: {{ include "release.name" . }}-ruler labels: app.kubernetes.io/name: {{ include "release.name" . }}-ruler - helm.sh/chart: {{ include "prometheus-thanos.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- include "plugin.labels" . | nindent 4 }} From 2525a23b57cdf463093849c6c72f91818eb6dcb9 Mon Sep 17 00:00:00 2001 From: Thomas Rouaux <33913696+trouaux@users.noreply.github.com> Date: Fri, 29 Nov 2024 12:19:54 +0100 Subject: [PATCH 3/7] lint clean up --- thanos/charts/templates/ruler.yaml | 96 +----------------------------- thanos/charts/values.yaml | 4 ++ 2 files changed, 5 insertions(+), 95 deletions(-) diff --git a/thanos/charts/templates/ruler.yaml b/thanos/charts/templates/ruler.yaml index f8699ee5..3f5f08a2 100644 --- a/thanos/charts/templates/ruler.yaml +++ b/thanos/charts/templates/ruler.yaml @@ -35,10 +35,6 @@ spec: prometheus.io/port: "10902" spec: serviceAccount: {{ include "release.name" . }}-ruler - # {{- with .Values.ruler.imagePullSecrets }} - # imagePullSecrets: - # {{- toYaml . | nindent 8 }} - # {{- end }} containers: - name: {{ .Chart.Name }}-ruler imagePullPolicy: {{ .Values.ruler.image.pullPolicy }} @@ -50,7 +46,7 @@ spec: - --eval-interval={{ .Values.ruler.evalInterval }} - --rule-file={{ .Values.ruler.ruleFile }} - --alertmanagers.url={{ .Values.ruler.alertmanagerUrl }} - - --query={{ include "release.name" . }}-querier:{{ .Values.service.querier.http.port }} + - --query={{ include "release.name" . }}-querier:10902 {{- range .Values.ruler.queries }} - --query={{ . }} {{- end }} @@ -67,12 +63,6 @@ spec: {{- end }} - --label=ruler_cluster="{{ .Values.ruler.clusterName }}" - --label=replica="$(POD_NAME)" - {{- if .Values.tracing.enabled }} - - | - --tracing.config=type: {{ .Values.tracing.type }} - config: - {{- toYaml .Values.tracing.config | nindent 14 }} - {{- end }} env: - name: POD_NAME valueFrom: @@ -87,8 +77,6 @@ spec: protocol: TCP resources: {{- toYaml .Values.ruler.resources | nindent 12 }} - # securityContext: - # {{- toYaml .Values.ruler.containerSecurityContext | nindent 12 }} volumeMounts: - mountPath: /etc/thanos-ruler name: config @@ -96,53 +84,6 @@ spec: name: external-config-volume - mountPath: /data name: storage-volume - # {{- with .Values.ruler.volumeMounts }} - # {{- toYaml . | nindent 10 }} - # {{- end }} - # {{- if .Values.ruler.sidecar.enabled }} - # - env: - # - name: LABEL - # value: {{ .Values.ruler.sidecar.watchLabel }} - # - name: FOLDER - # value: /etc/config - # - name: NAMESPACE - # value: ALL - # - name: REQ_URL - # value: http://127.0.0.1:10902/-/reload - # - name: REQ_METHOD - # value: POST - # imagePullPolicy: {{ .Values.ruler.sidecar.image.pullPolicy }} - # image: "{{ .Values.ruler.sidecar.image.repository }}:{{ .Values.ruler.sidecar.image.tag }}" - # name: k8s-configmap-watcher - # resources: - # limits: - # cpu: 20m - # memory: 64Mi - # requests: - # cpu: 10m - # memory: 32Mi - # terminationMessagePath: /dev/termination-log - # terminationMessagePolicy: File - # volumeMounts: - # - mountPath: /etc/config - # name: external-config-volume - # {{- end }} - # {{- with .Values.ruler.nodeSelector }} - # nodeSelector: - # {{- toYaml . | nindent 8 }} - # {{- end }} - # {{- with .Values.ruler.affinity }} - # affinity: - # {{- toYaml . | nindent 8 }} - # {{- end }} - # {{- with .Values.ruler.tolerations }} - # tolerations: - # {{- toYaml . | nindent 6 }} - # {{- end }} - # {{- with .Values.ruler.podSecurityContext }} - # securityContext: - # {{- toYaml . | nindent 8 }} - # {{- end }} volumes: - name: external-config-volume emptyDir: {} @@ -152,39 +93,4 @@ spec: {{- with .Values.ruler.volumes }} {{- toYaml . | nindent 6 }} {{- end }} - # {{- if not .Values.ruler.persistentVolume.enabled }} - # - name: storage-volume - # emptyDir: {} - # {{- else if .Values.ruler.persistentVolume.existingClaim }} - # - name: storage-volume - # persistentVolumeClaim: - # claimName: {{ .Values.ruler.persistentVolume.existingClaim }} - # {{- end }} - # {{- if .Values.ruler.podNumericalPriorityEnabled }} - # priority: {{ .Values.ruler.podPriority }} - # {{- else if .Values.ruler.podPriorityClassName }} - # priorityClassName: {{ .Values.ruler.podPriorityClassName }} - # {{- end }} - # {{- if and .Values.ruler.persistentVolume.enabled (not .Values.ruler.persistentVolume.existingClaim) }} - # volumeClaimTemplates: - # - metadata: - # name: storage-volume - # {{- if .Values.ruler.persistentVolume.annotations }} - # annotations: - # {{- toYaml .Values.ruler.persistentVolume.annotations | nindent 10 }} - # {{- end }} - # spec: - # accessModes: - # {{- toYaml .Values.ruler.persistentVolume.accessModes | nindent 10 }} - # resources: - # requests: - # storage: "{{ .Values.ruler.persistentVolume.size }}" - # {{- if .Values.ruler.persistentVolume.storageClass }} - # {{- if (eq "-" .Values.ruler.persistentVolume.storageClass) }} - # storageClassName: "" - # {{- else }} - # storageClassName: "{{ .Values.ruler.persistentVolume.storageClass }}" - # {{- end }} - # {{- end }} - # {{- end }} {{- end }} \ No newline at end of file diff --git a/thanos/charts/values.yaml b/thanos/charts/values.yaml index c4dcd7eb..e5790e05 100644 --- a/thanos/charts/values.yaml +++ b/thanos/charts/values.yaml @@ -46,6 +46,10 @@ ruler: queryEndpoints: + queries: + + additionalFlags: + resources: externalUrl: "https://" From 7b0adb2b3635616c24e9ab6e03bf58515396fbcc Mon Sep 17 00:00:00 2001 From: Thomas Rouaux <33913696+trouaux@users.noreply.github.com> Date: Fri, 29 Nov 2024 12:46:46 +0100 Subject: [PATCH 4/7] ruler bats testing + extra labels --- thanos/charts/templates/ruler.yaml | 3 +++ thanos/charts/templates/tests/test-thanos-config.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/thanos/charts/templates/ruler.yaml b/thanos/charts/templates/ruler.yaml index 3f5f08a2..e6e1b38a 100644 --- a/thanos/charts/templates/ruler.yaml +++ b/thanos/charts/templates/ruler.yaml @@ -30,6 +30,9 @@ spec: {{- with .Values.ruler.additionalLabels }} {{- toYaml . | nindent 8 }} {{- end }} + region: {{ .Values.global.region }} + cluster: {{ .Values.global.cluster }} + cluster_type: {{ .Values.global.clusterType }} annotations: prometheus.io/scrape: "true" prometheus.io/port: "10902" diff --git a/thanos/charts/templates/tests/test-thanos-config.yaml b/thanos/charts/templates/tests/test-thanos-config.yaml index 2ad58f1e..2e77c6d4 100644 --- a/thanos/charts/templates/tests/test-thanos-config.yaml +++ b/thanos/charts/templates/tests/test-thanos-config.yaml @@ -26,12 +26,15 @@ data: verify "there is 1 deployment named '{{ .Release.Name }}-compactor'" verify "there is 1 deployment named '{{ .Release.Name }}-query'" verify "there is 1 deployment named '{{ .Release.Name }}-store'" + verify "there is 1 statefulset named '{{ .Release.Name }}-ruler'" verify "there is 1 service named '{{ .Release.Name }}-compactor'" verify "there is 1 service named '{{ .Release.Name }}-query'" verify "there is 1 service named '{{ .Release.Name }}-store'" + verify "there is 1 service named '{{ .Release.Name }}-ruler'" try "at most 2 times every 5s to get pods named '{{ .Release.Name }}-compactor' and verify that '.status.phase' is 'running'" try "at most 4 times every 5s to get pods named '{{ .Release.Name }}-query' and verify that '.status.phase' is 'running'" try "at most 4 times every 5s to get pods named '{{ .Release.Name }}-store' and verify that '.status.phase' is 'running'" + try "at most 4 times every 5s to get pods named '{{ .Release.Name }}-ruler' and verify that '.status.phase' is 'running'" } @test "Verify successful creation and bound status of {{ .Release.Name }} persistent volume claims" { From 68c91dc64622018485db9e108173102d2eb7bd9d Mon Sep 17 00:00:00 2001 From: Thomas Rouaux <33913696+trouaux@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:03:42 +0100 Subject: [PATCH 5/7] re-org values + clean-up ruler template --- thanos/charts/templates/ruler.yaml | 60 ++++++++------------- thanos/charts/values.yaml | 84 ++++++++++++++++-------------- 2 files changed, 68 insertions(+), 76 deletions(-) diff --git a/thanos/charts/templates/ruler.yaml b/thanos/charts/templates/ruler.yaml index e6e1b38a..3e264a59 100644 --- a/thanos/charts/templates/ruler.yaml +++ b/thanos/charts/templates/ruler.yaml @@ -1,21 +1,21 @@ -{{- if .Values.ruler.enabled -}} +{{- if .Values.thanos.ruler.enabled -}} apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "release.name" . }}-ruler labels: - app.kubernetes.io/name: {{ include "release.name" . }}-ruler - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} {{- include "plugin.labels" . | nindent 4 }} {{- include "thanos.labels" . | nindent 4 }} - {{- if .Values.ruler.deploymentLabels }} - {{ toYaml .Values.ruler.deploymentLabels | nindent 4 }} + {{- if .Values.thanos.ruler.deploymentLabels }} + {{ toYaml .Values.thanos.ruler.deploymentLabels | nindent 4 }} {{- end }} + name: {{ include "release.name" . }}-ruler spec: - replicas: {{ .Values.ruler.replicaCount }} - updateStrategy: - type: {{ .Values.ruler.updateStrategy }} + replicas: {{ default 1 .Values.thanos.ruler.replicas }} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate selector: matchLabels: app.kubernetes.io/name: {{ include "release.name" . }}-ruler @@ -31,7 +31,6 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} region: {{ .Values.global.region }} - cluster: {{ .Values.global.cluster }} cluster_type: {{ .Values.global.clusterType }} annotations: prometheus.io/scrape: "true" @@ -40,7 +39,7 @@ spec: serviceAccount: {{ include "release.name" . }}-ruler containers: - name: {{ .Chart.Name }}-ruler - imagePullPolicy: {{ .Values.ruler.image.pullPolicy }} + imagePullPolicy: {{ default "IfNotPresent" .Values.thanos.image.pullPolicy }} image: "{{ .Values.thanos.image.repository }}:{{ .Values.thanos.image.tag | default .Chart.AppVersion }}" args: - rule @@ -49,28 +48,10 @@ spec: - --eval-interval={{ .Values.ruler.evalInterval }} - --rule-file={{ .Values.ruler.ruleFile }} - --alertmanagers.url={{ .Values.ruler.alertmanagerUrl }} - - --query={{ include "release.name" . }}-querier:10902 - {{- range .Values.ruler.queries }} - - --query={{ . }} - {{- end }} - {{- if .Values.ruler.objStoreType }} - - | - --objstore.config=type: {{ .Values.ruler.objStoreType }} - config: - {{- toYaml .Values.ruler.objStoreConfig | nindent 14 }} - {{ else if .Values.ruler.objStoreConfigFile }} - - --objstore.config-file={{ .Values.ruler.objStoreConfigFile }} - {{- end }} - {{- range $key, $value := .Values.ruler.additionalFlags }} - - "--{{ $key }}{{if $value }}={{ $value }}{{end}}" - {{- end }} - - --label=ruler_cluster="{{ .Values.ruler.clusterName }}" - - --label=replica="$(POD_NAME)" - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + - --grpc-address={{ default "0.0.0.0:10901" .Values.thanos.grpcAddress }} + - --http-address={{ default "0.0.0.0:10902" .Values.thanos.httpAddress }} + - --query={{ include "release.name" . }}-query:10902 + - --objstore.config-file=/etc/config/thanos.yaml ports: - name: http containerPort: 10902 @@ -78,8 +59,6 @@ spec: - name: grpc containerPort: 10901 protocol: TCP - resources: - {{- toYaml .Values.ruler.resources | nindent 12 }} volumeMounts: - mountPath: /etc/thanos-ruler name: config @@ -87,13 +66,20 @@ spec: name: external-config-volume - mountPath: /data name: storage-volume + - mountPath: /etc/config/ + name: objectstore-secret + readOnly: true volumes: + - name: objectstore-secret + secret: + defaultMode: 420 + secretName: {{ include "release.name" . }}-metrics-objectstore - name: external-config-volume emptyDir: {} - configMap: name: {{ include "release.name" . }}-ruler name: config - {{- with .Values.ruler.volumes }} + {{- with .Values.thanos.ruler.volumes }} {{- toYaml . | nindent 6 }} {{- end }} {{- end }} \ No newline at end of file diff --git a/thanos/charts/values.yaml b/thanos/charts/values.yaml index e5790e05..1a5788a9 100644 --- a/thanos/charts/values.yaml +++ b/thanos/charts/values.yaml @@ -19,45 +19,6 @@ rbac: serviceAccount: create: true -ruler: - enabled: true - - annotations: - - deploymentLabels: {} - - serviceLabels: - - paused: - - logLevel: - - externalPrefix: - - evaluationInterval: - - externalClusterLabels: - enabled: - - alertmanagers: - hosts: - authentication: - enabled: - - queryEndpoints: - - queries: - - additionalFlags: - - resources: - - externalUrl: "https://" - - retention: true - image: - pullPolicy: "IfNotPresent" - thanos: image: repository: quay.io/thanos/thanos @@ -112,6 +73,12 @@ thanos: annotations: + chunkPoolSize: + + indexCacheSize: + + logLevel: + compactor: # if this is a standalone Thanos Query, we wouldn't want a compactor enabled: true @@ -153,6 +120,45 @@ thanos: # GiB size: labels: [] + + ruler: + enabled: true + + replicas: + + annotations: + + deploymentLabels: {} + + serviceLabels: + + paused: + + logLevel: + + externalPrefix: + + evaluationInterval: + + externalClusterLabels: + enabled: + + alertmanagers: + hosts: + authentication: + enabled: + + queryEndpoints: + + queries: + + additionalFlags: + + additionalLabels: + + externalUrl: "https://" + + retention: true testFramework: enabled: true From a0479183d8202e203d2027068388163b4ec63852 Mon Sep 17 00:00:00 2001 From: Thomas Rouaux <33913696+trouaux@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:17:04 +0100 Subject: [PATCH 6/7] values naming fix --- thanos/charts/templates/ruler-svc.yaml | 6 +++--- thanos/charts/templates/ruler.yaml | 15 +++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/thanos/charts/templates/ruler-svc.yaml b/thanos/charts/templates/ruler-svc.yaml index 3872f6a6..ecb0fd5e 100644 --- a/thanos/charts/templates/ruler-svc.yaml +++ b/thanos/charts/templates/ruler-svc.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors # SPDX-License-Identifier: Apache-2.0 -{{ if .Values.ruler.enabled }} +{{ if .Values.thanos.ruler.enabled }} apiVersion: v1 kind: Service metadata: @@ -12,8 +12,8 @@ metadata: labels: {{- include "plugin.labels" . | nindent 4 }} {{- include "thanos.labels" . | nindent 4 }} - {{- if .Values.ruler.serviceLabels }} - {{ toYaml .Values.ruler.serviceLabels | nindent 4 }} + {{- if .Values.thanos.ruler.serviceLabels }} + {{ toYaml .Values.thanos.ruler.serviceLabels | nindent 4 }} {{- end }} name: {{ include "release.name" . }}-ruler spec: diff --git a/thanos/charts/templates/ruler.yaml b/thanos/charts/templates/ruler.yaml index 3e264a59..29a442c4 100644 --- a/thanos/charts/templates/ruler.yaml +++ b/thanos/charts/templates/ruler.yaml @@ -1,4 +1,4 @@ -{{- if .Values.thanos.ruler.enabled -}} +# {{ if .Values.thanos.ruler.enabled }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -27,7 +27,7 @@ spec: app.kubernetes.io/name: {{ include "release.name" . }}-ruler app.kubernetes.io/instance: {{ .Release.Name }} prometheus-thanos-peer: "true" - {{- with .Values.ruler.additionalLabels }} + {{- with .Values.thanos.ruler.additionalLabels }} {{- toYaml . | nindent 8 }} {{- end }} region: {{ .Values.global.region }} @@ -44,14 +44,13 @@ spec: args: - rule - --data-dir=/data - - --log.level={{ .Values.ruler.logLevel }} - - --eval-interval={{ .Values.ruler.evalInterval }} - - --rule-file={{ .Values.ruler.ruleFile }} - - --alertmanagers.url={{ .Values.ruler.alertmanagerUrl }} + - --log.level={{ .Values.thanos.ruler.logLevel }} + - --eval-interval={{ .Values.thanos.ruler.evalInterval }} + - --rule-file={{ .Values.thanos.ruler.ruleFile }} + - --alertmanagers.url={{ .Values.thanos.ruler.alertmanagerUrl }} - --grpc-address={{ default "0.0.0.0:10901" .Values.thanos.grpcAddress }} - --http-address={{ default "0.0.0.0:10902" .Values.thanos.httpAddress }} - --query={{ include "release.name" . }}-query:10902 - - --objstore.config-file=/etc/config/thanos.yaml ports: - name: http containerPort: 10902 @@ -82,4 +81,4 @@ spec: {{- with .Values.thanos.ruler.volumes }} {{- toYaml . | nindent 6 }} {{- end }} -{{- end }} \ No newline at end of file +# {{- end }} \ No newline at end of file From 5e2d85263aa46b2f9ed000ffb2c9c1313badc348 Mon Sep 17 00:00:00 2001 From: Thomas Rouaux <33913696+trouaux@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:03:40 +0100 Subject: [PATCH 7/7] update container args + volume mounts --- thanos/charts/templates/ruler.yaml | 62 ++++++++++++++++++------------ thanos/charts/values.yaml | 4 +- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/thanos/charts/templates/ruler.yaml b/thanos/charts/templates/ruler.yaml index 29a442c4..ac139958 100644 --- a/thanos/charts/templates/ruler.yaml +++ b/thanos/charts/templates/ruler.yaml @@ -11,7 +11,7 @@ metadata: name: {{ include "release.name" . }}-ruler spec: replicas: {{ default 1 .Values.thanos.ruler.replicas }} - strategy: + updateStrategy: rollingUpdate: maxSurge: 25% maxUnavailable: 25% @@ -43,14 +43,22 @@ spec: image: "{{ .Values.thanos.image.repository }}:{{ .Values.thanos.image.tag | default .Chart.AppVersion }}" args: - rule - - --data-dir=/data + - --data-dir=/thanos/data - --log.level={{ .Values.thanos.ruler.logLevel }} - --eval-interval={{ .Values.thanos.ruler.evalInterval }} - - --rule-file={{ .Values.thanos.ruler.ruleFile }} - - --alertmanagers.url={{ .Values.thanos.ruler.alertmanagerUrl }} - - --grpc-address={{ default "0.0.0.0:10901" .Values.thanos.grpcAddress }} - - --http-address={{ default "0.0.0.0:10902" .Values.thanos.httpAddress }} + - --tsdb.retention=24h + - --rule-file=/etc/thanos/rules/*/*.yaml - --query={{ include "release.name" . }}-query:10902 + - --label=thanos_ruler_replica="$(POD_NAME)" + - --label=cluster="qa-de-1" + - --label=cluster_type="metal" + - --label=region="qa-de-1" + - --alert.label-drop=thanos_ruler_replica + - --alertmanagers.config-file=/etc/thanos/config/alertmanager-config/alertManagerConfig.yaml + - --web.external-prefix=/ruler + - --web.route-prefix=/ruler + - --alert.query-url=https://metrics.metal.qa-de-1.cloud.sap + - --http.config=/etc/thanos/web_config/web-config.yaml ports: - name: http containerPort: 10902 @@ -59,26 +67,32 @@ spec: containerPort: 10901 protocol: TCP volumeMounts: - - mountPath: /etc/thanos-ruler - name: config - - mountPath: /etc/thanos-ruler/external - name: external-config-volume - - mountPath: /data - name: storage-volume - - mountPath: /etc/config/ - name: objectstore-secret - readOnly: true + - mountPath: /etc/thanos/rules/{{ include "release.name" . }}-rulefiles + name: {{ include "release.name" . }}-rulefiles + - mountPath: /etc/thanos/config/alertmanager-config + name: alertmanager-config + - mountPath: /etc/thanos/web_config/web-config.yaml + name: web-config + - mountPath: /etc/thanos/secrets/thanos-metal-ruler-alertmanager-sso-cert + name: {{ include "release.name" . }}-alertmanager-sso-cert + - mountPath: /thanos/data + name: data volumes: - - name: objectstore-secret + - name: alertmanager-config secret: defaultMode: 420 - secretName: {{ include "release.name" . }}-metrics-objectstore - - name: external-config-volume + secretName: {{ include "release.name" . }}-alertmanager-config + - name: web-config + secret: + defaultMode: 420 + secretName: {{ include "release.name" . }}-web-config + - name: {{ include "release.name" . }}-alertmanager-sso-cert + secret: + defaultMode: 420 + secretName: {{ include "release.name" . }}-ruler-alertmanager-sso-cert + - name: {{ include "release.name" . }}-rulefiles #what if multiple replicas ? + configMap: + name: {{ include "release.name" . }}-rulerrulefiles + - name: data emptyDir: {} - - configMap: - name: {{ include "release.name" . }}-ruler - name: config - {{- with .Values.thanos.ruler.volumes }} - {{- toYaml . | nindent 6 }} - {{- end }} # {{- end }} \ No newline at end of file diff --git a/thanos/charts/values.yaml b/thanos/charts/values.yaml index 1a5788a9..6ecf798e 100644 --- a/thanos/charts/values.yaml +++ b/thanos/charts/values.yaml @@ -123,7 +123,7 @@ thanos: ruler: enabled: true - + replicas: annotations: @@ -138,7 +138,7 @@ thanos: externalPrefix: - evaluationInterval: + evalInterval: 30s externalClusterLabels: enabled: