Add blooms support to our distributed helm chart (#12434)

grafana · Apr 5, 2024 · 91754f6 · 91754f6
1 parent 2327984
commit 91754f6
Show file tree

Hide file tree

Showing 9 changed files with 1,044 additions and 2 deletions.
diff --git a/production/helm/loki/ci/distributed-values.yaml b/production/helm/loki/ci/distributed-values.yaml
@@ -4,6 +4,7 @@ loki:
     replication_factor: 1
   image:
     tag: "k195-51c54ad"
+  useBloomFilters: false
 deploymentMode: Distributed
 backend:
   replicas: 0
@@ -25,5 +26,9 @@ compactor:
   replicas: 1
 indexGateway:
   replicas: 1
+bloomCompactor:
+  replicas: 0
+bloomGateway:
+  replicas: 0
 minio:
-  enabled: true
+  enabled: true
diff --git a/production/helm/loki/templates/bloomfilters/_helpers-bloom-compactor.tpl b/production/helm/loki/templates/bloomfilters/_helpers-bloom-compactor.tpl
@@ -0,0 +1,58 @@
+{{/*
+bloom compactor fullname
+*/}}
+{{- define "loki.bloomCompactorFullname" -}}
+{{ include "loki.fullname" . }}-bloom-compactor
+{{- end }}
+
+{{/*
+bloom compactor common labels
+*/}}
+{{- define "loki.bloomCompactorLabels" -}}
+{{ include "loki.labels" . }}
+app.kubernetes.io/component: bloom-compactor
+{{- end }}
+
+{{/*
+bloom compactor selector labels
+*/}}
+{{- define "loki.bloomCompactorSelectorLabels" -}}
+{{ include "loki.selectorLabels" . }}
+app.kubernetes.io/component: bloom-compactor
+{{- end }}
+
+{{/*
+bloom compactor readinessProbe
+*/}}
+{{- define "loki.bloomCompactor.readinessProbe" -}}
+{{- with .Values.bloomCompactor.readinessProbe }}
+readinessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- else }}
+{{- with .Values.loki.readinessProbe }}
+readinessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+bloom compactor priority class name
+*/}}
+{{- define "loki.bloomCompactorPriorityClassName" }}
+{{- $pcn := coalesce .Values.global.priorityClassName .Values.bloomCompactor.priorityClassName -}}
+{{- if $pcn }}
+priorityClassName: {{ $pcn }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the bloom compactor service account
+*/}}
+{{- define "loki.bloomCompactorServiceAccountName" -}}
+{{- if .Values.bloomCompactor.serviceAccount.create -}}
+    {{ default (print (include "loki.serviceAccountName" .) "-bloom-compactor") .Values.bloomCompactor.serviceAccount.name }}
+{{- else -}}
+    {{ default (include "loki.serviceAccountName" .) .Values.bloomCompactor.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/production/helm/loki/templates/bloomfilters/_helpers-bloom-gateway.tpl b/production/helm/loki/templates/bloomfilters/_helpers-bloom-gateway.tpl
@@ -0,0 +1,58 @@
+{{/*
+bloom gateway fullname
+*/}}
+{{- define "loki.bloomGatewayFullname" -}}
+{{ include "loki.fullname" . }}-bloom-gateway
+{{- end }}
+
+{{/*
+bloom gateway common labels
+*/}}
+{{- define "loki.bloomGatewayLabels" -}}
+{{ include "loki.labels" . }}
+app.kubernetes.io/component: bloom-gateway
+{{- end }}
+
+{{/*
+bloom gateway selector labels
+*/}}
+{{- define "loki.bloomGatewaySelectorLabels" -}}
+{{ include "loki.selectorLabels" . }}
+app.kubernetes.io/component: bloom-gateway
+{{- end }}
+
+{{/*
+bloom gateway readinessProbe
+*/}}
+{{- define "loki.bloomGateway.readinessProbe" -}}
+{{- with .Values.bloomGateway.readinessProbe }}
+readinessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- else }}
+{{- with .Values.loki.readinessProbe }}
+readinessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+bloom gateway priority class name
+*/}}
+{{- define "loki.bloomGatewayPriorityClassName" }}
+{{- $pcn := coalesce .Values.global.priorityClassName .Values.bloomGateway.priorityClassName -}}
+{{- if $pcn }}
+priorityClassName: {{ $pcn }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the bloom gateway service account
+*/}}
+{{- define "loki.bloomGatewayServiceAccountName" -}}
+{{- if .Values.bloomGateway.serviceAccount.create -}}
+    {{ default (print (include "loki.serviceAccountName" .) "-bloom-gateway") .Values.bloomGateway.serviceAccount.name }}
+{{- else -}}
+    {{ default (include "loki.serviceAccountName" .) .Values.bloomGateway.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/production/helm/loki/templates/bloomfilters/statefulset-bloom-compactor.yaml b/production/helm/loki/templates/bloomfilters/statefulset-bloom-compactor.yaml
@@ -0,0 +1,179 @@
+{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
+{{- if $isDistributed }}
+{{- if (eq .Values.loki.useBloomFilters true) -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "loki.bloomCompactorFullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "loki.bloomCompactorLabels" . | nindent 4 }}
+  {{- with .Values.loki.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: {{ .Values.bloomCompactor.replicas }}
+  podManagementPolicy: Parallel
+  updateStrategy:
+    rollingUpdate:
+      partition: 0
+  serviceName: {{ include "loki.bloomCompactorFullname" . }}-headless
+  revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }}
+  {{- if and (semverCompare ">= 1.23-0" .Capabilities.KubeVersion.Version) (.Values.bloomCompactor.persistence.enableStatefulSetAutoDeletePVC)  }}
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: {{ .Values.bloomCompactor.persistence.whenDeleted }}
+    whenScaled: {{ .Values.bloomCompactor.persistence.whenScaled }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "loki.bloomCompactorSelectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- include "loki.config.checksum" . | nindent 8 }}
+        {{- with .Values.loki.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.bloomCompactor.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "loki.bloomCompactorSelectorLabels" . | nindent 8 }}
+        app.kubernetes.io/part-of: memberlist
+        {{- with .Values.loki.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.bloomCompactor.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ include "loki.serviceAccountName" . }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.bloomCompactor.hostAliases }}
+      hostAliases:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- include "loki.bloomCompactorPriorityClassName" . | nindent 6 }}
+      securityContext:
+        {{- toYaml .Values.loki.podSecurityContext | nindent 8 }}
+      terminationGracePeriodSeconds: {{ .Values.bloomCompactor.terminationGracePeriodSeconds }}
+      {{- with .Values.bloomCompactor.initContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: bloom-compactor
+          image: {{ include "loki.image" . }}
+          imagePullPolicy: {{ .Values.loki.image.pullPolicy }}
+          {{- if or .Values.loki.command .Values.bloomCompactor.command }}
+          command:
+            - {{ coalesce .Values.bloomCompactor.command .Values.loki.command | quote }}
+          {{- end }}
+          args:
+            - -config.file=/etc/loki/config/config.yaml
+            - -target=bloom-compactor
+            {{- with .Values.bloomCompactor.extraArgs }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          ports:
+            - name: http-metrics
+              containerPort: 3100
+              protocol: TCP
+            - name: grpc
+              containerPort: 9095
+              protocol: TCP
+            - name: http-memberlist
+              containerPort: 7946
+              protocol: TCP
+          {{- with .Values.bloomCompactor.extraEnv }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.bloomCompactor.extraEnvFrom }}
+          envFrom:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          securityContext:
+            {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }}
+          {{- include "loki.bloomCompactor.readinessProbe" . | nindent 10 }}
+          volumeMounts:
+            - name: temp
+              mountPath: /tmp
+            - name: config
+              mountPath: /etc/loki/config
+            - name: runtime-config
+              mountPath: /etc/loki/runtime-config
+            - name: data
+              mountPath: /var/loki
+            {{- if .Values.enterprise.enabled }}
+            - name: license
+              mountPath: /etc/loki/license
+            {{- end }}
+            {{- with .Values.bloomCompactor.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with .Values.bloomCompactor.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- if .Values.bloomCompactor.extraContainers }}
+        {{- toYaml .Values.bloomCompactor.extraContainers | nindent 8}}
+        {{- end }}
+      {{- with .Values.bloomCompactor.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.bloomCompactor.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: temp
+          emptyDir: {}
+        - name: config
+          {{- include "loki.configVolume" . | nindent 10 }}
+        - name: runtime-config
+          configMap:
+            name: {{ template "loki.name" . }}-runtime
+        {{- if .Values.enterprise.enabled }}
+        - name: license
+          secret:
+          {{- if .Values.enterprise.useExternalLicense }}
+            secretName: {{ .Values.enterprise.externalLicenseName }}
+          {{- else }}
+            secretName: enterprise-logs-license
+          {{- end }}
+        {{- end }}
+        {{- if not .Values.bloomCompactor.persistence.enabled }}
+        - name: data
+          emptyDir: {}
+        {{- end }}
+        {{- with .Values.bloomCompactor.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+  {{- if .Values.bloomCompactor.persistence.enabled }}
+  volumeClaimTemplates:
+  {{- range .Values.bloomCompactor.persistence.claims }}
+    - metadata:
+        name: {{ .name }}
+        {{- with .annotations }}
+        annotations:
+          {{- . | toYaml | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        {{- with .storageClass }}
+        storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }}
+        {{- end }}
+        resources:
+          requests:
+            storage: {{ .size | quote }}
+  {{- end }}
+  {{- end }}
+{{- end -}}
+{{- end -}}