Add new pattern-ingester component

grafana · Apr 4, 2024 · 0c9b787 · 0c9b787
1 parent 4587ff5
commit 0c9b787
Show file tree

Hide file tree

Showing 3 changed files with 346 additions and 0 deletions.
diff --git a/production/helm/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl b/production/helm/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl
@@ -0,0 +1,58 @@
+{{/*
+pattern ingester fullname
+*/}}
+{{- define "loki.patternIngesterFullname" -}}
+{{ include "loki.fullname" . }}-pattern-ingester
+{{- end }}
+
+{{/*
+pattern ingester common labels
+*/}}
+{{- define "loki.patternIngesterLabels" -}}
+{{ include "loki.labels" . }}
+app.kubernetes.io/component: pattern-ingester
+{{- end }}
+
+{{/*
+pattern ingester selector labels
+*/}}
+{{- define "loki.patternIngesterSelectorLabels" -}}
+{{ include "loki.selectorLabels" . }}
+app.kubernetes.io/component: pattern-ingester
+{{- end }}
+
+{{/*
+pattern ingester readinessProbe
+*/}}
+{{- define "loki.patternIngester.readinessProbe" -}}
+{{- with .Values.patternIngester.readinessProbe }}
+readinessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- else }}
+{{- with .Values.loki.readinessProbe }}
+readinessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+pattern ingester priority class name
+*/}}
+{{- define "loki.patternIngesterPriorityClassName" }}
+{{- $pcn := coalesce .Values.global.priorityClassName .Values.patternIngester.priorityClassName -}}
+{{- if $pcn }}
+priorityClassName: {{ $pcn }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create the name of the pattern ingester service account
+*/}}
+{{- define "loki.patternIngesterServiceAccountName" -}}
+{{- if .Values.patternIngester.serviceAccount.create -}}
+    {{ default (print (include "loki.serviceAccountName" .) "-pattern-ingester") .Values.patternIngester.serviceAccount.name }}
+{{- else -}}
+    {{ default (include "loki.serviceAccountName" .) .Values.patternIngester.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/production/helm/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml b/production/helm/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml
@@ -0,0 +1,179 @@
+{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
+{{- if $isDistributed }}
+{{- if (eq .Values.loki.usePatternIngester true) -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "loki.patternIngesterFullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "loki.patternIngesterLabels" . | nindent 4 }}
+  {{- with .Values.loki.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: {{ .Values.patternIngester.replicas }}
+  podManagementPolicy: Parallel
+  updateStrategy:
+    rollingUpdate:
+      partition: 0
+  serviceName: {{ include "loki.patternIngesterFullname" . }}-headless
+  revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }}
+  {{- if and (semverCompare ">= 1.23-0" .Capabilities.KubeVersion.Version) (.Values.patternIngester.persistence.enableStatefulSetAutoDeletePVC)  }}
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: {{ .Values.patternIngester.persistence.whenDeleted }}
+    whenScaled: {{ .Values.patternIngester.persistence.whenScaled }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "loki.patternIngesterSelectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- include "loki.config.checksum" . | nindent 8 }}
+        {{- with .Values.loki.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.patternIngester.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "loki.patternIngesterSelectorLabels" . | nindent 8 }}
+        app.kubernetes.io/part-of: memberlist
+        {{- with .Values.loki.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- with .Values.patternIngester.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ include "loki.serviceAccountName" . }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.patternIngester.hostAliases }}
+      hostAliases:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- include "loki.patternIngesterPriorityClassName" . | nindent 6 }}
+      securityContext:
+        {{- toYaml .Values.loki.podSecurityContext | nindent 8 }}
+      terminationGracePeriodSeconds: {{ .Values.patternIngester.terminationGracePeriodSeconds }}
+      {{- with .Values.patternIngester.initContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: pattern-ingester
+          image: {{ include "loki.image" . }}
+          imagePullPolicy: {{ .Values.loki.image.pullPolicy }}
+          {{- if or .Values.loki.command .Values.patternIngester.command }}
+          command:
+            - {{ coalesce .Values.patternIngester.command .Values.loki.command | quote }}
+          {{- end }}
+          args:
+            - -config.file=/etc/loki/config/config.yaml
+            - -target=pattern-ingester
+            {{- with .Values.patternIngester.extraArgs }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          ports:
+            - name: http-metrics
+              containerPort: 3100
+              protocol: TCP
+            - name: grpc
+              containerPort: 9095
+              protocol: TCP
+            - name: http-memberlist
+              containerPort: 7946
+              protocol: TCP
+          {{- with .Values.patternIngester.extraEnv }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.patternIngester.extraEnvFrom }}
+          envFrom:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          securityContext:
+            {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }}
+          {{- include "loki.patternIngester.readinessProbe" . | nindent 10 }}
+          volumeMounts:
+            - name: temp
+              mountPath: /tmp
+            - name: config
+              mountPath: /etc/loki/config
+            - name: runtime-config
+              mountPath: /etc/loki/runtime-config
+            - name: data
+              mountPath: /var/loki
+            {{- if .Values.enterprise.enabled }}
+            - name: license
+              mountPath: /etc/loki/license
+            {{- end }}
+            {{- with .Values.patternIngester.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with .Values.patternIngester.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- if .Values.patternIngester.extraContainers }}
+        {{- toYaml .Values.patternIngester.extraContainers | nindent 8}}
+        {{- end }}
+      {{- with .Values.patternIngester.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.patternIngester.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: temp
+          emptyDir: {}
+        - name: config
+          {{- include "loki.configVolume" . | nindent 10 }}
+        - name: runtime-config
+          configMap:
+            name: {{ template "loki.name" . }}-runtime
+        {{- if .Values.enterprise.enabled }}
+        - name: license
+          secret:
+          {{- if .Values.enterprise.useExternalLicense }}
+            secretName: {{ .Values.enterprise.externalLicenseName }}
+          {{- else }}
+            secretName: enterprise-logs-license
+          {{- end }}
+        {{- end }}
+        {{- if not .Values.patternIngester.persistence.enabled }}
+        - name: data
+          emptyDir: {}
+        {{- end }}
+        {{- with .Values.patternIngester.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+  {{- if .Values.patternIngester.persistence.enabled }}
+  volumeClaimTemplates:
+  {{- range .Values.patternIngester.persistence.claims }}
+    - metadata:
+        name: {{ .name }}
+        {{- with .annotations }}
+        annotations:
+          {{- . | toYaml | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        {{- with .storageClass }}
+        storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }}
+        {{- end }}
+        resources:
+          requests:
+            storage: {{ .size | quote }}
+  {{- end }}
+  {{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml
@@ -412,6 +412,7 @@ loki:
   tracing:
     enabled: false
   useBloomFilters: false
+  usePatternIngester: false
 
 ######################################################################################################################
 #
@@ -2551,6 +2552,114 @@ bloomCompactor:
     # -- Set this toggle to false to opt out of automounting API credentials for the service account
     automountServiceAccountToken: true
 
+# -- Configuration for the pattern ingester
+patternIngester:
+  # -- Number of replicas for the pattern ingester
+  replicas: 0
+  # -- hostAliases to add
+  hostAliases: []
+  #  - ip: 1.2.3.4
+  #    hostnames:
+  #      - domain.tld
+  image:
+    # -- The Docker registry for the pattern ingester image. Overrides `loki.image.registry`
+    registry: null
+    # -- Docker image repository for the pattern ingester image. Overrides `loki.image.repository`
+    repository: null
+    # -- Docker image tag for the pattern ingester image. Overrides `loki.image.tag`
+    tag: null
+  # -- Command to execute instead of defined in Docker image
+  command: null
+  # -- The name of the PriorityClass for pattern ingester pods
+  priorityClassName: null
+  # -- Labels for pattern ingester pods
+  podLabels: {}
+  # -- Annotations for pattern ingester pods
+  podAnnotations: {}
+  # -- Affinity for pattern ingester pods.
+  # @default -- Hard node anti-affinity
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              app.kubernetes.io/component: pattern-ingester
+          topologyKey: kubernetes.io/hostname
+  # -- Labels for pattern ingester service
+  serviceLabels: {}
+  # -- Additional CLI args for the pattern ingester
+  extraArgs: []
+  # -- Environment variables to add to the pattern ingester pods
+  extraEnv: []
+  # -- Environment variables from secrets or configmaps to add to the pattern ingester pods
+  extraEnvFrom: []
+  # -- Volume mounts to add to the pattern ingester pods
+  extraVolumeMounts: []
+  # -- Volumes to add to the pattern ingester pods
+  extraVolumes: []
+  # -- readiness probe settings for ingester pods. If empty, use `loki.readinessProbe`
+  readinessProbe: {}
+  # -- liveness probe settings for ingester pods. If empty use `loki.livenessProbe`
+  livenessProbe: {}
+  # -- Resource requests and limits for the pattern ingester
+  resources: {}
+  # -- Containers to add to the pattern ingester pods
+  extraContainers: []
+  # -- Init containers to add to the pattern ingester pods
+  initContainers: []
+  # -- Grace period to allow the pattern ingester to shutdown before it is killed
+  terminationGracePeriodSeconds: 30
+  # -- Node selector for pattern ingester pods
+  nodeSelector: {}
+  # -- Tolerations for pattern ingester pods
+  tolerations: []
+  # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https"
+  appProtocol:
+    grpc: ""
+  persistence:
+    # -- Enable creating PVCs for the pattern ingester
+    enabled: false
+    # -- Size of persistent disk
+    size: 10Gi
+    # -- Storage class to be used.
+    # If defined, storageClassName: <storageClass>.
+    # If set to "-", storageClassName: "", which disables dynamic provisioning.
+    # If empty or set to null, no storageClassName spec is
+    # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
+    storageClass: null
+    # -- Annotations for pattern ingester PVCs
+    annotations: {}
+    # -- List of the pattern ingester PVCs
+    # @notationType -- list
+    claims:
+      - name: data
+        size: 10Gi
+        #   -- Storage class to be used.
+        #   If defined, storageClassName: <storageClass>.
+        #   If set to "-", storageClassName: "", which disables dynamic provisioning.
+        #   If empty or set to null, no storageClassName spec is
+        #   set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
+        storageClass: null
+      # - name: wal
+      #   size: 150Gi
+    # -- Enable StatefulSetAutoDeletePVC feature
+    enableStatefulSetAutoDeletePVC: false
+    whenDeleted: Retain
+    whenScaled: Retain
+
+  serviceAccount:
+    create: false
+    # -- The name of the ServiceAccount to use for the pattern ingester.
+    # If not set and create is true, a name is generated by appending
+    # "-pattern-ingester" to the common ServiceAccount.
+    name: null
+    # -- Image pull secrets for the pattern ingester service account
+    imagePullSecrets: []
+    # -- Annotations for the pattern ingester service account
+    annotations: {}
+    # -- Set this toggle to false to opt out of automounting API credentials for the service account
+    automountServiceAccountToken: true
+
 # -- Configuration for the ruler
 ruler:
   # -- The ruler component is optional and can be disabled if desired.