UPDATE - manifest for meteo metrics substatus

playbooks/deploy_beats.yml

@@ -47,6 +47,7 @@
     loop: 
     - metricbeat-kubernetes
     - filebeat-kubernetes
+    - metricbeat-kubernetes-substatus
 
   - name: Apply metricbeat manifest to the cluster.
     kubernetes.core.k8s:

playbooks/destroy_all.yml

@@ -7,7 +7,7 @@
         msg: Destroy operatation for beat agents 
 
 
-    - name: Apply metrics-server manifest to the cluster.
+    - name: Remove metrics-server manifest to the cluster.
       kubernetes.core.k8s:
         state: absent
         src: ~/{{ item }}.yaml
@@ -15,6 +15,7 @@
       - metricbeat-kubernetes
       - filebeat-kubernetes
       - metrics-server
+      - metricbeat-kubernetes-substatus
 
     - name: Delete manifests from the cluster
       ansible.builtin.file:
@@ -24,3 +25,5 @@
       - metricbeat-kubernetes
       - filebeat-kubernetes
       - metrics-server
+      - metricbeat-kubernetes-substatus
+

playbooks/templates/metricbeat-kubernetes-substatus.j2

+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: kube-system
+  labels:
+    k8s-app: metricbeat-meteo
+data:
+  metricbeat.yml: |-
+    metricbeat.config.modules:
+      # Mounted `metricbeat-meteo-modules` configmap:
+      path: ${path.config}/modules.d/*.yml
+      # Reload module configs as they change:
+      reload.enabled: false
+
+    setup.dashboards.enabled: true
+    setup.kibana:
+      host: ${KIBANA_HOSTNAME}:${KIBANA_PORT}
+      # username: ${KIBANA_USERNAME}
+      # password: ${KIBANA_PASSWORD}
+
+
+    #processors:
+    #  - add_cloud_metadata:
+    processors:
+      - add_fields: 
+          target: orchestrator.cluster
+          fields:
+            name: msicd-meteo
+          #  url: clusterUrl 
+
+
+    cloud.id: ${ELASTIC_CLOUD_ID}
+    cloud.auth: ${ELASTIC_CLOUD_AUTH}
+
+    output.elasticsearch:
+      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
+      # username: ${ELASTICSEARCH_USERNAME}
+      # password: ${ELASTICSEARCH_PASSWORD}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: meteo-metricbeat-modules
+  namespace: kube-system
+  labels:
+    k8s-app: metricbeat-meteo
+data:
+  nginx.yml: |-
+    - module: nginx
+      metricsets: ["stubstatus"]
+      enabled: true
+      period: 10s
+
+      # Nginx hosts
+      hosts: ["http://${NGINX_HOST}"]
+
+      # Path to server status. Default nginx_status
+      server_status_path: "nginx_status"
+
+---
+# Deploy a Metricbeat instance per node for node metrics retrieval
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metricbeat
+  namespace: kube-system
+  labels:
+    k8s-app: metricbeat-meteo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: metricbeat-meteo
+  template:
+    metadata:
+      labels:
+        k8s-app: metricbeat-meteo
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: metricbeat-meteo
+        image: docker.elastic.co/beats/metricbeat:8.6.2
+        args: [
+          "-c", "/etc/metricbeat.yml",
+        ]
+        #args: [
+        #  "-c", "/etc/metricbeat.yml",
+        #  "-e",
+        #  "-system.hostfs=/hostfs",
+        #]
+        env:
+        - name: ELASTICSEARCH_HOST
+          value: {{ elasticsearch_ip }}
+        - name: ELASTICSEARCH_PORT
+          value: "{{ elasticsearch_port }}"
+        - name: ELASTICSEARCH_USERNAME
+          value: elastic
+        - name: ELASTICSEARCH_PASSWORD
+          value: changeme
+        - name: KIBANA_HOSTNAME
+          value: {{ kibana_ip }}
+        - name: KIBANA_PORT
+          value: "{{ kibana_port }}"
+        - name: NGINX_HOST
+          value: {{ nginx_service_name }}
+        - name: KIBANA_USERNAME
+          value: elastic
+        - name: KIBANA_PASSWORD
+          value: changeme
+        - name: ELASTIC_CLOUD_ID
+          value:
+        - name: ELASTIC_CLOUD_AUTH
+          value:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          runAsUser: 0
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/metricbeat.yml
+          readOnly: true
+          subPath: metricbeat.yml
+        - name: data
+          mountPath: /usr/share/metricbeat/data
+        - name: meteo-metricbeat-modules
+          mountPath: /usr/share/metricbeat/modules.d
+          readOnly: true
+      #   - name: proc
+      #     mountPath: /hostfs/proc
+      #     readOnly: true
+      #   - name: cgroup
+      #     mountPath: /hostfs/sys/fs/cgroup
+      #     readOnly: true
+      volumes:
+      # - name: proc
+      #   hostPath:
+      #     path: /proc
+      # - name: cgroup
+      #   hostPath:
+      #     path: /sys/fs/cgroup
+      # - name: config
+      #   configMap:
+      #     defaultMode: 0640
+      #     name: metricbeat-daemonset-config
+      - name: meteo-metricbeat
+        configMap:
+          defaultMode: 0640
+          name: meteo-metricbeat-modules
+      - name: data
+        hostPath:
+          # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w)
+          path: /var/lib/metricbeat-meteo-data
+          type: DirectoryOrCreate
+

playbooks/templates/metricbeat-kubernetes.j2

@@ -208,7 +208,7 @@ data:
       - add_fields: 
           target: orchestrator.cluster
           fields:
-            name: msicd-vapormap
+            name: msicd-system
           #  url: clusterUrl