fix

2024-01-24 10:41:15 +01:00
parent a66dcce956
commit 109304f06d
22 changed files with 200 additions and 196 deletions
--- a/monitor/prometheus/datas.tf
+++ b/monitor/prometheus/datas.tf
@@ -213,6 +213,17 @@ data "kustomization_overlay" "data" {
              > 3
    EOF
  }
+  patches {
+    target {
+      kind = "ServiceMonitor"
+      name = "prometheus-community-prometheus-node-exporter"
+    }
+    patch = <<-EOF
+    - op: replace
+      path: /spec/selector/matchLabels/app.kubernetes.io~1instance
+      value: "${var.instance}"
+    EOF
+  }
 }

 data "kustomization_overlay" "data_no_ns" {
--- a/monitor/prometheus/index.yaml
+++ b/monitor/prometheus/index.yaml
@@ -6,36 +6,11 @@ metadata:
  name: prometheus
  description: null
 options:
-  domain:
-    default: your-company
-    examples:
-    - your-company
-    type: string
  ingress-class:
    default: traefik
    examples:
    - traefik
    type: string
-  retention:
-    default: 10d
-    examples:
-    - 10d
-    type: string
-  listenLocal:
-    default: false
-    examples:
-    - false
-    type: boolean
-  shards:
-    default: 1
-    examples:
-    - 1
-    type: integer
-  domain-name:
-    default: your_company.com
-    examples:
-    - your_company.com
-    type: string
  app-group:
    default: monitor
    examples:
@@ -46,20 +21,30 @@ options:
    examples:
    - info
    type: string
+  listenLocal:
+    default: false
+    examples:
+    - false
+    type: boolean
+  retention:
+    default: 10d
+    examples:
+    - 10d
+    type: string
+  issuer:
+    default: letsencrypt-prod
+    examples:
+    - letsencrypt-prod
+    type: string
  enableAdminAPI:
    default: false
    examples:
    - false
    type: boolean
-  replicas:
-    default: 1
+  sub-domain:
+    default: prometheus
    examples:
-    - 1
-    type: integer
-  issuer:
-    default: letsencrypt-prod
-    examples:
-    - letsencrypt-prod
+    - prometheus
    type: string
  images:
    default:
@@ -100,11 +85,26 @@ options:
            type: string
        type: object
    type: object
-  sub-domain:
-    default: prometheus
+  shards:
+    default: 1
    examples:
-    - prometheus
+    - 1
+    type: integer
+  domain-name:
+    default: your_company.com
+    examples:
+    - your_company.com
    type: string
+  domain:
+    default: your-company
+    examples:
+    - your-company
+    type: string
+  replicas:
+    default: 1
+    examples:
+    - 1
+    type: integer
 dependencies:
 - dist: null
  category: share
--- a/monitor/prometheus/monitoring.coreos.com_v1_PrometheusRule_prometheus-kube-prometheus-kubelet.rules.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_PrometheusRule_prometheus-kube-prometheus-kubelet.rules.yaml
@@ -2,17 +2,17 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: prometheus-community-kube-kubelet.rules
+  name: prometheus-kube-prometheus-kubelet.rules
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 spec:
  groups:
--- a/monitor/prometheus/monitoring.coreos.com_v1_PrometheusRule_prometheus-kube-prometheus-kubernetes-system-kube-proxy.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_PrometheusRule_prometheus-kube-prometheus-kubernetes-system-kube-proxy.yaml
@@ -2,17 +2,17 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: prometheus-community-kube-kubernetes-system-kube-proxy
+  name: prometheus-kube-prometheus-kubernetes-system-kube-proxy
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 spec:
  groups:
--- a/monitor/prometheus/monitoring.coreos.com_v1_PrometheusRule_prometheus-kube-prometheus-prometheus.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_PrometheusRule_prometheus-kube-prometheus-prometheus.yaml
@@ -2,17 +2,17 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 spec:
  groups:
@@ -26,7 +26,7 @@ spec:
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
-        max_over_time(prometheus_config_last_reload_successful{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) == 0
+        max_over_time(prometheus_config_last_reload_successful{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) == 0
      for: 10m
      labels:
        severity: critical
@@ -35,7 +35,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to refresh SD with mechanism {{$labels.mechanism}}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheussdrefreshfailure
        summary: Failed Prometheus SD refresh.
-      expr: increase(prometheus_sd_refresh_failures_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[10m]) > 0
+      expr: increase(prometheus_sd_refresh_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[10m]) > 0
      for: 20m
      labels:
        severity: warning
@@ -48,9 +48,9 @@ spec:
        # Without min_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        (
-          predict_linear(prometheus_notifications_queue_length{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m], 60 * 30)
+          predict_linear(prometheus_notifications_queue_length{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m], 60 * 30)
        >
-          min_over_time(prometheus_notifications_queue_capacity{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m])
+          min_over_time(prometheus_notifications_queue_capacity{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m])
        )
      for: 15m
      labels:
@@ -62,9 +62,9 @@ spec:
        summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
      expr: |-
        (
-          rate(prometheus_notifications_errors_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m])
+          rate(prometheus_notifications_errors_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m])
        /
-          rate(prometheus_notifications_sent_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m])
+          rate(prometheus_notifications_sent_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m])
        )
        * 100
        > 1
@@ -79,7 +79,7 @@ spec:
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
-        max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) < 1
+        max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) < 1
      for: 10m
      labels:
        severity: warning
@@ -88,7 +88,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbreloadsfailing
        summary: Prometheus has issues reloading blocks from disk.
-      expr: increase(prometheus_tsdb_reloads_failures_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[3h]) > 0
+      expr: increase(prometheus_tsdb_reloads_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[3h]) > 0
      for: 4h
      labels:
        severity: warning
@@ -97,7 +97,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbcompactionsfailing
        summary: Prometheus has issues compacting blocks.
-      expr: increase(prometheus_tsdb_compactions_failed_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[3h]) > 0
+      expr: increase(prometheus_tsdb_compactions_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[3h]) > 0
      for: 4h
      labels:
        severity: warning
@@ -108,12 +108,12 @@ spec:
        summary: Prometheus is not ingesting samples.
      expr: |-
        (
-          rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) <= 0
+          rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) <= 0
        and
          (
-            sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}) > 0
+            sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}) > 0
          or
-            sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}) > 0
+            sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}) > 0
          )
        )
      for: 10m
@@ -124,7 +124,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with different values but duplicated timestamp.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusduplicatetimestamps
        summary: Prometheus is dropping samples with duplicate timestamps.
-      expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 10m
      labels:
        severity: warning
@@ -133,7 +133,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with timestamps arriving out of order.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusoutofordertimestamps
        summary: Prometheus drops samples with out-of-order timestamps.
-      expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 10m
      labels:
        severity: warning
@@ -144,12 +144,12 @@ spec:
        summary: Prometheus fails to send samples to remote storage.
      expr: |-
        (
-          (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]))
+          (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]))
        /
          (
-            (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]))
+            (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]))
          +
-            (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]))
+            (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]))
          )
        )
        * 100
@@ -166,9 +166,9 @@ spec:
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        (
-          max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m])
+          max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m])
        - ignoring(remote_name, url) group_right
-          max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m])
+          max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m])
        )
        > 120
      for: 15m
@@ -176,16 +176,16 @@ spec:
        severity: critical
    - alert: PrometheusRemoteWriteDesiredShards
      annotations:
-        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}` $labels.instance | query | first | value }}.
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}` $labels.instance | query | first | value }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritedesiredshards
        summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        (
-          max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m])
+          max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m])
        >
-          max_over_time(prometheus_remote_storage_shards_max{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m])
+          max_over_time(prometheus_remote_storage_shards_max{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m])
        )
      for: 15m
      labels:
@@ -195,7 +195,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusrulefailures
        summary: Prometheus is failing rule evaluations.
-      expr: increase(prometheus_rule_evaluation_failures_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: increase(prometheus_rule_evaluation_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 15m
      labels:
        severity: critical
@@ -204,7 +204,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusmissingruleevaluations
        summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
-      expr: increase(prometheus_rule_group_iterations_missed_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: increase(prometheus_rule_group_iterations_missed_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
@@ -213,7 +213,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetlimithit
        summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit.
-      expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
@@ -222,7 +222,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuslabellimithit
        summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit.
-      expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
@@ -231,7 +231,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapebodysizelimithit
        summary: Prometheus has dropped some targets that exceeded body size limit.
-      expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
@@ -240,7 +240,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapesamplelimithit
        summary: Prometheus has failed scrapes that have exceeded the configured sample limit.
-      expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0
+      expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
@@ -249,7 +249,7 @@ spec:
        description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} have failed to sync because invalid configuration was supplied.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetsyncfailure
        summary: Prometheus has failed to sync targets.
-      expr: increase(prometheus_target_sync_failed_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[30m]) > 0
+      expr: increase(prometheus_target_sync_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[30m]) > 0
      for: 5m
      labels:
        severity: critical
@@ -258,7 +258,7 @@ spec:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has less than 20% available capacity in its query engine for the last 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheushighqueryload
        summary: Prometheus is reaching its maximum capacity serving concurrent requests.
-      expr: avg_over_time(prometheus_engine_queries{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-community-kube-prometheus",namespace="vynil-monitor"}[5m]) > 0.8
+      expr: avg_over_time(prometheus_engine_queries{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor"}[5m]) > 0.8
      for: 15m
      labels:
        severity: warning
@@ -269,9 +269,9 @@ spec:
        summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
      expr: |-
        min without (alertmanager) (
-          rate(prometheus_notifications_errors_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor",alertmanager!~``}[5m])
+          rate(prometheus_notifications_errors_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor",alertmanager!~``}[5m])
        /
-          rate(prometheus_notifications_sent_total{job="prometheus-community-kube-prometheus",namespace="vynil-monitor",alertmanager!~``}[5m])
+          rate(prometheus_notifications_sent_total{job="prometheus-kube-prometheus-prometheus",namespace="vynil-monitor",alertmanager!~``}[5m])
        )
        * 100
        > 3
--- a/monitor/prometheus/monitoring.coreos.com_v1_Prometheus_prometheus-community-kube-prometheus.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_Prometheus_prometheus-community-kube-prometheus.yaml
@@ -1,67 +0,0 @@
-# Source: kube-prometheus-stack/templates/prometheus/prometheus.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: Prometheus
-metadata:
-  name: prometheus-community-kube-prometheus
-  namespace: vynil-monitor
-  labels:
-    app: kube-prometheus-stack-prometheus
-    
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
-    app.kubernetes.io/version: "56.0.3"
-    app.kubernetes.io/part-of: kube-prometheus-stack
-    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
-    heritage: "Helm"
-spec:
-  image: "quay.io/prometheus/prometheus:v2.49.1"
-  version: v2.49.1
-  externalUrl: http://prometheus-community-kube-prometheus.vynil-monitor:9090
-  paused: false
-  replicas: 1
-  shards: 1
-  logLevel:  info
-  logFormat:  logfmt
-  listenLocal: false
-  enableAdminAPI: false
-  retention: "10d"
-  tsdb:
-    outOfOrderTimeWindow: 0s
-  walCompression: true
-  routePrefix: "/"
-  serviceAccountName: prometheus-community-kube-prometheus
-  serviceMonitorSelector:
-    matchLabels:
-      release: "prometheus-community"
-
-  serviceMonitorNamespaceSelector: {}
-  podMonitorSelector:
-    matchLabels:
-      release: "prometheus-community"
-
-  podMonitorNamespaceSelector: {}
-  probeSelector:
-    matchLabels:
-      release: "prometheus-community"
-
-  probeNamespaceSelector: {}
-  securityContext:
-    fsGroup: 2000
-    runAsGroup: 2000
-    runAsNonRoot: true
-    runAsUser: 1000
-    seccompProfile:
-      type: RuntimeDefault
-  ruleNamespaceSelector: {}
-  ruleSelector:
-    matchLabels:
-      release: "prometheus-community"
-
-  scrapeConfigSelector:
-    matchLabels:
-      release: "prometheus-community"
-
-  scrapeConfigNamespaceSelector: {}
-  portName: http-web
-  hostNetwork: false
--- a/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-coredns.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-coredns.yaml
@@ -2,17 +2,17 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: prometheus-community-kube-coredns
+  name: prometheus-kube-prometheus-coredns
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack-coredns
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 spec:
  jobLabel: jobLabel
@@ -20,7 +20,7 @@ spec:
  selector:
    matchLabels:
      app: kube-prometheus-stack-coredns
-      release: "prometheus-community"
+      release: "prometheus"
  namespaceSelector:
    matchNames:
      - "kube-system"
--- a/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-kube-proxy.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-kube-proxy.yaml
@@ -2,17 +2,17 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: prometheus-community-kube-kube-proxy
+  name: prometheus-kube-prometheus-kube-proxy
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack-kube-proxy
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 spec:
  jobLabel: jobLabel
@@ -20,7 +20,7 @@ spec:
  selector:
    matchLabels:
      app: kube-prometheus-stack-kube-proxy
-      release: "prometheus-community"
+      release: "prometheus"
  namespaceSelector:
    matchNames:
      - "kube-system"
--- a/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-kubelet.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-kubelet.yaml
@@ -2,16 +2,16 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: prometheus-community-kube-kubelet
+  name: prometheus-kube-prometheus-kubelet
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack-kubelet    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 spec:
  
--- a/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-prometheus.yaml
+++ b/monitor/prometheus/monitoring.coreos.com_v1_ServiceMonitor_prometheus-kube-prometheus-prometheus.yaml
@@ -2,24 +2,24 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack-prometheus
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 spec:
  
  selector:
    matchLabels:
      app: kube-prometheus-stack-prometheus
-      release: "prometheus-community"
+      release: "prometheus"
      self-monitor: "true"
  namespaceSelector:
    matchNames:
--- a/monitor/prometheus/prometheus.tf
+++ b/monitor/prometheus/prometheus.tf
@@ -0,0 +1,60 @@
+resource "kubectl_manifest" "prometheus" {
+  yaml_body  = <<-EOF
+    apiVersion: monitoring.coreos.com/v1
+    kind: Prometheus
+    metadata:
+      name: prometheus
+    spec:
+      image: "${var.images.prometheus.registry}/${var.images.prometheus.repository}:${var.images.prometheus.tag}"
+      version: ${var.images.prometheus.tag}
+      externalUrl: http://prometheus-community-kube-prometheus.${var.namespace}:9090
+      replicas: ${var.replicas}
+      shards: ${var.shards}
+      logLevel:  ${var.logLevel}
+      listenLocal: ${var.listenLocal}
+      enableAdminAPI: ${var.enableAdminAPI}
+      retention: "${var.retention}"
+      tsdb:
+        outOfOrderTimeWindow: 0s
+      walCompression: true
+      routePrefix: "/"
+      alerting:
+        alertmanagers:
+          - namespace: ${var.namespace}
+            name: prometheus-community-kube-alertmanager
+            port: http-web
+            pathPrefix: "/"
+            apiVersion: v2
+      scrapeConfigNamespaceSelector: {}
+      scrapeConfigSelector:
+        matchLabels:
+          app.kubernetes.io/managed-by: vynil
+      serviceMonitorNamespaceSelector: {}
+      serviceMonitorSelector:
+        matchLabels:
+          app.kubernetes.io/managed-by: vynil
+      podMonitorNamespaceSelector: {}
+      podMonitorSelector:
+        matchLabels:
+          app.kubernetes.io/managed-by: vynil
+      probeNamespaceSelector: {}
+      probeSelector:
+        matchLabels:
+          app.kubernetes.io/managed-by: vynil
+      ruleNamespaceSelector: {}
+      ruleSelector:
+        matchLabels:
+          app.kubernetes.io/managed-by: vynil
+      serviceAccountName: prometheus-kube-prometheus-prometheus
+      securityContext:
+        fsGroup: 2000
+        runAsGroup: 2000
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+
+      portName: http-web
+      hostNetwork: false
+  EOF
+}
--- a/monitor/prometheus/rbac.authorization.k8s.io_v1_ClusterRoleBinding_prometheus-kube-prometheus-prometheus.yaml
+++ b/monitor/prometheus/rbac.authorization.k8s.io_v1_ClusterRoleBinding_prometheus-kube-prometheus-prometheus.yaml
@@ -2,22 +2,22 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
  labels:
    app: kube-prometheus-stack-prometheus
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
 subjects:
  - kind: ServiceAccount
-    name: prometheus-community-kube-prometheus
+    name: prometheus-kube-prometheus-prometheus
    namespace: vynil-monitor
--- a/monitor/prometheus/rbac.authorization.k8s.io_v1_ClusterRole_prometheus-kube-prometheus-prometheus.yaml
+++ b/monitor/prometheus/rbac.authorization.k8s.io_v1_ClusterRole_prometheus-kube-prometheus-prometheus.yaml
@@ -2,16 +2,16 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
  labels:
    app: kube-prometheus-stack-prometheus
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 rules:
 # This permission are not in the kube-prometheus repo
--- a/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-grafana-datasource.yaml
+++ b/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-grafana-datasource.yaml
@@ -2,18 +2,18 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: prometheus-community-kube-grafana-datasource
+  name: prometheus-kube-prometheus-grafana-datasource
  namespace: vynil-monitor
  labels:
    grafana_datasource: "1"
    app: kube-prometheus-stack-grafana
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 data:
  datasource.yaml: |-
@@ -22,7 +22,7 @@ data:
    - name: Prometheus
      type: prometheus
      uid: prometheus
-      url: http://prometheus-community-kube-prometheus.vynil-monitor:9090/
+      url: http://prometheus-kube-prometheus-prometheus.vynil-monitor:9090/
      access: proxy
      isDefault: true
      jsonData:
@@ -31,7 +31,7 @@ data:
    - name: Alertmanager
      type: alertmanager
      uid: alertmanager
-      url: http://prometheus-community-kube-alertmanager.vynil-monitor:9093/
+      url: http://prometheus-kube-prometheus-alertmanager.vynil-monitor:9093/
      access: proxy
      jsonData:
        handleGrafanaManagedAlerts: false
--- a/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-k8s-coredns.yaml
+++ b/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-k8s-coredns.yaml
@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
  namespace: vynil-monitor
-  name: prometheus-community-kube-k8s-coredns
+  name: prometheus-kube-prometheus-k8s-coredns
  annotations:
    {}
  labels:
@@ -11,11 +11,11 @@ metadata:
    app: kube-prometheus-stack-grafana
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 data:
  k8s-coredns.json: |-
--- a/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-kubelet.yaml
+++ b/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-kubelet.yaml
@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
  namespace: vynil-monitor
-  name: prometheus-community-kube-kubelet
+  name: prometheus-kube-prometheus-kubelet
  annotations:
    {}
  labels:
@@ -11,11 +11,11 @@ metadata:
    app: kube-prometheus-stack-grafana
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 data:
  kubelet.json: |-
--- a/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-prometheus.yaml
+++ b/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-prometheus.yaml
@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
  namespace: vynil-monitor
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
  annotations:
    {}
  labels:
@@ -11,11 +11,11 @@ metadata:
    app: kube-prometheus-stack-grafana
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 data:
  prometheus.json: |-
--- a/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-proxy.yaml
+++ b/monitor/prometheus/v1_ConfigMap_prometheus-kube-prometheus-proxy.yaml
@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
  namespace: vynil-monitor
-  name: prometheus-community-kube-proxy
+  name: prometheus-kube-prometheus-proxy
  annotations:
    {}
  labels:
@@ -11,11 +11,11 @@ metadata:
    app: kube-prometheus-stack-grafana
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 data:
  proxy.json: |-
--- a/monitor/prometheus/v1_Secret_prometheus-kube-prometheus-prometheus.yaml
+++ b/monitor/prometheus/v1_Secret_prometheus-kube-prometheus-prometheus.yaml
@@ -2,17 +2,17 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack-prometheus
    app.kubernetes.io/component: prometheus
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
 data:
--- a/monitor/prometheus/v1_ServiceAccount_prometheus-kube-prometheus-prometheus.yaml
+++ b/monitor/prometheus/v1_ServiceAccount_prometheus-kube-prometheus-prometheus.yaml
@@ -3,7 +3,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: prometheus-community-kube-prometheus
+  name: prometheus-kube-prometheus-prometheus
  namespace: vynil-monitor
  labels:
    app: kube-prometheus-stack-prometheus
@@ -11,9 +11,9 @@ metadata:
    app.kubernetes.io/component: prometheus
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
--- a/monitor/prometheus/v1_Service_prometheus-kube-prometheus-coredns.yaml
+++ b/monitor/prometheus/v1_Service_prometheus-kube-prometheus-coredns.yaml
@@ -2,17 +2,17 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: prometheus-community-kube-coredns
+  name: prometheus-kube-prometheus-coredns
  labels:
    app: kube-prometheus-stack-coredns
    jobLabel: coredns
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
  namespace: kube-system
 spec:
--- a/monitor/prometheus/v1_Service_prometheus-kube-prometheus-kube-proxy.yaml
+++ b/monitor/prometheus/v1_Service_prometheus-kube-prometheus-kube-proxy.yaml
@@ -2,17 +2,17 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: prometheus-community-kube-kube-proxy
+  name: prometheus-kube-prometheus-kube-proxy
  labels:
    app: kube-prometheus-stack-kube-proxy
    jobLabel: kube-proxy
    
    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/instance: prometheus-community
+    app.kubernetes.io/instance: prometheus
    app.kubernetes.io/version: "56.0.3"
    app.kubernetes.io/part-of: kube-prometheus-stack
    chart: kube-prometheus-stack-56.0.3
-    release: "prometheus-community"
+    release: "prometheus"
    heritage: "Helm"
  namespace: kube-system
 spec: