locals { authentik_url = "http://authentik.${var.domain}-auth.svc" authentik_token = data.kubernetes_secret_v1.authentik.data["AUTHENTIK_BOOTSTRAP_TOKEN"] common_labels = { "vynil.solidite.fr/owner-name" = var.instance "vynil.solidite.fr/owner-namespace" = var.namespace "vynil.solidite.fr/owner-category" = var.category "vynil.solidite.fr/owner-component" = var.component "app.kubernetes.io/managed-by" = "vynil" "app.kubernetes.io/instance" = var.instance } rb-patch = <<-EOF - op: replace path: /subjects/0/namespace value: "${var.namespace}" EOF } data "kubernetes_secret_v1" "authentik" { metadata { name = "authentik" namespace = "${var.domain}-auth" } } data "kustomization_overlay" "data" { common_labels = local.common_labels namespace = var.namespace resources = [for file in fileset(path.module, "*.yaml"): file if file != "index.yaml" && length(regexall("ClusterRole",file))<1 && length(regexall("Service_prometheus",file))<1] patches { target { kind = "ConfigMap" name = "prometheus-kube-prometheus-grafana-datasource" } patch = <<-EOF apiVersion: v1 kind: ConfigMap metadata: name: prometheus-kube-prometheus-grafana-datasource data: datasource.yaml: |- apiVersion: 1 datasources: - name: Prometheus type: prometheus uid: prometheus url: http://${var.instance}-${var.component}.${var.namespace}:9090/ access: proxy isDefault: false jsonData: httpMethod: POST timeInterval: 30s EOF } patches { target { kind = "ServiceMonitor" name = "prometheus-kube-prometheus-prometheus" } patch = <<-EOF - op: replace path: /spec/namespaceSelector/matchNames/0 value: "${var.namespace}" EOF } patches { target { kind = "PrometheusRule" name = "prometheus-kube-prometheus-prometheus" } patch = <<-EOF apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: prometheus-kube-prometheus-prometheus spec: groups: - name: prometheus rules: - alert: PrometheusBadConfig expr: |- # Without max_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. max_over_time(prometheus_config_last_reload_successful{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) == 0 - alert: PrometheusSDRefreshFailure expr: increase(prometheus_sd_refresh_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[10m]) > 0 - alert: PrometheusNotificationQueueRunningFull expr: |- # Without min_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. ( predict_linear(prometheus_notifications_queue_length{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m], 60 * 30) > min_over_time(prometheus_notifications_queue_capacity{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) ) - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers expr: |- ( rate(prometheus_notifications_errors_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) / rate(prometheus_notifications_sent_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) ) * 100 > 1 - alert: PrometheusNotConnectedToAlertmanagers expr: |- # Without max_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) < 1 - alert: PrometheusTSDBReloadsFailing expr: increase(prometheus_tsdb_reloads_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[3h]) > 0 - alert: PrometheusTSDBCompactionsFailing expr: increase(prometheus_tsdb_compactions_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[3h]) > 0 - alert: PrometheusNotIngestingSamples expr: |- ( rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) <= 0 and ( sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}) > 0 or sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}) > 0 ) ) - alert: PrometheusDuplicateTimestamps expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusOutOfOrderTimestamps expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusRemoteStorageFailures expr: |- ( (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])) / ( (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])) + (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])) ) ) * 100 > 1 - alert: PrometheusRemoteWriteBehind expr: |- # Without max_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. ( max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) - ignoring(remote_name, url) group_right max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) ) > 120 - alert: PrometheusRemoteWriteDesiredShards expr: |- # Without max_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. ( max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > max_over_time(prometheus_remote_storage_shards_max{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) ) - alert: PrometheusRuleFailures expr: increase(prometheus_rule_evaluation_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusMissingRuleEvaluations expr: increase(prometheus_rule_group_iterations_missed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusTargetLimitHit expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusLabelLimitHit expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusScrapeBodySizeLimitHit expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusScrapeSampleLimitHit expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0 - alert: PrometheusTargetSyncFailure expr: increase(prometheus_target_sync_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[30m]) > 0 - alert: PrometheusHighQueryLoad expr: avg_over_time(prometheus_engine_queries{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0.8 - alert: PrometheusErrorSendingAlertsToAnyAlertmanager expr: |- min without (alertmanager) ( rate(prometheus_notifications_errors_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}",alertmanager!~``}[5m]) / rate(prometheus_notifications_sent_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}",alertmanager!~``}[5m]) ) * 100 > 3 EOF } patches { target { kind = "ServiceMonitor" name = "prometheus-community-prometheus-node-exporter" } patch = <<-EOF - op: replace path: /spec/selector/matchLabels/app.kubernetes.io~1instance value: "${var.instance}" EOF } } data "kustomization_overlay" "data_no_ns" { resources = [for file in fileset(path.module, "*.yaml"): file if (length(regexall("ClusterRole",file))>0 || length(regexall("Service_prometheus",file))>0)] patches { target { kind = "ClusterRoleBinding" name = "prometheus-kube-prometheus-prometheus" } patch = local.rb-patch } }