213 lines
11 KiB
HCL
213 lines
11 KiB
HCL
locals {
|
|
authentik_url = "http://authentik.${var.domain}-auth.svc"
|
|
authentik_token = data.kubernetes_secret_v1.authentik.data["AUTHENTIK_BOOTSTRAP_TOKEN"]
|
|
common_labels = {
|
|
"vynil.solidite.fr/owner-name" = var.instance
|
|
"vynil.solidite.fr/owner-namespace" = var.namespace
|
|
"vynil.solidite.fr/owner-category" = var.category
|
|
"vynil.solidite.fr/owner-component" = var.component
|
|
"app.kubernetes.io/managed-by" = "vynil"
|
|
"app.kubernetes.io/instance" = var.instance
|
|
}
|
|
rb-patch = <<-EOF
|
|
- op: replace
|
|
path: /subjects/0/namespace
|
|
value: "${var.namespace}"
|
|
EOF
|
|
}
|
|
|
|
data "kubernetes_secret_v1" "authentik" {
|
|
metadata {
|
|
name = "authentik"
|
|
namespace = "${var.domain}-auth"
|
|
}
|
|
}
|
|
|
|
data "kustomization_overlay" "data" {
|
|
common_labels = local.common_labels
|
|
namespace = var.namespace
|
|
resources = [for file in fileset(path.module, "*.yaml"): file if file != "index.yaml" && length(regexall("ClusterRole",file))<1 && length(regexall("Service_prometheus",file))<1]
|
|
patches {
|
|
target {
|
|
kind = "ConfigMap"
|
|
name = "prometheus-kube-prometheus-grafana-datasource"
|
|
}
|
|
patch = <<-EOF
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-kube-prometheus-grafana-datasource
|
|
data:
|
|
datasource.yaml: |-
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: Prometheus
|
|
type: prometheus
|
|
uid: prometheus
|
|
url: http://${var.instance}-${var.component}.${var.namespace}:9090/
|
|
access: proxy
|
|
isDefault: false
|
|
jsonData:
|
|
httpMethod: POST
|
|
timeInterval: 30s
|
|
EOF
|
|
}
|
|
patches {
|
|
target {
|
|
kind = "ServiceMonitor"
|
|
name = "prometheus-kube-prometheus-prometheus"
|
|
}
|
|
patch = <<-EOF
|
|
- op: replace
|
|
path: /spec/namespaceSelector/matchNames/0
|
|
value: "${var.namespace}"
|
|
EOF
|
|
}
|
|
|
|
patches {
|
|
target {
|
|
kind = "PrometheusRule"
|
|
name = "prometheus-kube-prometheus-prometheus"
|
|
}
|
|
patch = <<-EOF
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: prometheus-kube-prometheus-prometheus
|
|
spec:
|
|
groups:
|
|
- name: prometheus
|
|
rules:
|
|
- alert: PrometheusBadConfig
|
|
expr: |-
|
|
# Without max_over_time, failed scrapes could create false negatives, see
|
|
# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
|
|
max_over_time(prometheus_config_last_reload_successful{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) == 0
|
|
- alert: PrometheusSDRefreshFailure
|
|
expr: increase(prometheus_sd_refresh_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[10m]) > 0
|
|
- alert: PrometheusNotificationQueueRunningFull
|
|
expr: |-
|
|
# Without min_over_time, failed scrapes could create false negatives, see
|
|
# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
|
|
(
|
|
predict_linear(prometheus_notifications_queue_length{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m], 60 * 30)
|
|
>
|
|
min_over_time(prometheus_notifications_queue_capacity{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])
|
|
)
|
|
- alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
|
|
expr: |-
|
|
(
|
|
rate(prometheus_notifications_errors_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])
|
|
/
|
|
rate(prometheus_notifications_sent_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])
|
|
)
|
|
* 100
|
|
> 1
|
|
- alert: PrometheusNotConnectedToAlertmanagers
|
|
expr: |-
|
|
# Without max_over_time, failed scrapes could create false negatives, see
|
|
# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
|
|
max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) < 1
|
|
- alert: PrometheusTSDBReloadsFailing
|
|
expr: increase(prometheus_tsdb_reloads_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[3h]) > 0
|
|
- alert: PrometheusTSDBCompactionsFailing
|
|
expr: increase(prometheus_tsdb_compactions_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[3h]) > 0
|
|
- alert: PrometheusNotIngestingSamples
|
|
expr: |-
|
|
(
|
|
rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) <= 0
|
|
and
|
|
(
|
|
sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}) > 0
|
|
or
|
|
sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}) > 0
|
|
)
|
|
)
|
|
- alert: PrometheusDuplicateTimestamps
|
|
expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusOutOfOrderTimestamps
|
|
expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusRemoteStorageFailures
|
|
expr: |-
|
|
(
|
|
(rate(prometheus_remote_storage_failed_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]))
|
|
/
|
|
(
|
|
(rate(prometheus_remote_storage_failed_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]))
|
|
+
|
|
(rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]))
|
|
)
|
|
)
|
|
* 100
|
|
> 1
|
|
- alert: PrometheusRemoteWriteBehind
|
|
expr: |-
|
|
# Without max_over_time, failed scrapes could create false negatives, see
|
|
# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
|
|
(
|
|
max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])
|
|
- ignoring(remote_name, url) group_right
|
|
max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])
|
|
)
|
|
> 120
|
|
- alert: PrometheusRemoteWriteDesiredShards
|
|
expr: |-
|
|
# Without max_over_time, failed scrapes could create false negatives, see
|
|
# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
|
|
(
|
|
max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])
|
|
>
|
|
max_over_time(prometheus_remote_storage_shards_max{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m])
|
|
)
|
|
- alert: PrometheusRuleFailures
|
|
expr: increase(prometheus_rule_evaluation_failures_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusMissingRuleEvaluations
|
|
expr: increase(prometheus_rule_group_iterations_missed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusTargetLimitHit
|
|
expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusLabelLimitHit
|
|
expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusScrapeBodySizeLimitHit
|
|
expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusScrapeSampleLimitHit
|
|
expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0
|
|
- alert: PrometheusTargetSyncFailure
|
|
expr: increase(prometheus_target_sync_failed_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[30m]) > 0
|
|
- alert: PrometheusHighQueryLoad
|
|
expr: avg_over_time(prometheus_engine_queries{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}"}[5m]) > 0.8
|
|
- alert: PrometheusErrorSendingAlertsToAnyAlertmanager
|
|
expr: |-
|
|
min without (alertmanager) (
|
|
rate(prometheus_notifications_errors_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}",alertmanager!~``}[5m])
|
|
/
|
|
rate(prometheus_notifications_sent_total{job="prometheus-kube-prometheus-prometheus",namespace="${var.namespace}",alertmanager!~``}[5m])
|
|
)
|
|
* 100
|
|
> 3
|
|
EOF
|
|
}
|
|
patches {
|
|
target {
|
|
kind = "ServiceMonitor"
|
|
name = "prometheus-community-prometheus-node-exporter"
|
|
}
|
|
patch = <<-EOF
|
|
- op: replace
|
|
path: /spec/selector/matchLabels/app.kubernetes.io~1instance
|
|
value: "${var.instance}"
|
|
EOF
|
|
}
|
|
}
|
|
|
|
data "kustomization_overlay" "data_no_ns" {
|
|
resources = [for file in fileset(path.module, "*.yaml"): file if (length(regexall("ClusterRole",file))>0 || length(regexall("Service_prometheus",file))>0)]
|
|
|
|
patches {
|
|
target {
|
|
kind = "ClusterRoleBinding"
|
|
name = "prometheus-kube-prometheus-prometheus"
|
|
}
|
|
patch = local.rb-patch
|
|
}
|
|
}
|