Project 7: Monitoring Stack

Context

Your infrastructure is growing and you need to set up a complete monitoring solution. This project deploys Prometheus, Grafana, and AlertManager on Kubernetes (EKS).

Requirements

Collection of infrastructure and application metrics
Pre-configured Grafana dashboards
Alerting with Slack/PagerDuty notifications
30-day metrics retention
High availability

Target architecture

Project structure

projet-monitoring/
├── main.tf
├── variables.tf
├── outputs.tf
├── versions.tf
├── namespace.tf
├── prometheus.tf
├── grafana.tf
├── alertmanager.tf
├── thanos.tf
├── dashboards/
│   ├── kubernetes.json
│   ├── node-exporter.json
│   └── application.json
├── alerts/
│   ├── infrastructure.yaml
│   └── application.yaml
└── terraform.tfvars

Implementation

variables.tf

variable "cluster_name" {
  description = "EKS cluster name"
  type        = string
}

variable "monitoring_namespace" {
  description = "Kubernetes namespace for monitoring"
  type        = string
  default     = "monitoring"
}

variable "prometheus_retention_days" {
  description = "Prometheus data retention in days"
  type        = number
  default     = 15
}

variable "grafana_admin_password" {
  description = "Grafana admin password"
  type        = string
  sensitive   = true
}

variable "slack_webhook_url" {
  description = "Slack webhook URL for alerts"
  type        = string
  sensitive   = true
}

variable "pagerduty_service_key" {
  description = "PagerDuty service key"
  type        = string
  sensitive   = true
  default     = ""
}

variable "enable_thanos" {
  description = "Enable Thanos for long-term storage"
  type        = bool
  default     = true
}

variable "thanos_bucket_name" {
  description = "S3 bucket for Thanos"
  type        = string
}

variable "grafana_domain" {
  description = "Domain for Grafana ingress"
  type        = string
}

namespace.tf

resource "kubernetes_namespace" "monitoring" {
  metadata {
    name = var.monitoring_namespace

    labels = {
      name        = var.monitoring_namespace
      environment = var.environment
    }
  }
}

prometheus.tf

# Helm release pour kube-prometheus-stack
resource "helm_release" "kube_prometheus_stack" {
  name       = "kube-prometheus-stack"
  namespace  = kubernetes_namespace.monitoring.metadata[0].name
  repository = "https://prometheus-community.github.io/helm-charts"
  chart      = "kube-prometheus-stack"
  version    = "54.0.1"

  values = [
    templatefile("${path.module}/values/prometheus-stack.yaml", {
      retention_days           = var.prometheus_retention_days
      slack_webhook_url        = var.slack_webhook_url
      pagerduty_service_key    = var.pagerduty_service_key
      grafana_admin_password   = var.grafana_admin_password
      grafana_domain           = var.grafana_domain
      enable_thanos            = var.enable_thanos
      thanos_bucket_name       = var.thanos_bucket_name
      thanos_service_account   = kubernetes_service_account.thanos[0].metadata[0].name
    })
  ]

  set {
    name  = "prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues"
    value = "false"
  }

  set {
    name  = "prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues"
    value = "false"
  }

  depends_on = [kubernetes_namespace.monitoring]
}

values/prometheus-stack.yaml

# Prometheus Configuration
prometheus:
  prometheusSpec:
    retention: ${retention_days}d
    retentionSize: "40GB"
    
    resources:
      requests:
        memory: 2Gi
        cpu: 500m
      limits:
        memory: 4Gi
        cpu: 2000m
    
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: gp3
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 50Gi
    
    # Haute disponibilité
    replicas: 2
    podAntiAffinity: "hard"
    
    # Thanos sidecar
    %{ if enable_thanos }
    thanos:
      image: quay.io/thanos/thanos:v0.32.5
      objectStorageConfig:
        existingSecret:
          name: thanos-objstore-config
          key: objstore.yml
    %{ endif }

    # Règles d'alerte additionnelles
    additionalAlertRelabelConfigs:
      - source_labels: [severity]
        regex: (.+)
        target_label: severity
        action: replace

# AlertManager Configuration
alertmanager:
  alertmanagerSpec:
    replicas: 2
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: gp3
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 10Gi

  config:
    global:
      resolve_timeout: 5m
      slack_api_url: '${slack_webhook_url}'
      
    route:
      group_by: ['alertname', 'namespace', 'severity']
      group_wait: 10s
      group_interval: 5m
      repeat_interval: 12h
      receiver: 'slack-notifications'
      routes:
        - match:
            severity: critical
          receiver: 'pagerduty-critical'
          continue: true
        - match:
            severity: critical
          receiver: 'slack-critical'
        - match:
            severity: warning
          receiver: 'slack-warnings'

    receivers:
      - name: 'slack-notifications'
        slack_configs:
          - channel: '#alerts'
            send_resolved: true
            title: '{{ template "slack.default.title" . }}'
            text: '{{ template "slack.default.text" . }}'
            
      - name: 'slack-critical'
        slack_configs:
          - channel: '#alerts-critical'
            send_resolved: true
            color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'
            
      - name: 'slack-warnings'
        slack_configs:
          - channel: '#alerts-warning'
            send_resolved: true
            
      %{ if pagerduty_service_key != "" }
      - name: 'pagerduty-critical'
        pagerduty_configs:
          - service_key: '${pagerduty_service_key}'
            severity: '{{ .CommonLabels.severity }}'
      %{ endif }

# Grafana Configuration
grafana:
  enabled: true
  replicas: 2
  
  adminPassword: '${grafana_admin_password}'
  
  ingress:
    enabled: true
    ingressClassName: alb
    annotations:
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/target-type: ip
      alb.ingress.kubernetes.io/certificate-arn: ${certificate_arn}
      alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
    hosts:
      - ${grafana_domain}
    tls:
      - hosts:
          - ${grafana_domain}

  persistence:
    enabled: true
    storageClassName: gp3
    size: 10Gi

  datasources:
    datasources.yaml:
      apiVersion: 1
      datasources:
        - name: Prometheus
          type: prometheus
          url: http://kube-prometheus-stack-prometheus:9090
          access: proxy
          isDefault: true
        - name: Loki
          type: loki
          url: http://loki:3100
          access: proxy
        - name: CloudWatch
          type: cloudwatch
          jsonData:
            authType: default
            defaultRegion: eu-west-1

  dashboardProviders:
    dashboardproviders.yaml:
      apiVersion: 1
      providers:
        - name: 'default'
          orgId: 1
          folder: ''
          type: file
          disableDeletion: false
          editable: true
          options:
            path: /var/lib/grafana/dashboards/default

  dashboards:
    default:
      kubernetes-cluster:
        gnetId: 7249
        revision: 1
        datasource: Prometheus
      node-exporter:
        gnetId: 1860
        revision: 31
        datasource: Prometheus
      nginx-ingress:
        gnetId: 9614
        revision: 1
        datasource: Prometheus
      postgres:
        gnetId: 9628
        revision: 7
        datasource: Prometheus

# Node Exporter
nodeExporter:
  enabled: true

# Kube State Metrics
kubeStateMetrics:
  enabled: true

# Default Rules
defaultRules:
  create: true
  rules:
    alertmanager: true
    etcd: true
    configReloaders: true
    general: true
    k8s: true
    kubeApiserverAvailability: true
    kubeApiserverBurnrate: true
    kubeApiserverHistogram: true
    kubeApiserverSlos: true
    kubeControllerManager: true
    kubelet: true
    kubeProxy: true
    kubePrometheusGeneral: true
    kubePrometheusNodeRecording: true
    kubernetesApps: true
    kubernetesResources: true
    kubernetesStorage: true
    kubernetesSystem: true
    kubeSchedulerAlerting: true
    kubeSchedulerRecording: true
    kubeStateMetrics: true
    network: true
    node: true
    nodeExporterAlerting: true
    nodeExporterRecording: true
    prometheus: true
    prometheusOperator: true

thanos.tf

# S3 Bucket pour Thanos
resource "aws_s3_bucket" "thanos" {
  count  = var.enable_thanos ? 1 : 0
  bucket = var.thanos_bucket_name

  tags = {
    Name = var.thanos_bucket_name
  }
}

resource "aws_s3_bucket_lifecycle_configuration" "thanos" {
  count  = var.enable_thanos ? 1 : 0
  bucket = aws_s3_bucket.thanos[0].id

  rule {
    id     = "cleanup"
    status = "Enabled"

    transition {
      days          = 30
      storage_class = "STANDARD_IA"
    }

    transition {
      days          = 90
      storage_class = "GLACIER"
    }

    expiration {
      days = 365
    }
  }
}

# IAM Role pour Thanos (IRSA)
resource "aws_iam_role" "thanos" {
  count = var.enable_thanos ? 1 : 0
  name  = "${local.name_prefix}-thanos"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          Federated = var.oidc_provider_arn
        }
        Action = "sts:AssumeRoleWithWebIdentity"
        Condition = {
          StringEquals = {
            "${var.oidc_provider}:sub" = "system:serviceaccount:${var.monitoring_namespace}:thanos"
            "${var.oidc_provider}:aud" = "sts.amazonaws.com"
          }
        }
      }
    ]
  })
}

resource "aws_iam_role_policy" "thanos_s3" {
  count = var.enable_thanos ? 1 : 0
  name  = "thanos-s3-access"
  role  = aws_iam_role.thanos[0].id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "s3:GetObject",
          "s3:PutObject",
          "s3:DeleteObject",
          "s3:ListBucket"
        ]
        Resource = [
          aws_s3_bucket.thanos[0].arn,
          "${aws_s3_bucket.thanos[0].arn}/*"
        ]
      }
    ]
  })
}

# Service Account pour Thanos
resource "kubernetes_service_account" "thanos" {
  count = var.enable_thanos ? 1 : 0

  metadata {
    name      = "thanos"
    namespace = kubernetes_namespace.monitoring.metadata[0].name

    annotations = {
      "eks.amazonaws.com/role-arn" = aws_iam_role.thanos[0].arn
    }
  }
}

# Secret pour configuration Thanos
resource "kubernetes_secret" "thanos_objstore_config" {
  count = var.enable_thanos ? 1 : 0

  metadata {
    name      = "thanos-objstore-config"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
  }

  data = {
    "objstore.yml" = yamlencode({
      type = "s3"
      config = {
        bucket   = var.thanos_bucket_name
        endpoint = "s3.${var.region}.amazonaws.com"
        region   = var.region
      }
    })
  }
}

# Thanos Query (pour requêter les métriques historiques)
resource "helm_release" "thanos" {
  count = var.enable_thanos ? 1 : 0

  name       = "thanos"
  namespace  = kubernetes_namespace.monitoring.metadata[0].name
  repository = "https://charts.bitnami.com/bitnami"
  chart      = "thanos"
  version    = "12.13.3"

  values = [
    yamlencode({
      query = {
        enabled = true
        stores = [
          "dnssrv+_grpc._tcp.kube-prometheus-stack-thanos-discovery.${var.monitoring_namespace}.svc.cluster.local"
        ]
      }
      
      queryFrontend = {
        enabled = true
      }
      
      bucketweb = {
        enabled = true
      }
      
      compactor = {
        enabled = true
        persistence = {
          enabled = true
          size    = "20Gi"
        }
      }
      
      storegateway = {
        enabled = true
        persistence = {
          enabled = true
          size    = "20Gi"
        }
      }
      
      existingObjstoreSecret = kubernetes_secret.thanos_objstore_config[0].metadata[0].name
    })
  ]

  depends_on = [helm_release.kube_prometheus_stack]
}

alerts/infrastructure.yaml

# Alertes personnalisées
groups:
  - name: infrastructure
    rules:
      # Noeud indisponible
      - alert: NodeNotReady
        expr: kube_node_status_condition{condition="Ready",status="true"} == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Node {{ $labels.node }} is not ready"
          description: "Node {{ $labels.node }} has been unready for more than 5 minutes."

      # CPU élevé
      - alert: HighCPUUsage
        expr: (1 - avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100 > 80
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU usage is above 80% for more than 10 minutes."

      # Mémoire faible
      - alert: LowMemory
        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 < 15
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low memory on {{ $labels.instance }}"
          description: "Available memory is below 15%."

      # Disque plein
      - alert: DiskSpaceLow
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Disk space low on {{ $labels.instance }}"
          description: "Available disk space is below 10%."

      # Pod en CrashLoopBackOff
      - alert: PodCrashLooping
        expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping"
          description: "Pod has restarted more than 5 times in the last 15 minutes."

      # PVC presque plein
      - alert: PersistentVolumeUsageHigh
        expr: (kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "PVC {{ $labels.persistentvolumeclaim }} usage is high"
          description: "PVC usage is above 85%."

outputs.tf

output "grafana_url" {
  description = "Grafana URL"
  value       = "https://${var.grafana_domain}"
}

output "prometheus_service" {
  description = "Prometheus service name"
  value       = "kube-prometheus-stack-prometheus.${var.monitoring_namespace}.svc.cluster.local:9090"
}

output "alertmanager_service" {
  description = "AlertManager service name"
  value       = "kube-prometheus-stack-alertmanager.${var.monitoring_namespace}.svc.cluster.local:9093"
}

output "thanos_query_service" {
  description = "Thanos Query service"
  value       = var.enable_thanos ? "thanos-query.${var.monitoring_namespace}.svc.cluster.local:9090" : "N/A"
}

Deployment

# Initialiser
terraform init

# Déployer
terraform apply

# Vérifier les pods
kubectl get pods -n monitoring

# Accéder à Grafana
echo "https://$(terraform output -raw grafana_url)"

Recommended dashboards

Dashboard	Grafana ID	Description
Kubernetes Cluster	7249	Cluster overview
Node Exporter Full	1860	Detailed system metrics
NGINX Ingress	9614	HTTP traffic and errors
PostgreSQL	9628	Database performance
CoreDNS	5926	Cluster DNS

Estimated costs

Resource	Monthly cost
EBS (Prometheus 50GB x2)	~$10
EBS (AlertManager 10GB x2)	~$2
EBS (Thanos 40GB)	~$4
S3 (Thanos 100GB)	~$3
Total	~$20/month

← Project 6 | Project 8: Multi-Cloud →

Context​

Requirements​

Target architecture​

Project structure​

Implementation​

variables.tf​

namespace.tf​

prometheus.tf​

values/prometheus-stack.yaml​

thanos.tf​

alerts/infrastructure.yaml​

outputs.tf​

Deployment​

Recommended dashboards​

Estimated costs​

Context

Requirements

Target architecture

Project structure

Implementation

variables.tf

namespace.tf

prometheus.tf

values/prometheus-stack.yaml

thanos.tf

alerts/infrastructure.yaml

outputs.tf

Deployment

Recommended dashboards

Estimated costs