Exercises and Projects

---

1 - Hands-on exercises

Exercise 1: Prometheus configuration

Objective: Configure Prometheus with targets and rules.

Solution

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'applications'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2

# rules/alerts.yml
groups:
  - name: application
    rules:
      - alert: HighErrorRate
        expr: sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate detected"

---

Exercise 2: Grafana Dashboard

Objective: Create a dashboard with the Golden Signals.

Solution

{
  "title": "Service Overview",
  "panels": [
    {
      "title": "Request Rate",
      "type": "timeseries",
      "targets": [
        {
          "expr": "sum(rate(http_requests_total{service=\"$service\"}[5m]))",
          "legendFormat": "Requests/s"
        }
      ]
    },
    {
      "title": "Error Rate",
      "type": "stat",
      "targets": [
        {
          "expr": "sum(rate(http_requests_total{service=\"$service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{service=\"$service\"}[5m])) * 100"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "thresholds": {
            "steps": [
              { "color": "green", "value": 0 },
              { "color": "yellow", "value": 1 },
              { "color": "red", "value": 5 }
            ]
          }
        }
      }
    },
    {
      "title": "P99 Latency",
      "type": "timeseries",
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{service=\"$service\"}[5m])) by (le))",
          "legendFormat": "P99"
        }
      ],
      "fieldConfig": {
        "defaults": { "unit": "s" }
      }
    },
    {
      "title": "CPU Saturation",
      "type": "gauge",
      "targets": [
        {
          "expr": "100 - avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100"
        }
      ]
    }
  ],
  "templating": {
    "list": [
      {
        "name": "service",
        "type": "query",
        "query": "label_values(http_requests_total, service)"
      }
    ]
  }
}

---

Exercise 3: LogQL Queries

Objective: Write advanced LogQL queries.

Solution

# Errors from the last 5 minutes
{namespace="production", level="error"} |= "error"

# Parse JSON and filter
{app="api"} | json | status >= 500

# Top 10 endpoints with errors
topk(10, sum(rate({app="api"} | json | status >= 500 [1h])) by (endpoint))

# P99 latency per endpoint
quantile_over_time(0.99, 
  {app="api"} | json | unwrap latency [5m]
) by (endpoint)

# Logs with trace correlation
{app="api"} |= "error" | json | line_format "{{.trace_id}} - {{.message}}"

---

Exercise 4: Alertmanager Routing

Objective: Configure complex routing.

Solution

# alertmanager.yml
route:
  receiver: 'default'
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    # Critical -> PagerDuty + Slack
    - receiver: 'pagerduty-critical'
      match:
        severity: critical
      continue: true
    
    - receiver: 'slack-critical'
      match:
        severity: critical
    
    # Warning -> Slack only
    - receiver: 'slack-warning'
      match:
        severity: warning
    
    # Team-based routing
    - receiver: 'team-backend'
      match_re:
        service: 'api|database|cache'
    
    - receiver: 'team-frontend'
      match:
        service: 'web'

receivers:
  - name: 'default'
    slack_configs:
      - channel: '#alerts'
      
  - name: 'pagerduty-critical'
    pagerduty_configs:
      - routing_key: 'xxx'
        severity: critical
        
  - name: 'slack-critical'
    slack_configs:
      - channel: '#alerts-critical'
        
  - name: 'slack-warning'
    slack_configs:
      - channel: '#alerts-warning'
        
  - name: 'team-backend'
    slack_configs:
      - channel: '#backend-alerts'
        
  - name: 'team-frontend'
    slack_configs:
      - channel: '#frontend-alerts'

inhibit_rules:
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['alertname', 'service']

---

2 - Complete project: Observability Stack

Architecture

Complete Docker Compose

# docker-compose.yml
version: '3.8'

services:
  # Prometheus
  prometheus:
    image: prom/prometheus:v2.47.0
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules:/etc/prometheus/rules
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=15d'

  # Loki
  loki:
    image: grafana/loki:2.9.0
    ports:
      - "3100:3100"
    volumes:
      - ./loki/loki-config.yml:/etc/loki/local-config.yaml
      - loki_data:/loki

  # Tempo
  tempo:
    image: grafana/tempo:2.3.0
    ports:
      - "3200:3200"
      - "4317:4317"
    volumes:
      - ./tempo/tempo-config.yml:/etc/tempo/tempo.yaml
    command: ["-config.file=/etc/tempo/tempo.yaml"]

  # Promtail
  promtail:
    image: grafana/promtail:2.9.0
    volumes:
      - ./promtail/promtail-config.yml:/etc/promtail/config.yml
      - /var/log:/var/log:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro

  # OTel Collector
  otel-collector:
    image: otel/opentelemetry-collector:0.88.0
    ports:
      - "4318:4318"
    volumes:
      - ./otel/otel-collector-config.yml:/etc/otel/config.yaml
    command: ["--config=/etc/otel/config.yaml"]

  # Grafana
  grafana:
    image: grafana/grafana:10.2.0
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
      - grafana_data:/var/lib/grafana

  # Alertmanager
  alertmanager:
    image: prom/alertmanager:v0.26.0
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml

  # Sample Application
  api:
    build: ./app
    ports:
      - "8080:8080"
    environment:
      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318

volumes:
  prometheus_data:
  loki_data:
  grafana_data:

OTel Collector configuration

# otel/otel-collector-config.yml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024

exporters:
  prometheus:
    endpoint: "0.0.0.0:8889"
    
  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp/tempo]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

Instrumented application

// app/tracing.js
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-http');
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');

const sdk = new NodeSDK({
  serviceName: 'api-service',
  traceExporter: new OTLPTraceExporter(),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter(),
    exportIntervalMillis: 10000,
  }),
  instrumentations: [getNodeAutoInstrumentations()],
});

sdk.start();

---

3 - Review quiz

1. What are the 3 pillars of observability?

2. What is the difference between rate() and irate()?

3. How do you calculate a 99th percentile in PromQL?

4. What is the difference between Loki and Elasticsearch?

5. What is sampling in tracing?

Answers

1. Metrics, Logs, Traces

2. rate() calculates the average rate over the period, irate() uses the last 2 points (instantaneous).

3. histogram_quantile(0.99, sum(rate(metric_bucket[5m])) by (le))

4. Loki only indexes labels (lightweight, cost-effective), Elasticsearch does full-text indexing (powerful but expensive).

5. Sampling is the percentage of traces captured to reduce costs and data volume.

---

4 - Certifications

Recommended certifications

Certification	Provider	Focus
Prometheus Certified Associate (PCA)	CNCF	Prometheus
Grafana Certified Professional	Grafana Labs	Grafana
Elastic Certified Engineer	Elastic	ELK Stack
AWS Certified DevOps Engineer	AWS	CloudWatch

Resources

• Prometheus Documentation
• Grafana Tutorials
• OpenTelemetry
• SRE Books (Google)

---

Course summary

Congratulations! You have completed the Monitoring & Logs course.

You now have a solid grasp of:

• The 3 pillars of observability
• Prometheus and PromQL
• Grafana dashboards
• ELK Stack and Loki
• Alertmanager configuration
• Distributed Tracing
• APM and profiling
• Best practices

Next steps

• Deploy a complete stack
• Create dashboards for your services
• Configure SLO-based alerts
• Explore auto-remediation

---

← Back to table of contents