Skip to main content

Advanced log management

Table of contents​

  1. Logging architecture
  2. journald - The systemd journal
  3. rsyslog - Advanced configuration
  4. Log centralization
  5. Analysis and monitoring
  6. Practical exercises

1 - Logging architecture​

Overview​

Standard log files​

FileContent
/var/log/syslogGeneral system messages (Debian)
/var/log/messagesSystem messages (RHEL)
/var/log/auth.logAuthentication (Debian)
/var/log/secureAuthentication (RHEL)
/var/log/kern.logKernel messages
/var/log/dmesgBoot messages
/var/log/cronCron jobs
/var/log/nginx/Nginx logs
/var/log/apache2/Apache logs

Severity levels (syslog)​

LevelCodeDescription
emerg0System unusable
alert1Immediate action required
crit2Critical conditions
err3Errors
warning4Warnings
notice5Normal but significant
info6Informational
debug7Debug

🔝 Back to table of contents​


2 - journald - The systemd journal​

Basic commands​

# Voir tous les logs
journalctl

# Logs du boot actuel
journalctl -b

# Logs du boot précédent
journalctl -b -1

# Suivre en temps réel
journalctl -f

# Dernières entrées
journalctl -n 50

# Depuis une date
journalctl --since "2024-01-15 10:00:00"
journalctl --since "1 hour ago"
journalctl --since today

Advanced filters​

# Par unité systemd
journalctl -u nginx.service
journalctl -u nginx -u php-fpm

# Par priorité
journalctl -p err      # Erreurs et plus grave
journalctl -p warning..err  # De warning à err

# Par PID
journalctl _PID=1234

# Par utilisateur
journalctl _UID=1000

# Par exécutable
journalctl /usr/bin/sudo

# Kernel
journalctl -k

Output format​

# JSON
journalctl -o json-pretty

# Format court
journalctl -o short

# Avec tous les champs
journalctl -o verbose

# Export binaire
journalctl -o export > logs.export

journald configuration​

# /etc/systemd/journald.conf

[Journal]
# Stockage persistent (survive aux reboots)
Storage=persistent

# Taille maximum
SystemMaxUse=500M
RuntimeMaxUse=100M

# Rotation
MaxRetentionSec=1month

# Compression
Compress=yes

# Forward vers syslog
ForwardToSyslog=yes
# Appliquer
systemctl restart systemd-journald

# Vérifier l'espace utilisé
journalctl --disk-usage

# Nettoyer les anciens logs
journalctl --vacuum-size=200M
journalctl --vacuum-time=7d

🔝 Back to table of contents​


3 - rsyslog - Advanced configuration​

Configuration structure​

# /etc/rsyslog.conf
# /etc/rsyslog.d/*.conf

# Modules
module(load="imuxsock")    # Socket local
module(load="imklog")      # Kernel
module(load="imtcp")       # Réception TCP
module(load="imudp")       # Réception UDP

# Templates
template(name="CustomFormat" type="string"
    string="%TIMESTAMP% %HOSTNAME% %syslogtag%%msg%\n")

# Règles
facility.priority    destination

Rule syntax​

# Format: facility.priority  action

# Exemples
*.info                       /var/log/messages
authpriv.*                   /var/log/secure
mail.*                       /var/log/maillog
cron.*                       /var/log/cron
*.emerg                      :omusrmsg:*

# Opérateurs de priorité
*.=debug                     # Exactement debug
*.!crit                      # Tout sauf crit et plus
*.crit;*.!=info              # crit et plus, sauf info

Property-based filters​

# Filtrer par programme
:programname, isequal, "nginx" /var/log/nginx/nginx.log

# Filtrer par message
:msg, contains, "error" /var/log/errors.log

# Expression régulière
:msg, regex, "failed.*authentication" /var/log/auth-failures.log

Custom templates​

# Template JSON
template(name="JsonFormat" type="list") {
    constant(value="{")
    constant(value="\"timestamp\":\"")  property(name="timestamp" dateFormat="rfc3339")
    constant(value="\",\"host\":\"")    property(name="hostname")
    constant(value="\",\"severity\":\"")property(name="syslogseverity-text")
    constant(value="\",\"facility\":\"")property(name="syslogfacility-text")
    constant(value="\",\"tag\":\"")     property(name="syslogtag")
    constant(value="\",\"message\":\"") property(name="msg" format="json")
    constant(value="\"}\n")
}

# Utiliser le template
*.* /var/log/all-json.log;JsonFormat

🔝 Back to table of contents​


4 - Log centralization​

Centralized architecture​

Central server configuration​

# /etc/rsyslog.d/server.conf

# Charger les modules
module(load="imtcp")
module(load="imudp")

# Écouter sur TCP 514
input(type="imtcp" port="514")

# Écouter sur UDP 514
input(type="imudp" port="514")

# Template pour séparer par host
template(name="PerHostLog" type="string"
    string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log")

# Stocker les logs distants
*.* ?PerHostLog

Client configuration​

# /etc/rsyslog.d/remote.conf

# Envoyer tous les logs au serveur central
*.* @@logserver.example.com:514  # TCP (@@)
# *.* @logserver.example.com:514   # UDP (@)

# Queue pour fiabilité
$ActionQueueType LinkedList
$ActionQueueFileName remote
$ActionQueueMaxDiskSpace 1g
$ActionQueueSaveOnShutdown on
$ActionResumeRetryCount -1

Secure transport (TLS)​

# Serveur - /etc/rsyslog.d/tls-server.conf
module(load="imtcp"
    StreamDriver.Name="gtls"
    StreamDriver.Mode="1"
    StreamDriver.Authmode="x509/name")

global(
    DefaultNetstreamDriver="gtls"
    DefaultNetstreamDriverCAFile="/etc/pki/rsyslog/ca.pem"
    DefaultNetstreamDriverCertFile="/etc/pki/rsyslog/server-cert.pem"
    DefaultNetstreamDriverKeyFile="/etc/pki/rsyslog/server-key.pem"
)

input(type="imtcp" port="6514")

# Client - /etc/rsyslog.d/tls-client.conf
global(
    DefaultNetstreamDriver="gtls"
    DefaultNetstreamDriverCAFile="/etc/pki/rsyslog/ca.pem"
    DefaultNetstreamDriverCertFile="/etc/pki/rsyslog/client-cert.pem"
    DefaultNetstreamDriverKeyFile="/etc/pki/rsyslog/client-key.pem"
)

action(type="omfwd"
    target="logserver.example.com"
    port="6514"
    protocol="tcp"
    StreamDriver="gtls"
    StreamDriverMode="1"
    StreamDriverAuthMode="x509/name")

🔝 Back to table of contents​


5 - Analysis and monitoring​

CLI tools​

# grep avec contexte
grep -B2 -A2 "error" /var/log/syslog

# Compter les occurrences
grep -c "Failed password" /var/log/auth.log

# Top des IPs qui échouent
grep "Failed password" /var/log/auth.log | \
    awk '{print $(NF-3)}' | sort | uniq -c | sort -rn | head

# Logs par heure
awk '{print $1, $2, $3}' /var/log/syslog | cut -d: -f1,2 | uniq -c

# Analyse avec awk
awk '/error/ {count++} END {print count}' /var/log/syslog

Logwatch - Automatic reports​

# Installer
apt install logwatch

# Lancer un rapport
logwatch --detail High --mailto [email protected] --range today

# Configuration
# /etc/logwatch/conf/logwatch.conf
Output = mail
MailTo = [email protected]
Detail = Med
Range = yesterday

Fail2ban - Automatic protection​

# Installer
apt install fail2ban

# /etc/fail2ban/jail.local
[sshd]
enabled = true
port = ssh
filter = sshd
logpath = /var/log/auth.log
maxretry = 5
bantime = 3600
findtime = 600

# Commandes
fail2ban-client status
fail2ban-client status sshd
fail2ban-client set sshd unbanip 192.168.1.100

Rotation with logrotate​

# /etc/logrotate.d/myapp
/var/log/myapp/*.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    create 640 myapp myapp
    sharedscripts
    postrotate
        systemctl reload myapp || true
    endscript
}

# Tester
logrotate -d /etc/logrotate.d/myapp
logrotate -f /etc/logrotate.d/myapp

🔝 Back to table of contents​


6 - Practical exercises​

Exercise 1: journalctl filters​

Find all SSH authentication failures since yesterday:

Solution
journalctl -u sshd --since yesterday | grep -i "failed\|invalid"

# Ou plus précis
journalctl -u sshd --since yesterday -p warning

Exercise 2: rsyslog configuration​

Configure rsyslog to send all auth logs to a separate file:

Solution
# /etc/rsyslog.d/10-auth.conf
authpriv.*    /var/log/auth-custom.log

# Redémarrer
systemctl restart rsyslog

Quiz​

Q1. Which command displays kernel logs in real time?

Answer

journalctl -kf or dmesg -w

Q2. What is the difference between @ and @@ in rsyslog?

Answer

@ = UDP, @@ = TCP

🔝 Back to table of contents​


Key takeaways​

  • journald: systemd journal, powerful queries with journalctl
  • rsyslog: flexible log processing and routing
  • Storage=persistent to keep logs after reboot
  • Centralization with TCP/TLS for reliability and security
  • logrotate to avoid filling the disk
  • fail2ban for automatic protection

🔝 Back to table of contents​

← Previous chapter | Next chapter: Performance →