LAB-156
Prometheus · Alertmanager · Python · kubectl
Prometheus Alert gọi Remediation Script qua Webhook
🎯 Mục tiêu: Xây dựng pipeline hoàn chỉnh: Prometheus phát hiện pod error rate cao → Alertmanager gửi webhook → Python server nhận alert → tự động restart deployment.
🧰 Công cụ / nền tảng: minikube hoặc kind, kubectl, Helm, Python 3.8+, Prometheus stack (kube-prometheus-stack).
📦 Chuẩn bị:
# Cài minikube nếu chưa có
curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
sudo install minikube-linux-amd64 /usr/local/bin/minikube
minikube start --memory=4096 --cpus=2
# Cài Prometheus stack qua Helm
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm install kube-prom prometheus-community/kube-prometheus-stack \
--namespace monitoring --create-namespace \
--set alertmanager.enabled=true \
--set prometheus.prometheusSpec.retention=2h
# Xác nhận các pod đang running
kubectl -n monitoring get pods | grep -E 'prometheus|alertmanager'▶️ Các bước:
# BƯỚC 1: Tạo webhook server (Python Flask)
mkdir -p auto-remediation && cd auto-remediation
cat > webhook_server.py << 'PYEOF'
#!/usr/bin/env python3
"""
Auto-remediation webhook server.
Nhận alert từ Alertmanager, thực hiện hành động tự động.
"""
from flask import Flask, request, jsonify
import subprocess
import logging
import json
app = Flask(__name__)
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
ALLOWED_ACTIONS = {
"HighErrorRate": {"action": "rollout_restart", "namespace": "default"},
"PodCrashLooping": {"action": "rollout_restart", "namespace": "default"},
"HighMemoryUsage": {"action": "scale_up", "namespace": "default"},
}
def rollout_restart(namespace: str, deployment: str) -> dict:
cmd = ["kubectl", "rollout", "restart", f"deployment/{deployment}", "-n", namespace]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
return {"returncode": result.returncode, "stdout": result.stdout, "stderr": result.stderr}
def scale_up(namespace: str, deployment: str, replicas: int = 3) -> dict:
cmd = ["kubectl", "scale", f"deployment/{deployment}", f"--replicas={replicas}", "-n", namespace]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
return {"returncode": result.returncode, "stdout": result.stdout}
@app.route('/webhook', methods=['POST'])
def handle_alert():
data = request.get_json(force=True)
alerts = data.get('alerts', [])
results = []
for alert in alerts:
name = alert.get('labels', {}).get('alertname', '')
deployment = alert.get('labels', {}).get('deployment', 'unknown')
status = alert.get('status', '') # firing | resolved
app.logger.info(f"Received alert: {name} status={status} deployment={deployment}")
if status != 'firing':
results.append({"alert": name, "skipped": "not firing"})
continue
action_cfg = ALLOWED_ACTIONS.get(name)
if not action_cfg:
app.logger.warning(f"No action configured for alert: {name}")
results.append({"alert": name, "skipped": "no action configured"})
continue
if action_cfg["action"] == "rollout_restart":
result = rollout_restart(action_cfg["namespace"], deployment)
elif action_cfg["action"] == "scale_up":
result = scale_up(action_cfg["namespace"], deployment)
else:
result = {"error": "unknown action"}
app.logger.info(f"Action result: {json.dumps(result)}")
results.append({"alert": name, "action": action_cfg["action"], "result": result})
return jsonify({"processed": len(results), "results": results}), 200
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5001)
PYEOF
pip install flask
python webhook_server.py & # chạy nền trên port 5001# BƯỚC 2: Expose webhook agar Alertmanager trong cluster có thể gọi
# Lấy IP của host machine trong minikube network
WEBHOOK_HOST=$(minikube ssh "route -n | awk '/^0.0.0.0/{print \$2}'" 2>/dev/null || echo "192.168.49.1")
echo "Webhook host IP: $WEBHOOK_HOST"# BƯỚC 3: Cấu hình Alertmanager gửi đến webhook
cat > alertmanager-config.yaml << EOF
apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
name: auto-remediation
namespace: monitoring
spec:
route:
groupBy: ['alertname', 'deployment']
groupWait: 10s
groupInterval: 30s
repeatInterval: 1h
receiver: 'auto-remediation-webhook'
matchers:
- name: severity
value: critical
receivers:
- name: 'auto-remediation-webhook'
webhookConfigs:
- url: 'http://${WEBHOOK_HOST}:5001/webhook'
sendResolved: true
httpConfig:
followRedirects: true
EOF
kubectl apply -f alertmanager-config.yaml# BƯỚC 4: Tạo PrometheusRule để fire alert test
cat > test-alert-rule.yaml << 'EOF'
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: auto-remediation-rules
namespace: monitoring
labels:
release: kube-prom # label phải match với prometheus selector
spec:
groups:
- name: auto-remediation
interval: 15s
rules:
- alert: HighErrorRate
# Trigger thủ công bằng cách tạo metric giả
expr: vector(1) # luôn firing — chỉ dùng để test
for: 15s
labels:
severity: critical
deployment: test-app
annotations:
summary: "High error rate on {{ $labels.deployment }}"
description: "Error rate > 5% for 1 minute"
EOF
kubectl apply -f test-alert-rule.yaml# BƯỚC 5: Tạo deployment test để webhook có thứ để restart
kubectl create deployment test-app --image=nginx:alpine --replicas=2
# BƯỚC 6: Quan sát luồng
# Xem alert trong Prometheus UI:
kubectl -n monitoring port-forward svc/kube-prom-kube-prometheus-prometheus 9090 &
# Mở http://localhost:9090/alerts
# Xem Alertmanager:
kubectl -n monitoring port-forward svc/kube-prom-kube-prometheus-alertmanager 9093 &
# Mở http://localhost:9093
# Xem log webhook server:
# (terminal chạy webhook_server.py sẽ in log khi nhận alert)✅ Kết quả mong đợi:
# Log webhook server in ra:
# 2026-05-23 10:xx:xx INFO Received alert: HighErrorRate status=firing deployment=test-app
# 2026-05-23 10:xx:xx INFO Action result: {"returncode": 0, "stdout": "deployment.apps/test-app restarted\n"}
# Xác nhận deployment đã được restart:
kubectl rollout history deployment/test-app
# REVISION CHANGE-CAUSE
# 1
# 2 ← restart mới từ auto-remediation 🧹 Cleanup:
kubectl delete -f test-alert-rule.yaml
kubectl delete -f alertmanager-config.yaml
kubectl delete deployment test-app
kill $(lsof -t -i:5001) # dừng webhook server