pveum user add terraform@pve --password 'tokenP@ss'
pveum aclmod / -user terraform@pve -role PVEAdmin
pveum user token add terraform@pve tf-token --privsep 0
# → lưu secret UUID trả về

terraform {
  required_providers {
    proxmox = {
      source  = "bpg/proxmox"
      version = "~> 0.68"
    }
  }
}

provider "proxmox" {
  endpoint  = "https://10.0.0.11:8006/"
  api_token = "terraform@pve!tf-token=xxxxxxxx-xxxx"
  insecure  = true
  ssh {
    agent    = false
    username = "root"
    password = var.root_password
  }
}

# Clone VM từ template 9000
resource "proxmox_virtual_environment_vm" "web" {
  count     = 3
  name      = "web-${count.index + 1}"
  node_name = "pve0${(count.index % 3) + 1}"
  clone {
    vm_id = 9000
    full  = true
  }
  memory   { dedicated = 2048 }
  cpu      { cores = 2 }
  initialization {
    ip_config {
      ipv4 {
        address = "10.0.10.${50 + count.index}/24"
        gateway = "10.0.10.1"
      }
    }
    user_account {
      username = "ubuntu"
      keys     = [file("~/.ssh/id_rsa.pub")]
    }
  }
  network_device {
    bridge  = "vmbr1"
    vlan_id = 10
  }
}

output "web_ips" {
  value = [for v in proxmox_virtual_environment_vm.web :
           v.initialization[0].ip_config[0].ipv4[0].address]
}

terraform init
terraform plan
terraform apply -auto-approve
# → 3 VM web-1, web-2, web-3 tạo tự động

terraform destroy  # teardown

all:
  hosts:
    web-1: { ansible_host: 10.0.10.50 }
    web-2: { ansible_host: 10.0.10.51 }
    web-3: { ansible_host: 10.0.10.52 }
  vars:
    ansible_user: ubuntu
    ansible_ssh_private_key_file: ~/.ssh/id_rsa

- hosts: all
  become: yes
  tasks:
    - name: Install nginx
      apt:
        name: nginx
        update_cache: yes
        state: present
    - name: Start nginx
      service: { name: nginx, state: started, enabled: yes }
    - name: Deploy index.html
      copy:
        dest: /var/www/html/index.html
        content: "<h1>{{ inventory_hostname }}</h1>"

ansible-playbook -i inventory.yml site.yml

$ terraform apply
Plan: 3 to add, 0 to change, 0 to destroy.
proxmox_virtual_environment_vm.web[0]: Creation complete after 45s
proxmox_virtual_environment_vm.web[1]: Creation complete after 47s
proxmox_virtual_environment_vm.web[2]: Creation complete after 46s
Outputs:
web_ips = ["10.0.10.50/24", "10.0.10.51/24", "10.0.10.52/24"]

$ curl http://10.0.10.50
<h1>web-1</h1>

  Proxmox nodes          Prometheus          Grafana
  pve01 ─┐                  │                   │
  pve02 ─┼──→ PVE Exporter ─┤                   │
  pve03 ─┘     :9221        ├──→ scrape every 30s
                            │                   │
  PBS node ──→ pbs-exporter ┤                   │
                            │                   │
  Linux VM ──→ node_exporter┘                   │
                                                ▼
                            VM Grafana ────→ Dashboard UI :3000
                                     │
                                     └──→ Loki ←─── Promtail (VM)

# docker-compose.yml
services:
  prometheus:
    image: prom/prometheus:v3.1.0
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prom-data:/prometheus
    ports: ["9090:9090"]

  grafana:
    image: grafana/grafana:11.4.0
    ports: ["3000:3000"]
    environment:
      GF_SECURITY_ADMIN_PASSWORD: 'Gr@fana123'
    volumes:
      - grafana-data:/var/lib/grafana

  loki:
    image: grafana/loki:3.3.0
    ports: ["3100:3100"]

  pve-exporter:
    image: prompve/prometheus-pve-exporter:latest
    ports: ["9221:9221"]
    volumes:
      - ./pve.yml:/etc/pve.yml:ro

volumes:
  prom-data:
  grafana-data:

default:
  user: monitor@pve
  token_name: monitor-token
  token_value: 'xxxxxxxx-xxxx-xxxx'
  verify_ssl: false

global:
  scrape_interval: 30s

scrape_configs:
  - job_name: pve
    metrics_path: /pve
    params:
      module: [default]
      cluster: ['1']
      node: ['1']
    static_configs:
      - targets: [10.0.0.11, 10.0.0.12, 10.0.0.13]
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: pve-exporter:9221

  - job_name: node-exporter
    static_configs:
      - targets: [10.0.10.50:9100, 10.0.10.51:9100, 10.0.10.52:9100]

# alerts.yml
groups:
  - name: pve-critical
    rules:
      - alert: NodeDown
        expr: up{job="pve"} == 0
        for: 2m
        annotations:
          summary: "Proxmox node {{ $labels.instance }} down"
      - alert: HighCPU
        expr: pve_cpu_usage_ratio > 0.90
        for: 10m
      - alert: StorageFull
        expr: (pve_disk_usage_bytes / pve_disk_size_bytes) > 0.85
        for: 5m
      - alert: CephHealthWarn
        expr: ceph_health_status > 1
        for: 5m

# /etc/promtail/config.yml
clients:
  - url: http://monitor-vm:3100/loki/api/v1/push

scrape_configs:
  - job_name: pve-syslog
    static_configs:
      - targets: [localhost]
        labels:
          job: syslog
          host: pve01
          __path__: /var/log/syslog
  - job_name: pve-daemon
    journal:
      matches: _SYSTEMD_UNIT=pvedaemon.service
      labels: { job: pvedaemon }

$ curl http://pve-exporter:9221/pve?target=10.0.0.11 | head -20
# HELP pve_up Node/VM/Storage/... is online/running
# TYPE pve_up gauge
pve_up{id="node/pve01"} 1.0
pve_up{id="qemu/100"} 1.0
# HELP pve_cpu_usage_ratio CPU usage ratio
pve_cpu_usage_ratio{id="node/pve01"} 0.23
pve_cpu_usage_ratio{id="qemu/100"} 0.45
# HELP pve_memory_usage_bytes Memory usage
pve_memory_usage_bytes{id="node/pve01"} 137438953472

# AWS side (console hoặc CLI):
# 1. Tạo Customer Gateway (on-prem public IP)
aws ec2 create-customer-gateway \
    --type ipsec.1 --public-ip <ON_PREM_PUB_IP> --bgp-asn 65001

# 2. Tạo Virtual Private Gateway + attach VPC
aws ec2 create-vpn-gateway --type ipsec.1
aws ec2 attach-vpn-gateway --vpc-id vpc-xxx --vpn-gateway-id vgw-xxx

# 3. Tạo VPN connection → download config cho router on-prem
aws ec2 create-vpn-connection \
    --type ipsec.1 \
    --customer-gateway-id cgw-xxx \
    --vpn-gateway-id vgw-xxx \
    --options StaticRoutesOnly=false

# On-prem side: cấu hình VPN trên pfSense/Fortigate/VM Proxmox chạy StrongSwan
# Subnet on-prem: 10.0.0.0/16, Subnet AWS: 172.16.0.0/16

# Trên PBS:
proxmox-backup-manager s3-endpoint create aws-sg \
    --endpoint s3.ap-southeast-1.amazonaws.com \
    --access-key AKIAxxx --secret-key xxx

proxmox-backup-manager datastore create dr-s3 /mnt/s3-cache \
    --backend-type s3 --s3-endpoint aws-sg \
    --bucket hoatranlab-dr

# Sync job: copy từ local-ds → dr-s3 hàng đêm
proxmox-backup-manager sync-job create sync-to-s3 \
    --store dr-s3 --remote local --remote-store big-ds \
    --schedule 'daily'

# 1. Restore VM critical từ S3 backup lên EC2
# Tạo EC2 instance cho mỗi VM critical (pre-sized template)
aws ec2 run-instances --image-id ami-xxx \
    --instance-type m5.large --count 1 \
    --subnet-id subnet-dr --security-group-ids sg-xxx \
    --tag-specifications 'ResourceType=instance,Tags=[{Key=Role,Value=web-prod-dr}]'

# 2. Mount S3 bucket, restore file từ PBS backup
aws s3 cp s3://hoatranlab-dr/vm/100/latest.tar.zst /tmp/
tar xzf /tmp/latest.tar.zst -C /restore

# 3. Chuyển DNS: Route 53 failover record
aws route53 change-resource-record-sets --hosted-zone-id Zxxx \
    --change-batch file://failover-to-aws.json

# 4. Thông báo user, kiểm tra

$ aws ec2 describe-vpn-connections --query 'VpnConnections[*].[VpnConnectionId,State]'
[
    ["vpn-0abc123", "available"]
]

$ ping 172.16.10.5  # EC2 AD DC từ on-prem
PING 172.16.10.5: 64 bytes from 172.16.10.5: icmp_seq=1 ttl=253 time=48.2 ms
64 bytes from 172.16.10.5: icmp_seq=2 ttl=253 time=47.8 ms

$ proxmox-backup-manager sync-job status sync-to-s3
┌────────────┬──────────┬─────────┬──────────┐
│ Last Run   │ Duration │ Status  │ Size     │
├────────────┼──────────┼─────────┼──────────┤
│ 2026-04-23 │ 02:34:12 │ Success │ 78.4 GiB │
│ 02:00:00   │          │         │ (dedup)  │
└────────────┴──────────┴─────────┴──────────┘