deploy first 3 node to pve

.gitignore

@@ -2,8 +2,8 @@
 .terraform/
 *.tfstate
 *.tfstate.*
-*.tfvars
+# terraform.tfvars описывает топологию и коммитится; секреты только через env или *.secret.tfvars
-!*.tfvars.example
+*.secret.tfvars
 crash.log
 .terraform.lock.hcl.bak
 
@@ -16,6 +16,7 @@ vault_password*
 *.pem
 *.key
 .env
+.envrc
 secrets/
 
 # macOS / IDE

ansible/inventories/install-1/hosts.yml

@@ -0,0 +1,38 @@
+# Сгенерировано Terraform (terraform/envs/install-1), не редактировать вручную
+"all":
+  "children":
+    "ceph":
+      "hosts":
+        "i1-node-01":
+          "ansible_host": "192.168.3.201"
+        "i1-node-02":
+          "ansible_host": "192.168.3.202"
+        "i1-node-03":
+          "ansible_host": "192.168.3.203"
+    "mds":
+      "hosts":
+        "i1-node-01": null
+        "i1-node-02": null
+        "i1-node-03": null
+    "mgr":
+      "hosts":
+        "i1-node-01": null
+        "i1-node-02": null
+        "i1-node-03": null
+    "mon":
+      "hosts":
+        "i1-node-01": null
+        "i1-node-02": null
+        "i1-node-03": null
+    "osd":
+      "hosts":
+        "i1-node-01": null
+        "i1-node-02": null
+        "i1-node-03": null
+    "rgw":
+      "hosts":
+        "i1-node-01": null
+        "i1-node-02": null
+        "i1-node-03": null
+  "vars":
+    "ansible_user": "aprokopenko"

terraform/envs/install-1/.terraform.lock.hcl

@@ -0,0 +1,18 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/bpg/proxmox" {
+  version     = "0.114.0"
+  constraints = "~> 0.114"
+  hashes = [
+    "h1:TyMVdaMr3Qd/EHcoyEqRZ4D9wqaoqHN/segKFpNZnM8=",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/local" {
+  version     = "2.9.1"
+  constraints = "~> 2.5"
+  hashes = [
+    "h1:OZGJN0LSSat5QIxZPxDXtVr4XpHb2oG7cVKJhSqBFIE=",
+  ]
+}

terraform/envs/install-1/main.tf

@@ -0,0 +1,59 @@
+module "vm" {
+  source   = "../../modules/proxmox-vm"
+  for_each = var.vms
+
+  name           = each.key
+  vm_id          = var.vm_id_base + tonumber(split(".", each.value.ip)[3])
+  node_name      = var.proxmox.node_name
+  template_vm_id = var.proxmox.template_vm_id
+  pool_id        = var.proxmox.pool_id
+  datastore_id   = var.proxmox.datastore_id
+  bridge         = var.proxmox.bridge
+
+  cores         = each.value.cores
+  memory        = each.value.memory
+  os_disk_size  = each.value.os_disk_size
+  osd_disks     = each.value.osd_disks
+  osd_disk_size = each.value.osd_disk_size
+
+  ipv4_address = "${each.value.ip}/${var.network.prefix}"
+  ipv4_gateway = var.network.gateway
+  dns_servers  = var.network.dns_servers
+
+  ssh_user        = var.ssh.user
+  ssh_public_keys = [trimspace(file(pathexpand(var.ssh.public_key_file)))]
+
+  tags = concat(["ceph-course", var.install_name], each.value.roles)
+}
+
+locals {
+  roles = sort(distinct(flatten([for vm in var.vms : vm.roles])))
+
+  # Роли из tfvars превращаются в группы Ansible, а дальше в labels cephadm
+  inventory = {
+    all = {
+      vars = {
+        ansible_user = var.ssh.user
+      }
+      children = merge(
+        {
+          ceph = {
+            hosts = { for name, vm in module.vm : name => { ansible_host = vm.ipv4 } }
+          }
+        },
+        {
+          for role in local.roles : role => {
+            hosts = { for name, vm in var.vms : name => null if contains(vm.roles, role) }
+          }
+        },
+      )
+    }
+  }
+}
+
+resource "local_file" "inventory" {
+  filename             = "${path.root}/../../../ansible/inventories/${var.install_name}/hosts.yml"
+  content              = "# Сгенерировано Terraform (terraform/envs/${var.install_name}), не редактировать вручную\n${yamlencode(local.inventory)}"
+  file_permission      = "0644"
+  directory_permission = "0755"
+}

terraform/envs/install-1/outputs.tf

@@ -0,0 +1,3 @@
+output "vms" {
+  value = { for name, vm in module.vm : name => { vm_id = vm.vm_id, ip = vm.ipv4 } }
+}

terraform/envs/install-1/terraform.tfvars

@@ -0,0 +1,30 @@
+# Установка 1: три узла, на каждом все роли, одна сеть
+install_name = "install-1"
+
+proxmox = {
+  node_name      = "pve1"
+  template_vm_id = 9000
+  pool_id        = "ceph-course"
+  datastore_id   = "zfs1"
+  bridge         = "vmbr0"
+}
+
+network = {
+  prefix      = 24
+  gateway     = "192.168.3.1"
+  dns_servers = ["192.168.3.1"]
+}
+
+# 192.168.3.201 -> VMID 3201
+vm_id_base = 3000
+
+ssh = {
+  user            = "aprokopenko"
+  public_key_file = "~/.ssh/aprokopenko_ed25519.pub"
+}
+
+vms = {
+  "i1-node-01" = { ip = "192.168.3.201", cores = 4, memory = 10240, osd_disks = 3, roles = ["mon", "mgr", "osd", "rgw", "mds"] }
+  "i1-node-02" = { ip = "192.168.3.202", cores = 4, memory = 10240, osd_disks = 3, roles = ["mon", "mgr", "osd", "rgw", "mds"] }
+  "i1-node-03" = { ip = "192.168.3.203", cores = 4, memory = 10240, osd_disks = 3, roles = ["mon", "mgr", "osd", "rgw", "mds"] }
+}

terraform/envs/install-1/variables.tf

@@ -0,0 +1,47 @@
+variable "install_name" {
+  description = "Имя установки, используется в тегах и пути к inventory"
+  type        = string
+}
+
+variable "proxmox" {
+  type = object({
+    node_name      = string
+    template_vm_id = number
+    pool_id        = string
+    datastore_id   = string
+    bridge         = string
+  })
+}
+
+variable "network" {
+  type = object({
+    prefix      = number
+    gateway     = string
+    dns_servers = list(string)
+  })
+}
+
+variable "vm_id_base" {
+  description = "VMID = vm_id_base + последний октет IP"
+  type        = number
+}
+
+variable "ssh" {
+  type = object({
+    user            = string
+    public_key_file = string
+  })
+}
+
+variable "vms" {
+  description = "Топология установки: ВМ, их ресурсы и роли Ceph"
+  type = map(object({
+    ip            = string
+    cores         = number
+    memory        = number
+    os_disk_size  = optional(number, 40)
+    osd_disks     = optional(number, 0)
+    osd_disk_size = optional(number, 40)
+    roles         = list(string)
+  }))
+}

terraform/envs/install-1/versions.tf

@@ -0,0 +1,21 @@
+terraform {
+  required_version = ">= 1.9"
+
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "~> 0.114"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = "~> 2.5"
+    }
+  }
+}
+
+# Адрес и токен берутся из окружения:
+#   PROXMOX_VE_ENDPOINT=https://<pve1>:8006/
+#   PROXMOX_VE_API_TOKEN='terraform@pve!tf=<секрет>'
+provider "proxmox" {
+  insecure = true
+}

terraform/modules/proxmox-vm/main.tf

@@ -0,0 +1,91 @@
+resource "proxmox_virtual_environment_vm" "this" {
+  name        = var.name
+  vm_id       = var.vm_id
+  node_name   = var.node_name
+  pool_id     = var.pool_id
+  description = "Managed by Terraform (ceph course)"
+  # Proxmox хранит теги отсортированными, иначе будет вечный diff
+  tags = sort(distinct(var.tags))
+
+  clone {
+    vm_id        = var.template_vm_id
+    datastore_id = var.datastore_id
+    full         = true
+  }
+
+  # Стенд включаем вручную, не при загрузке хоста
+  on_boot         = false
+  stop_on_destroy = true
+
+  agent {
+    enabled = true
+    trim    = true
+    wait_for_ip {
+      ipv4 = true
+    }
+  }
+
+  cpu {
+    cores = var.cores
+    type  = "host"
+  }
+
+  # Без ballooning: Ceph плохо переносит, когда у него забирают память
+  memory {
+    dedicated = var.memory
+  }
+
+  operating_system {
+    type = "l26"
+  }
+
+  scsi_hardware = "virtio-scsi-single"
+
+  # Системный диск из шаблона, увеличиваем до нужного размера
+  disk {
+    interface    = "scsi0"
+    datastore_id = var.datastore_id
+    size         = var.os_disk_size
+    discard      = "on"
+    iothread     = true
+    ssd          = true
+  }
+
+  # Пустые диски под OSD. Serial osdN виден в ВМ в `lsblk -o NAME,SERIAL`
+  dynamic "disk" {
+    for_each = range(var.osd_disks)
+    content {
+      interface    = "scsi${disk.value + 1}"
+      datastore_id = var.datastore_id
+      size         = var.osd_disk_size
+      discard      = "on"
+      iothread     = true
+      ssd          = true
+      serial       = "osd${disk.value}"
+    }
+  }
+
+  network_device {
+    bridge = var.bridge
+  }
+
+  initialization {
+    datastore_id = var.datastore_id
+
+    dns {
+      servers = var.dns_servers
+    }
+
+    ip_config {
+      ipv4 {
+        address = var.ipv4_address
+        gateway = var.ipv4_gateway
+      }
+    }
+
+    user_account {
+      username = var.ssh_user
+      keys     = var.ssh_public_keys
+    }
+  }
+}

terraform/modules/proxmox-vm/outputs.tf

@@ -0,0 +1,12 @@
+output "name" {
+  value = proxmox_virtual_environment_vm.this.name
+}
+
+output "vm_id" {
+  value = proxmox_virtual_environment_vm.this.vm_id
+}
+
+output "ipv4" {
+  description = "Адрес без префикса, для inventory"
+  value       = split("/", var.ipv4_address)[0]
+}

terraform/modules/proxmox-vm/variables.tf

@@ -0,0 +1,87 @@
+variable "name" {
+  description = "Имя ВМ (оно же hostname)"
+  type        = string
+}
+
+variable "vm_id" {
+  description = "VMID в Proxmox"
+  type        = number
+}
+
+variable "node_name" {
+  description = "Узел Proxmox, на котором создаётся ВМ"
+  type        = string
+}
+
+variable "template_vm_id" {
+  description = "VMID cloud-init шаблона, из которого клонируется ВМ"
+  type        = number
+}
+
+variable "pool_id" {
+  description = "Пул Proxmox, в который попадает ВМ"
+  type        = string
+}
+
+variable "datastore_id" {
+  description = "Хранилище для дисков ВМ и cloud-init диска"
+  type        = string
+}
+
+variable "cores" {
+  type = number
+}
+
+variable "memory" {
+  description = "RAM в MiB"
+  type        = number
+}
+
+variable "os_disk_size" {
+  description = "Размер системного диска, GB"
+  type        = number
+  default     = 40
+}
+
+variable "osd_disks" {
+  description = "Количество пустых дисков под OSD"
+  type        = number
+  default     = 0
+}
+
+variable "osd_disk_size" {
+  description = "Размер каждого диска под OSD, GB"
+  type        = number
+  default     = 40
+}
+
+variable "bridge" {
+  type    = string
+  default = "vmbr0"
+}
+
+variable "ipv4_address" {
+  description = "Статический адрес в CIDR, например 192.168.3.201/24"
+  type        = string
+}
+
+variable "ipv4_gateway" {
+  type = string
+}
+
+variable "dns_servers" {
+  type = list(string)
+}
+
+variable "ssh_user" {
+  type = string
+}
+
+variable "ssh_public_keys" {
+  type = list(string)
+}
+
+variable "tags" {
+  type    = list(string)
+  default = []
+}

terraform/modules/proxmox-vm/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.9"
+
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = "~> 0.114"
+    }
+  }
+}