From 97e0dd9c4a8e2982b55ae8abaf2412b44bb31f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=9F=D1=80=D0=BE=D0=BA?= =?UTF-8?q?=D0=BE=D0=BF=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Fri, 25 Sep 2026 11:43:00 +0300 Subject: [PATCH] deploy first 3 node to pve --- .gitignore | 5 +- ansible/inventories/install-1/hosts.yml | 38 ++++++++ terraform/envs/install-1/.terraform.lock.hcl | 18 ++++ terraform/envs/install-1/main.tf | 59 +++++++++++++ terraform/envs/install-1/outputs.tf | 3 + terraform/envs/install-1/terraform.tfvars | 30 +++++++ terraform/envs/install-1/variables.tf | 47 ++++++++++ terraform/envs/install-1/versions.tf | 21 +++++ terraform/modules/proxmox-vm/main.tf | 91 ++++++++++++++++++++ terraform/modules/proxmox-vm/outputs.tf | 12 +++ terraform/modules/proxmox-vm/variables.tf | 87 +++++++++++++++++++ terraform/modules/proxmox-vm/versions.tf | 10 +++ 12 files changed, 419 insertions(+), 2 deletions(-) create mode 100644 ansible/inventories/install-1/hosts.yml create mode 100644 terraform/envs/install-1/.terraform.lock.hcl create mode 100644 terraform/envs/install-1/main.tf create mode 100644 terraform/envs/install-1/outputs.tf create mode 100644 terraform/envs/install-1/terraform.tfvars create mode 100644 terraform/envs/install-1/variables.tf create mode 100644 terraform/envs/install-1/versions.tf create mode 100644 terraform/modules/proxmox-vm/main.tf create mode 100644 terraform/modules/proxmox-vm/outputs.tf create mode 100644 terraform/modules/proxmox-vm/variables.tf create mode 100644 terraform/modules/proxmox-vm/versions.tf diff --git a/.gitignore b/.gitignore index 6c20b58..f395033 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,8 @@ .terraform/ *.tfstate *.tfstate.* -*.tfvars -!*.tfvars.example +# terraform.tfvars описывает топологию и коммитится; секреты только через env или *.secret.tfvars +*.secret.tfvars crash.log .terraform.lock.hcl.bak @@ -16,6 +16,7 @@ vault_password* *.pem *.key .env +.envrc secrets/ # macOS / IDE diff --git a/ansible/inventories/install-1/hosts.yml b/ansible/inventories/install-1/hosts.yml new file mode 100644 index 0000000..e916740 --- /dev/null +++ b/ansible/inventories/install-1/hosts.yml @@ -0,0 +1,38 @@ +# Сгенерировано Terraform (terraform/envs/install-1), не редактировать вручную +"all": + "children": + "ceph": + "hosts": + "i1-node-01": + "ansible_host": "192.168.3.201" + "i1-node-02": + "ansible_host": "192.168.3.202" + "i1-node-03": + "ansible_host": "192.168.3.203" + "mds": + "hosts": + "i1-node-01": null + "i1-node-02": null + "i1-node-03": null + "mgr": + "hosts": + "i1-node-01": null + "i1-node-02": null + "i1-node-03": null + "mon": + "hosts": + "i1-node-01": null + "i1-node-02": null + "i1-node-03": null + "osd": + "hosts": + "i1-node-01": null + "i1-node-02": null + "i1-node-03": null + "rgw": + "hosts": + "i1-node-01": null + "i1-node-02": null + "i1-node-03": null + "vars": + "ansible_user": "aprokopenko" diff --git a/terraform/envs/install-1/.terraform.lock.hcl b/terraform/envs/install-1/.terraform.lock.hcl new file mode 100644 index 0000000..593875a --- /dev/null +++ b/terraform/envs/install-1/.terraform.lock.hcl @@ -0,0 +1,18 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/bpg/proxmox" { + version = "0.114.0" + constraints = "~> 0.114" + hashes = [ + "h1:TyMVdaMr3Qd/EHcoyEqRZ4D9wqaoqHN/segKFpNZnM8=", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.9.1" + constraints = "~> 2.5" + hashes = [ + "h1:OZGJN0LSSat5QIxZPxDXtVr4XpHb2oG7cVKJhSqBFIE=", + ] +} diff --git a/terraform/envs/install-1/main.tf b/terraform/envs/install-1/main.tf new file mode 100644 index 0000000..e4ffeea --- /dev/null +++ b/terraform/envs/install-1/main.tf @@ -0,0 +1,59 @@ +module "vm" { + source = "../../modules/proxmox-vm" + for_each = var.vms + + name = each.key + vm_id = var.vm_id_base + tonumber(split(".", each.value.ip)[3]) + node_name = var.proxmox.node_name + template_vm_id = var.proxmox.template_vm_id + pool_id = var.proxmox.pool_id + datastore_id = var.proxmox.datastore_id + bridge = var.proxmox.bridge + + cores = each.value.cores + memory = each.value.memory + os_disk_size = each.value.os_disk_size + osd_disks = each.value.osd_disks + osd_disk_size = each.value.osd_disk_size + + ipv4_address = "${each.value.ip}/${var.network.prefix}" + ipv4_gateway = var.network.gateway + dns_servers = var.network.dns_servers + + ssh_user = var.ssh.user + ssh_public_keys = [trimspace(file(pathexpand(var.ssh.public_key_file)))] + + tags = concat(["ceph-course", var.install_name], each.value.roles) +} + +locals { + roles = sort(distinct(flatten([for vm in var.vms : vm.roles]))) + + # Роли из tfvars превращаются в группы Ansible, а дальше в labels cephadm + inventory = { + all = { + vars = { + ansible_user = var.ssh.user + } + children = merge( + { + ceph = { + hosts = { for name, vm in module.vm : name => { ansible_host = vm.ipv4 } } + } + }, + { + for role in local.roles : role => { + hosts = { for name, vm in var.vms : name => null if contains(vm.roles, role) } + } + }, + ) + } + } +} + +resource "local_file" "inventory" { + filename = "${path.root}/../../../ansible/inventories/${var.install_name}/hosts.yml" + content = "# Сгенерировано Terraform (terraform/envs/${var.install_name}), не редактировать вручную\n${yamlencode(local.inventory)}" + file_permission = "0644" + directory_permission = "0755" +} diff --git a/terraform/envs/install-1/outputs.tf b/terraform/envs/install-1/outputs.tf new file mode 100644 index 0000000..5edc017 --- /dev/null +++ b/terraform/envs/install-1/outputs.tf @@ -0,0 +1,3 @@ +output "vms" { + value = { for name, vm in module.vm : name => { vm_id = vm.vm_id, ip = vm.ipv4 } } +} diff --git a/terraform/envs/install-1/terraform.tfvars b/terraform/envs/install-1/terraform.tfvars new file mode 100644 index 0000000..3003095 --- /dev/null +++ b/terraform/envs/install-1/terraform.tfvars @@ -0,0 +1,30 @@ +# Установка 1: три узла, на каждом все роли, одна сеть +install_name = "install-1" + +proxmox = { + node_name = "pve1" + template_vm_id = 9000 + pool_id = "ceph-course" + datastore_id = "zfs1" + bridge = "vmbr0" +} + +network = { + prefix = 24 + gateway = "192.168.3.1" + dns_servers = ["192.168.3.1"] +} + +# 192.168.3.201 -> VMID 3201 +vm_id_base = 3000 + +ssh = { + user = "aprokopenko" + public_key_file = "~/.ssh/aprokopenko_ed25519.pub" +} + +vms = { + "i1-node-01" = { ip = "192.168.3.201", cores = 4, memory = 10240, osd_disks = 3, roles = ["mon", "mgr", "osd", "rgw", "mds"] } + "i1-node-02" = { ip = "192.168.3.202", cores = 4, memory = 10240, osd_disks = 3, roles = ["mon", "mgr", "osd", "rgw", "mds"] } + "i1-node-03" = { ip = "192.168.3.203", cores = 4, memory = 10240, osd_disks = 3, roles = ["mon", "mgr", "osd", "rgw", "mds"] } +} diff --git a/terraform/envs/install-1/variables.tf b/terraform/envs/install-1/variables.tf new file mode 100644 index 0000000..62606dd --- /dev/null +++ b/terraform/envs/install-1/variables.tf @@ -0,0 +1,47 @@ +variable "install_name" { + description = "Имя установки, используется в тегах и пути к inventory" + type = string +} + +variable "proxmox" { + type = object({ + node_name = string + template_vm_id = number + pool_id = string + datastore_id = string + bridge = string + }) +} + +variable "network" { + type = object({ + prefix = number + gateway = string + dns_servers = list(string) + }) +} + +variable "vm_id_base" { + description = "VMID = vm_id_base + последний октет IP" + type = number +} + +variable "ssh" { + type = object({ + user = string + public_key_file = string + }) +} + +variable "vms" { + description = "Топология установки: ВМ, их ресурсы и роли Ceph" + type = map(object({ + ip = string + cores = number + memory = number + os_disk_size = optional(number, 40) + osd_disks = optional(number, 0) + osd_disk_size = optional(number, 40) + roles = list(string) + })) +} diff --git a/terraform/envs/install-1/versions.tf b/terraform/envs/install-1/versions.tf new file mode 100644 index 0000000..e6cadb7 --- /dev/null +++ b/terraform/envs/install-1/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 1.9" + + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.114" + } + local = { + source = "hashicorp/local" + version = "~> 2.5" + } + } +} + +# Адрес и токен берутся из окружения: +# PROXMOX_VE_ENDPOINT=https://:8006/ +# PROXMOX_VE_API_TOKEN='terraform@pve!tf=<секрет>' +provider "proxmox" { + insecure = true +} diff --git a/terraform/modules/proxmox-vm/main.tf b/terraform/modules/proxmox-vm/main.tf new file mode 100644 index 0000000..e26e2a1 --- /dev/null +++ b/terraform/modules/proxmox-vm/main.tf @@ -0,0 +1,91 @@ +resource "proxmox_virtual_environment_vm" "this" { + name = var.name + vm_id = var.vm_id + node_name = var.node_name + pool_id = var.pool_id + description = "Managed by Terraform (ceph course)" + # Proxmox хранит теги отсортированными, иначе будет вечный diff + tags = sort(distinct(var.tags)) + + clone { + vm_id = var.template_vm_id + datastore_id = var.datastore_id + full = true + } + + # Стенд включаем вручную, не при загрузке хоста + on_boot = false + stop_on_destroy = true + + agent { + enabled = true + trim = true + wait_for_ip { + ipv4 = true + } + } + + cpu { + cores = var.cores + type = "host" + } + + # Без ballooning: Ceph плохо переносит, когда у него забирают память + memory { + dedicated = var.memory + } + + operating_system { + type = "l26" + } + + scsi_hardware = "virtio-scsi-single" + + # Системный диск из шаблона, увеличиваем до нужного размера + disk { + interface = "scsi0" + datastore_id = var.datastore_id + size = var.os_disk_size + discard = "on" + iothread = true + ssd = true + } + + # Пустые диски под OSD. Serial osdN виден в ВМ в `lsblk -o NAME,SERIAL` + dynamic "disk" { + for_each = range(var.osd_disks) + content { + interface = "scsi${disk.value + 1}" + datastore_id = var.datastore_id + size = var.osd_disk_size + discard = "on" + iothread = true + ssd = true + serial = "osd${disk.value}" + } + } + + network_device { + bridge = var.bridge + } + + initialization { + datastore_id = var.datastore_id + + dns { + servers = var.dns_servers + } + + ip_config { + ipv4 { + address = var.ipv4_address + gateway = var.ipv4_gateway + } + } + + user_account { + username = var.ssh_user + keys = var.ssh_public_keys + } + } +} diff --git a/terraform/modules/proxmox-vm/outputs.tf b/terraform/modules/proxmox-vm/outputs.tf new file mode 100644 index 0000000..00a2776 --- /dev/null +++ b/terraform/modules/proxmox-vm/outputs.tf @@ -0,0 +1,12 @@ +output "name" { + value = proxmox_virtual_environment_vm.this.name +} + +output "vm_id" { + value = proxmox_virtual_environment_vm.this.vm_id +} + +output "ipv4" { + description = "Адрес без префикса, для inventory" + value = split("/", var.ipv4_address)[0] +} diff --git a/terraform/modules/proxmox-vm/variables.tf b/terraform/modules/proxmox-vm/variables.tf new file mode 100644 index 0000000..6df9f8e --- /dev/null +++ b/terraform/modules/proxmox-vm/variables.tf @@ -0,0 +1,87 @@ +variable "name" { + description = "Имя ВМ (оно же hostname)" + type = string +} + +variable "vm_id" { + description = "VMID в Proxmox" + type = number +} + +variable "node_name" { + description = "Узел Proxmox, на котором создаётся ВМ" + type = string +} + +variable "template_vm_id" { + description = "VMID cloud-init шаблона, из которого клонируется ВМ" + type = number +} + +variable "pool_id" { + description = "Пул Proxmox, в который попадает ВМ" + type = string +} + +variable "datastore_id" { + description = "Хранилище для дисков ВМ и cloud-init диска" + type = string +} + +variable "cores" { + type = number +} + +variable "memory" { + description = "RAM в MiB" + type = number +} + +variable "os_disk_size" { + description = "Размер системного диска, GB" + type = number + default = 40 +} + +variable "osd_disks" { + description = "Количество пустых дисков под OSD" + type = number + default = 0 +} + +variable "osd_disk_size" { + description = "Размер каждого диска под OSD, GB" + type = number + default = 40 +} + +variable "bridge" { + type = string + default = "vmbr0" +} + +variable "ipv4_address" { + description = "Статический адрес в CIDR, например 192.168.3.201/24" + type = string +} + +variable "ipv4_gateway" { + type = string +} + +variable "dns_servers" { + type = list(string) +} + +variable "ssh_user" { + type = string +} + +variable "ssh_public_keys" { + type = list(string) +} + +variable "tags" { + type = list(string) + default = [] +} diff --git a/terraform/modules/proxmox-vm/versions.tf b/terraform/modules/proxmox-vm/versions.tf new file mode 100644 index 0000000..907d9ac --- /dev/null +++ b/terraform/modules/proxmox-vm/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.9" + + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.114" + } + } +}