Skip to content

Commit

Permalink
fix affinity
Browse files Browse the repository at this point in the history
  • Loading branch information
@sergelogvinov
sergelogvinov committed Nov 2, 2024
1 parent efda28d commit 0cc3a88
Show file tree
Hide file tree
Showing 7 changed files with 38 additions and 32 deletions.
24 changes: 21 additions & 3 deletions proxmox/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Proxmox

It was tested on Proxmox version 7.4-3
It was tested on Proxmox version 8.2

Local utilities

Expand Down Expand Up @@ -94,9 +94,27 @@ machine:
First we need to define our cluster:

```hcl
proxmox_host = "node1.example.com"
# Proxmox API host
proxmox_host = "node1.example.com"
vpc_main_cidr = "172.16.0.0/24"
# Local proxmox subnets
vpc_main_cidr = ["172.16.0.0/24", "fd60:172:16::/64"]
# Node configuration
nodes = {
"node1" = {
storage = "data",
ip4 = "1.1.0.1"
ip6 = "2001:1:2:1::/64",
gw6 = "2001:1:2:1::64",
},
"node2" = {
storage = "data",
ip4 = "1.1.0.2"
ip6 = "2001:1:2:2::/64",
gw6 = "2001:1:2:2::64",
},
}
# We will create one control-plane node on the Proxmox node `node1` (count = 1)
controlplane = {
Expand Down
12 changes: 0 additions & 12 deletions proxmox/common.tf
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,4 @@

locals {
cpu_numa = {
for k, v in var.nodes : k => [for i in lookup(v, "cpu", "") :
flatten([for r in split(",", i) : (strcontains(r, "-") ? range(split("-", r)[0], split("-", r)[1] + 1, 1) : [r])])
]
}

cpus = { for k, v in local.cpu_numa : k =>
flatten([for numa in v : flatten([for r in range(length(numa) / 2) : [numa[r], numa[r + length(numa) / 2]]])])
}
}

data "proxmox_virtual_environment_node" "node" {
for_each = { for inx, zone in local.zones : zone => inx if lookup(try(var.instances[zone], {}), "enabled", false) }
node_name = each.key
Expand Down
2 changes: 1 addition & 1 deletion proxmox/cpuaffinity/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ locals {
flatten([flatten([for r in range(length(v) / 2) : [v[r], v[r + length(v) / 2]]])])
]

shift = var.shift * length(local.cpus[0])
shift = var.shift * length(try(local.cpus[0], []))

vm_arch = { for k in flatten([
for inx in range(var.vms) : {
Expand Down
4 changes: 2 additions & 2 deletions proxmox/instances-db.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ module "db_affinity" {
} if lookup(try(var.instances[zone], {}), "db_count", 0) > 0 }

source = "./cpuaffinity"
cpu_affinity = var.nodes[each.value.zone].cpu
cpu_affinity = length(lookup(try(var.nodes[each.value.zone], {}), "cpu", [])) > 0 ? var.nodes[each.value.zone].cpu : ["0-${2 * data.proxmox_virtual_environment_node.node[each.value.zone].cpu_count * data.proxmox_virtual_environment_node.node[each.value.zone].cpu_sockets - 1}"]
vms = each.value.vms
cpus = lookup(try(var.instances[each.value.zone], {}), "db_cpu", 1)
# shift = length(var.nodes[each.value.zone].cpu) - 1
Expand Down Expand Up @@ -96,7 +96,7 @@ resource "proxmox_virtual_environment_vm" "db" {
cpu {
architecture = "x86_64"
cores = each.value.cpu
affinity = join(",", module.db_affinity[each.value.zone].arch[each.value.inx].cpus)
affinity = length(lookup(try(var.nodes[each.value.zone], {}), "cpu", [])) > 0 ? join(",", module.db_affinity[each.value.zone].arch[each.value.inx].cpus) : null
sockets = 1
numa = true
type = "host"
Expand Down
6 changes: 3 additions & 3 deletions proxmox/instances-web.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ module "web_affinity" {
} if lookup(try(var.instances[zone], {}), "web_count", 0) > 0 }

source = "./cpuaffinity"
cpu_affinity = var.nodes[each.value.zone].cpu
cpu_affinity = length(lookup(try(var.nodes[each.value.zone], {}), "cpu", [])) > 0 ? var.nodes[each.value.zone].cpu : ["0-${2 * data.proxmox_virtual_environment_node.node[each.value.zone].cpu_count * data.proxmox_virtual_environment_node.node[each.value.zone].cpu_sockets - 1}"]
vms = each.value.vms
cpus = lookup(try(var.instances[each.value.zone], {}), "web_cpu", 1)
}
Expand Down Expand Up @@ -117,7 +117,7 @@ resource "proxmox_virtual_environment_vm" "web" {
cpu {
architecture = "x86_64"
cores = each.value.cpu
affinity = join(",", module.web_affinity[each.value.zone].arch[each.value.inx].cpus)
affinity = length(lookup(try(var.nodes[each.value.zone], {}), "cpu", [])) > 0 ? join(",", module.web_affinity[each.value.zone].arch[each.value.inx].cpus) : null
sockets = 1
numa = true
type = "host"
Expand All @@ -132,7 +132,7 @@ resource "proxmox_virtual_environment_vm" "web" {
device = "numa${index(keys(module.web_affinity[each.value.zone].arch[each.value.inx].numa), idx)}"
cpus = "${index(keys(module.web_affinity[each.value.zone].arch[each.value.inx].numa), idx) * (each.value.cpu / length(module.web_affinity[each.value.zone].arch[each.value.inx].numa))}-${(index(keys(module.web_affinity[each.value.zone].arch[each.value.inx].numa), idx) + 1) * (each.value.cpu / length(module.web_affinity[each.value.zone].arch[each.value.inx].numa)) - 1}"
mem = each.value.mem / length(module.web_affinity[each.value.zone].arch[each.value.inx].numa)
} }
} if length(lookup(try(var.nodes[each.value.zone], {}), "cpu", [])) > 0 }
content {
device = numa.value.device
cpus = numa.value.cpus
Expand Down
4 changes: 2 additions & 2 deletions proxmox/instances-worker.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ module "worker_affinity" {
} if lookup(try(var.instances[zone], {}), "worker_count", 0) > 0 }

source = "./cpuaffinity"
cpu_affinity = var.nodes[each.value.zone].cpu
cpu_affinity = length(lookup(try(var.nodes[each.value.zone], {}), "cpu", [])) > 0 ? var.nodes[each.value.zone].cpu : ["0-${2 * data.proxmox_virtual_environment_node.node[each.value.zone].cpu_count * data.proxmox_virtual_environment_node.node[each.value.zone].cpu_sockets - 1}"]
vms = each.value.vms
cpus = lookup(try(var.instances[each.value.zone], {}), "worker_cpu", 1)
}
Expand Down Expand Up @@ -95,7 +95,7 @@ resource "proxmox_virtual_environment_vm" "worker" {
cpu {
architecture = "x86_64"
cores = each.value.cpu
affinity = join(",", module.worker_affinity[each.value.zone].arch[each.value.inx].cpus)
affinity = length(lookup(try(var.nodes[each.value.zone], {}), "cpu", [])) > 0 ? join(",", module.worker_affinity[each.value.zone].arch[each.value.inx].cpus) : null
sockets = 1
numa = true
type = "host"
Expand Down
18 changes: 9 additions & 9 deletions proxmox/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ variable "network_shift" {
}

variable "vpc_main_cidr" {
description = "Local proxmox subnet"
description = "Local proxmox subnets"
type = list(string)
default = ["172.16.0.0/24", "fd60:172:16::/64"]
}
Expand Down Expand Up @@ -53,17 +53,17 @@ variable "nodes" {
default = {
"hvm-1" = {
storage = "data",
cpu = ["0-3,16-19", "4-7,20-23", "8-11,24-27", "12-15,28-31"],
ip4 = "1.1.0.1"
ip6 = "2001:1:2:1::/64",
gw6 = "2001:1:2:1::64",
# cpu = ["0-3,16-19", "4-7,20-23", "8-11,24-27", "12-15,28-31"],
ip4 = "1.1.0.1"
ip6 = "2001:1:2:1::/64",
gw6 = "2001:1:2:1::64",
},
"hvm-2" = {
storage = "data",
cpu = ["0-3,16-19", "4-7,20-23", "8-11,24-27", "12-15,28-31"],
ip4 = "1.1.0.2"
ip6 = "2001:1:2:2::/64",
gw6 = "2001:1:2:2::64",
# cpu = ["0-3,16-19", "4-7,20-23", "8-11,24-27", "12-15,28-31"],
ip4 = "1.1.0.2"
ip6 = "2001:1:2:2::/64",
gw6 = "2001:1:2:2::64",
},
}
}
Expand Down

0 comments on commit 0cc3a88

Please sign in to comment.