External disk mount path customisable

[garry-t]

Description

It is pain to make work longhorn by default on nvme and ssd. Because module has hardcoded value for mount path of disk and longhorn values file also pin path to /var/longhorn

So just customising mount path will give ability to create secondstorage class in longhorn with path and use both storages nvme and ssd

resource "hcloud_volume" "longhorn_volume" {
  for_each = { for k, v in local.agent_nodes : k => v if((v.longhorn_volume_size >= 10) && (v.longhorn_volume_size <= 10240) && var.enable_longhorn) }

  labels = {
    provisioner = "terraform"
    cluster     = var.cluster_name
    scope       = "longhorn"
  }
  name              = "${var.cluster_name}-longhorn-${module.agents[each.key].name}"
  size              = local.agent_nodes[each.key].longhorn_volume_size
  server_id         = module.agents[each.key].id
  automount         = true
  format            = var.longhorn_fstype
  delete_protection = var.enable_delete_protection.volume
}

resource "null_resource" "configure_longhorn_volume" {
  for_each = { for k, v in local.agent_nodes : k => v if((v.longhorn_volume_size >= 10) && (v.longhorn_volume_size <= 10240) && var.enable_longhorn) }

  triggers = {
    agent_id = module.agents[each.key].id
  }

  # Start the k3s agent and wait for it to have started
  provisioner "remote-exec" {
    inline = [
      "mkdir /var/longhorn >/dev/null 2>&1",
      "mount -o discard,defaults ${hcloud_volume.longhorn_volume[each.key].linux_device} ${var.mount_path}",
      "${var.longhorn_fstype == "ext4" ? "resize2fs" : "xfs_growfs"} ${hcloud_volume.longhorn_volume[each.key].linux_device}",
      "echo '${hcloud_volume.longhorn_volume[each.key].linux_device} ${var.mount_path} ${var.longhorn_fstype} discard,nofail,defaults 0 0' >> /etc/fstab"
    ]
  }

  connection {
    user           = "root"
    private_key    = var.ssh_private_key
    agent_identity = local.ssh_agent_identity
    host           = module.agents[each.key].ipv4_address
    port           = var.ssh_port
  }

  depends_on = [
    hcloud_volume.longhorn_volume
  ]
}

[garry-t]

found how to achieve it

[garry-t]

no, doesnt work. reopen

[garry-t]

PR is here #1800
Would be nice to review.

Basically how you can use it now
Set longhorn_values it will be default storage class created by longhorn. Path in my case /var/longhorn

  longhorn_values = <<EOT
defaultSettings:
  nodeDrainPolicy: allow-if-replica-is-stopped
  defaultDataPath: /var/longhorn
persistence:
  defaultFsType: ext4
  defaultClassReplicaCount: 3
  defaultClass: true
  EOT

After that later after cluster creation you can fetch nodes by some labels where you know that external ssd disk mounted. In my case nodes with label ssd=true have hetzner volume

data "kubernetes_nodes" "ssd_nodes" {
 metadata {
   labels = {
     "ssd" = "true"
   }
 }
}

# Patch nodes only with this label
resource "null_resource" "longhorn_patch_external_disk" {
 for_each = {
   for node in data.kubernetes_nodes.ssd_nodes.nodes : node.metadata[0].name => node.metadata[0].name
 }
 triggers = {
   always_run = timestamp()  # triggers re-run on every apply
 }
 provisioner "local-exec" {
   command = <<-EOT
     KUBECONFIG=${var.kubeconfig_path} kubectl -n longhorn-system patch nodes.longhorn.io ${each.key} --type merge -p '{
       "spec": {
         "disks": {
           "external-ssd": {
             "path": "/var/lib/longhorn",
             "allowScheduling": true,
             "tags": ["ssd"]
           }
         }
       }
     }'
   EOT
 }
}

resource "kubernetes_manifest" "longhorn_ssd_replica" {
 manifest = {
   apiVersion = "storage.k8s.io/v1"
   kind       = "StorageClass"
   metadata = {
     name = "longhorn-ssd-replica"
   }
   provisioner = "driver.longhorn.io"
   parameters = {
     numberOfReplicas    = "1"
     staleReplicaTimeout = "30"
     diskSelector        = "ssd"
#      nodeSelector        = "ssd=true"
     fromBackup          = ""
   }
   reclaimPolicy        = "Delete"
   allowVolumeExpansion = true
   volumeBindingMode    = "Immediate"
 }
 depends_on = [null_resource.longhorn_patch_external_disk]
}

[garry-t]

PR for this already completed #1800