ci: refine terraform scripts for debugging and add missing dmsetup/de…

…vice-mapper in some distros

Signed-off-by: Yang Chiu <[email protected]>

test_framework/terraform/aws/oracle/main.tf

@@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
+  ingress {
+    description = "Allow longhorn-ui nodeport"
+    from_port   = 30000
+    to_port     = 30000
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
   ingress {
     description = "Allow UDP connection for longhorn-webhooks"
     from_port   = 0

test_framework/terraform/aws/oracle/output.tf

@@ -47,6 +47,38 @@ output "load_balancer_url" {
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
 
+output "instance_mapping" {
+  depends_on = [
+    aws_instance.lh_aws_instance_controlplane_k3s,
+    aws_instance.lh_aws_instance_worker_k3s,
+  ]
+
+  value = jsonencode(
+    concat(
+     [
+      for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : {
+           "name": controlplane_instance.private_dns,
+           "id": controlplane_instance.id
+          }
+
+     ],
+     [
+      for worker_instance in aws_instance.lh_aws_instance_worker_k3s : {
+           "name": worker_instance.private_dns,
+           "id": worker_instance.id
+         }
+     ]
+    )
+  )
+}
+
+output "controlplane_public_ip" {
+  depends_on = [
+    aws_eip.lh_aws_eip_controlplane
+  ]
+  value = aws_eip.lh_aws_eip_controlplane[0].public_ip
+}
+
 output "resource_suffix" {
   depends_on = [
     random_string.random_suffix

test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -2,7 +2,7 @@
 
 sudo yum update -y
 sudo yum group install -y "Development Tools"
-sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup
+sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 # disable nm-cloud-setup otherwise k3s-agent service won’t start.
@@ -12,12 +12,14 @@ modprobe uio
 modprobe uio_pci_generic
 modprobe vfio_pci
 modprobe nvme-tcp
+modprobe dm_crypt
 touch /etc/modules-load.d/modules.conf
 cat > /etc/modules-load.d/modules.conf <<EOF
 uio
 uio_pci_generic
 vfio_pci
 nvme-tcp
+dm_crypt
 EOF
 
 echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages

test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl

@@ -2,7 +2,7 @@
 
 sudo yum update -y
 sudo yum group install -y "Development Tools"
-sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup nc
+sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper nc
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
@@ -11,12 +11,14 @@ modprobe uio
 modprobe uio_pci_generic
 modprobe vfio_pci
 modprobe nvme-tcp
+modprobe dm_crypt
 touch /etc/modules-load.d/modules.conf
 cat > /etc/modules-load.d/modules.conf <<EOF
 uio
 uio_pci_generic
 vfio_pci
 nvme-tcp
+dm_crypt
 EOF
 
 echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages

test_framework/terraform/aws/rhel/main.tf

@@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
+  ingress {
+    description = "Allow longhorn-ui nodeport"
+    from_port   = 30000
+    to_port     = 30000
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
   ingress {
     description = "Allow UDP connection for longhorn-webhooks"
     from_port   = 0

test_framework/terraform/aws/rhel/output.tf

@@ -47,6 +47,38 @@ output "load_balancer_url" {
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
 
+output "instance_mapping" {
+  depends_on = [
+    aws_instance.lh_aws_instance_controlplane_k3s,
+    aws_instance.lh_aws_instance_worker_k3s,
+  ]
+
+  value = jsonencode(
+    concat(
+     [
+      for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : {
+           "name": controlplane_instance.private_dns,
+           "id": controlplane_instance.id
+          }
+
+     ],
+     [
+      for worker_instance in aws_instance.lh_aws_instance_worker_k3s : {
+           "name": worker_instance.private_dns,
+           "id": worker_instance.id
+         }
+     ]
+    )
+  )
+}
+
+output "controlplane_public_ip" {
+  depends_on = [
+    aws_eip.lh_aws_eip_controlplane
+  ]
+  value = aws_eip.lh_aws_eip_controlplane[0].public_ip
+}
+
 output "resource_suffix" {
   depends_on = [
     random_string.random_suffix

test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -10,7 +10,7 @@ fi
 
 sudo yum update -y
 sudo yum group install -y "Development Tools"
-sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup
+sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
@@ -19,12 +19,14 @@ modprobe uio
 modprobe uio_pci_generic
 modprobe vfio_pci
 modprobe nvme-tcp
+modprobe dm_crypt
 touch /etc/modules-load.d/modules.conf
 cat > /etc/modules-load.d/modules.conf <<EOF
 uio
 uio_pci_generic
 vfio_pci
 nvme-tcp
+dm_crypt
 EOF
 
 echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages

test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl

@@ -10,7 +10,7 @@ fi
 
 sudo yum update -y
 sudo yum group install -y "Development Tools"
-sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup nc
+sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper nc
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
@@ -19,12 +19,14 @@ modprobe uio
 modprobe uio_pci_generic
 modprobe vfio_pci
 modprobe nvme-tcp
+modprobe dm_crypt
 touch /etc/modules-load.d/modules.conf
 cat > /etc/modules-load.d/modules.conf <<EOF
 uio
 uio_pci_generic
 vfio_pci
 nvme-tcp
+dm_crypt
 EOF
 
 echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages

test_framework/terraform/aws/rockylinux/main.tf

@@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
+  ingress {
+    description = "Allow longhorn-ui nodeport"
+    from_port   = 30000
+    to_port     = 30000
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
   ingress {
     description = "Allow UDP connection for longhorn-webhooks"
     from_port   = 0

test_framework/terraform/aws/rockylinux/output.tf

@@ -47,6 +47,38 @@ output "load_balancer_url" {
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
 
+output "instance_mapping" {
+  depends_on = [
+    aws_instance.lh_aws_instance_controlplane_k3s,
+    aws_instance.lh_aws_instance_worker_k3s,
+  ]
+
+  value = jsonencode(
+    concat(
+     [
+      for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : {
+           "name": controlplane_instance.private_dns,
+           "id": controlplane_instance.id
+          }
+
+     ],
+     [
+      for worker_instance in aws_instance.lh_aws_instance_worker_k3s : {
+           "name": worker_instance.private_dns,
+           "id": worker_instance.id
+         }
+     ]
+    )
+  )
+}
+
+output "controlplane_public_ip" {
+  depends_on = [
+    aws_eip.lh_aws_eip_controlplane
+  ]
+  value = aws_eip.lh_aws_eip_controlplane[0].public_ip
+}
+
 output "resource_suffix" {
   depends_on = [
     random_string.random_suffix

test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -10,20 +10,22 @@ fi
 
 # Do not arbitrarily run "dnf update", as this will effectively move us up to the latest minor release.
 sudo dnf group install -y "Development Tools"
-sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup
+sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
 modprobe uio
 modprobe uio_pci_generic
 modprobe vfio_pci
 modprobe nvme-tcp
+modprobe dm_crypt
 touch /etc/modules-load.d/modules.conf
 cat > /etc/modules-load.d/modules.conf <<EOF
 uio
 uio_pci_generic
 vfio_pci
 nvme-tcp
+dm_crypt
 EOF
 
 echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages

test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl

@@ -10,20 +10,22 @@ fi
 
 # Do not arbitrarily run "dnf update", as this will effectively move us up to the latest minor release.
 sudo dnf group install -y "Development Tools"
-sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup jq nmap-ncat
+sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper jq nmap-ncat
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
 modprobe uio
 modprobe uio_pci_generic
 modprobe vfio_pci
 modprobe nvme-tcp
+modprobe dm_crypt
 touch /etc/modules-load.d/modules.conf
 cat > /etc/modules-load.d/modules.conf <<EOF
 uio
 uio_pci_generic
 vfio_pci
 nvme-tcp
+dm_crypt
 EOF
 
 echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages

test_framework/terraform/aws/sle-micro/main.tf

@@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_public" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
+  ingress {
+    description = "Allow longhorn-ui nodeport"
+    from_port   = 30000
+    to_port     = 30000
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
   ingress {
     description = "Allow UDP connection for longhorn-webhooks"
     from_port   = 0

test_framework/terraform/aws/sle-micro/output.tf

@@ -46,6 +46,38 @@ output "load_balancer_url" {
   value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null
 }
 
+output "instance_mapping" {
+  depends_on = [
+    aws_instance.lh_aws_instance_controlplane_k3s,
+    aws_instance.lh_aws_instance_worker_k3s,
+  ]
+
+  value = jsonencode(
+    concat(
+     [
+      for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : {
+           "name": controlplane_instance.private_dns,
+           "id": controlplane_instance.id
+          }
+
+     ],
+     [
+      for worker_instance in aws_instance.lh_aws_instance_worker_k3s : {
+           "name": worker_instance.private_dns,
+           "id": worker_instance.id
+         }
+     ]
+    )
+  )
+}
+
+output "controlplane_public_ip" {
+  depends_on = [
+    aws_eip.lh_aws_eip_controlplane
+  ]
+  value = aws_eip.lh_aws_eip_controlplane[0].public_ip
+}
+
 output "resource_suffix" {
   depends_on = [
     random_string.random_suffix

test_framework/terraform/aws/sles/k3s_instances.tf

@@ -127,7 +127,7 @@ resource "null_resource" "rsync_kubeconfig_file" {
     inline = [
       "cloud-init status --wait",
       "if [ \"`cloud-init status | grep error`\" ]; then sudo cat /var/log/cloud-init-output.log; fi",
-      "RETRY=0; MAX_RETRY=450; until([ -f /etc/rancher/k3s/k3s.yaml ] && [ `sudo /usr/local/bin/kubectl get node -o jsonpath='{.items[*].status.conditions}'  | jq '.[] | select(.type  == \"Ready\").status' | grep -ci true` -eq $((${var.lh_aws_instance_count_controlplane} + ${var.lh_aws_instance_count_worker})) ]); do echo \"waiting for k3s cluster nodes to be running\"; sleep 2; if [ $RETRY -eq $MAX_RETRY ]; then break; fi; RETRY=$((RETRY+1)); done"
+      "RETRY=0; MAX_RETRY=450; until([ -f /etc/rancher/k3s/k3s.yaml ] && [ `sudo /usr/local/bin/kubectl get node -o jsonpath='{.items[*].status.conditions}'  | jq '.[] | select(.type  == \"Ready\").status' | grep -ci true` -eq $((${var.lh_aws_instance_count_controlplane} + ${var.lh_aws_instance_count_worker})) ]); do echo \"waiting for k3s cluster nodes to be running\"; sleep 2; if [ $RETRY -eq $MAX_RETRY ]; then echo \"cluster nodes initialization timeout ...\"; sleep 86400; fi; RETRY=$((RETRY+1)); done"
     ]
 
     connection {

test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -5,20 +5,22 @@ set -e
 sudo systemctl restart guestregister # Sometimes registration fails on first boot.
 sudo zypper ref
 sudo zypper install -y -t pattern devel_basis
-sudo zypper install -y open-iscsi nfs-client cryptsetup
+sudo zypper install -y open-iscsi nfs-client cryptsetup device-mapper
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
 modprobe uio
 modprobe uio_pci_generic
 modprobe vfio_pci
 modprobe nvme-tcp
+modprobe dm_crypt
 touch /etc/modules-load.d/modules.conf
 cat > /etc/modules-load.d/modules.conf <<EOF
 uio
 uio_pci_generic
 vfio_pci
 nvme-tcp
+dm_crypt
 EOF
 
 echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages