diff --git a/test_framework/terraform/aws/oracle/main.tf b/test_framework/terraform/aws/oracle/main.tf index 97f5367da0..47d6aefb3e 100644 --- a/test_framework/terraform/aws/oracle/main.tf +++ b/test_framework/terraform/aws/oracle/main.tf @@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + description = "Allow longhorn-ui nodeport" + from_port = 30000 + to_port = 30000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { description = "Allow UDP connection for longhorn-webhooks" from_port = 0 diff --git a/test_framework/terraform/aws/oracle/output.tf b/test_framework/terraform/aws/oracle/output.tf index 3e0941a2c1..46c64dcf2b 100644 --- a/test_framework/terraform/aws/oracle/output.tf +++ b/test_framework/terraform/aws/oracle/output.tf @@ -47,6 +47,38 @@ output "load_balancer_url" { value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null } +output "instance_mapping" { + depends_on = [ + aws_instance.lh_aws_instance_controlplane_k3s, + aws_instance.lh_aws_instance_worker_k3s, + ] + + value = jsonencode( + concat( + [ + for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : { + "name": controlplane_instance.private_dns, + "id": controlplane_instance.id + } + + ], + [ + for worker_instance in aws_instance.lh_aws_instance_worker_k3s : { + "name": worker_instance.private_dns, + "id": worker_instance.id + } + ] + ) + ) +} + +output "controlplane_public_ip" { + depends_on = [ + aws_eip.lh_aws_eip_controlplane + ] + value = aws_eip.lh_aws_eip_controlplane[0].public_ip +} + output "resource_suffix" { depends_on = [ random_string.random_suffix diff --git a/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl index bc8406a8d7..821f2b4723 100755 --- a/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl @@ -2,7 +2,7 @@ sudo yum update -y sudo yum group install -y "Development Tools" -sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup +sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper sudo systemctl -q enable iscsid sudo systemctl start iscsid # disable nm-cloud-setup otherwise k3s-agent service won’t start. @@ -12,12 +12,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl index 2fd54813f3..19f5b99fc2 100644 --- a/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl @@ -2,7 +2,7 @@ sudo yum update -y sudo yum group install -y "Development Tools" -sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup nc +sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper nc sudo systemctl -q enable iscsid sudo systemctl start iscsid sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer @@ -11,12 +11,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/rhel/main.tf b/test_framework/terraform/aws/rhel/main.tf index 97f5367da0..47d6aefb3e 100644 --- a/test_framework/terraform/aws/rhel/main.tf +++ b/test_framework/terraform/aws/rhel/main.tf @@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + description = "Allow longhorn-ui nodeport" + from_port = 30000 + to_port = 30000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { description = "Allow UDP connection for longhorn-webhooks" from_port = 0 diff --git a/test_framework/terraform/aws/rhel/output.tf b/test_framework/terraform/aws/rhel/output.tf index cbae005ce5..291f420816 100644 --- a/test_framework/terraform/aws/rhel/output.tf +++ b/test_framework/terraform/aws/rhel/output.tf @@ -47,6 +47,38 @@ output "load_balancer_url" { value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null } +output "instance_mapping" { + depends_on = [ + aws_instance.lh_aws_instance_controlplane_k3s, + aws_instance.lh_aws_instance_worker_k3s, + ] + + value = jsonencode( + concat( + [ + for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : { + "name": controlplane_instance.private_dns, + "id": controlplane_instance.id + } + + ], + [ + for worker_instance in aws_instance.lh_aws_instance_worker_k3s : { + "name": worker_instance.private_dns, + "id": worker_instance.id + } + ] + ) + ) +} + +output "controlplane_public_ip" { + depends_on = [ + aws_eip.lh_aws_eip_controlplane + ] + value = aws_eip.lh_aws_eip_controlplane[0].public_ip +} + output "resource_suffix" { depends_on = [ random_string.random_suffix diff --git a/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl index 81205dccc3..45c9e8580a 100755 --- a/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl @@ -10,7 +10,7 @@ fi sudo yum update -y sudo yum group install -y "Development Tools" -sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup +sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper sudo systemctl -q enable iscsid sudo systemctl start iscsid sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer @@ -19,12 +19,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl index 3ea4da17c5..5c3cebefd9 100755 --- a/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl @@ -10,7 +10,7 @@ fi sudo yum update -y sudo yum group install -y "Development Tools" -sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup nc +sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper nc sudo systemctl -q enable iscsid sudo systemctl start iscsid sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer @@ -19,12 +19,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/rockylinux/main.tf b/test_framework/terraform/aws/rockylinux/main.tf index dbb901d073..e40bb11033 100644 --- a/test_framework/terraform/aws/rockylinux/main.tf +++ b/test_framework/terraform/aws/rockylinux/main.tf @@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + description = "Allow longhorn-ui nodeport" + from_port = 30000 + to_port = 30000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { description = "Allow UDP connection for longhorn-webhooks" from_port = 0 diff --git a/test_framework/terraform/aws/rockylinux/output.tf b/test_framework/terraform/aws/rockylinux/output.tf index 804fa2d099..b02f45cdcd 100644 --- a/test_framework/terraform/aws/rockylinux/output.tf +++ b/test_framework/terraform/aws/rockylinux/output.tf @@ -47,6 +47,38 @@ output "load_balancer_url" { value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null } +output "instance_mapping" { + depends_on = [ + aws_instance.lh_aws_instance_controlplane_k3s, + aws_instance.lh_aws_instance_worker_k3s, + ] + + value = jsonencode( + concat( + [ + for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : { + "name": controlplane_instance.private_dns, + "id": controlplane_instance.id + } + + ], + [ + for worker_instance in aws_instance.lh_aws_instance_worker_k3s : { + "name": worker_instance.private_dns, + "id": worker_instance.id + } + ] + ) + ) +} + +output "controlplane_public_ip" { + depends_on = [ + aws_eip.lh_aws_eip_controlplane + ] + value = aws_eip.lh_aws_eip_controlplane[0].public_ip +} + output "resource_suffix" { depends_on = [ random_string.random_suffix diff --git a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl index d25035b75a..e5ea498944 100755 --- a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl @@ -10,7 +10,7 @@ fi # Do not arbitrarily run "dnf update", as this will effectively move us up to the latest minor release. sudo dnf group install -y "Development Tools" -sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup +sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper sudo systemctl -q enable iscsid sudo systemctl start iscsid @@ -18,12 +18,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl index bee8947b47..d2c2a65f70 100755 --- a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl @@ -10,7 +10,7 @@ fi # Do not arbitrarily run "dnf update", as this will effectively move us up to the latest minor release. sudo dnf group install -y "Development Tools" -sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup jq nmap-ncat +sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools cryptsetup device-mapper jq nmap-ncat sudo systemctl -q enable iscsid sudo systemctl start iscsid @@ -18,12 +18,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/sle-micro/main.tf b/test_framework/terraform/aws/sle-micro/main.tf index 50ff46ae25..9ee959809b 100644 --- a/test_framework/terraform/aws/sle-micro/main.tf +++ b/test_framework/terraform/aws/sle-micro/main.tf @@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_public" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + description = "Allow longhorn-ui nodeport" + from_port = 30000 + to_port = 30000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { description = "Allow UDP connection for longhorn-webhooks" from_port = 0 diff --git a/test_framework/terraform/aws/sle-micro/output.tf b/test_framework/terraform/aws/sle-micro/output.tf index 1946e2eaa4..f248731368 100644 --- a/test_framework/terraform/aws/sle-micro/output.tf +++ b/test_framework/terraform/aws/sle-micro/output.tf @@ -46,6 +46,38 @@ output "load_balancer_url" { value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null } +output "instance_mapping" { + depends_on = [ + aws_instance.lh_aws_instance_controlplane_k3s, + aws_instance.lh_aws_instance_worker_k3s, + ] + + value = jsonencode( + concat( + [ + for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : { + "name": controlplane_instance.private_dns, + "id": controlplane_instance.id + } + + ], + [ + for worker_instance in aws_instance.lh_aws_instance_worker_k3s : { + "name": worker_instance.private_dns, + "id": worker_instance.id + } + ] + ) + ) +} + +output "controlplane_public_ip" { + depends_on = [ + aws_eip.lh_aws_eip_controlplane + ] + value = aws_eip.lh_aws_eip_controlplane[0].public_ip +} + output "resource_suffix" { depends_on = [ random_string.random_suffix diff --git a/test_framework/terraform/aws/sles/k3s_instances.tf b/test_framework/terraform/aws/sles/k3s_instances.tf index 6ed5ef3ca4..4dddd7217d 100644 --- a/test_framework/terraform/aws/sles/k3s_instances.tf +++ b/test_framework/terraform/aws/sles/k3s_instances.tf @@ -127,7 +127,7 @@ resource "null_resource" "rsync_kubeconfig_file" { inline = [ "cloud-init status --wait", "if [ \"`cloud-init status | grep error`\" ]; then sudo cat /var/log/cloud-init-output.log; fi", - "RETRY=0; MAX_RETRY=450; until([ -f /etc/rancher/k3s/k3s.yaml ] && [ `sudo /usr/local/bin/kubectl get node -o jsonpath='{.items[*].status.conditions}' | jq '.[] | select(.type == \"Ready\").status' | grep -ci true` -eq $((${var.lh_aws_instance_count_controlplane} + ${var.lh_aws_instance_count_worker})) ]); do echo \"waiting for k3s cluster nodes to be running\"; sleep 2; if [ $RETRY -eq $MAX_RETRY ]; then break; fi; RETRY=$((RETRY+1)); done" + "RETRY=0; MAX_RETRY=450; until([ -f /etc/rancher/k3s/k3s.yaml ] && [ `sudo /usr/local/bin/kubectl get node -o jsonpath='{.items[*].status.conditions}' | jq '.[] | select(.type == \"Ready\").status' | grep -ci true` -eq $((${var.lh_aws_instance_count_controlplane} + ${var.lh_aws_instance_count_worker})) ]); do echo \"waiting for k3s cluster nodes to be running\"; sleep 2; if [ $RETRY -eq $MAX_RETRY ]; then echo \"cluster nodes initialization timeout ...\"; sleep 86400; fi; RETRY=$((RETRY+1)); done" ] connection { diff --git a/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl index eccb2e8ffb..d888d29949 100755 --- a/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl @@ -5,7 +5,7 @@ set -e sudo systemctl restart guestregister # Sometimes registration fails on first boot. sudo zypper ref sudo zypper install -y -t pattern devel_basis -sudo zypper install -y open-iscsi nfs-client cryptsetup +sudo zypper install -y open-iscsi nfs-client cryptsetup device-mapper sudo systemctl -q enable iscsid sudo systemctl start iscsid @@ -13,12 +13,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl index 676b87a32f..f1c8755125 100755 --- a/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl @@ -5,7 +5,7 @@ set -e sudo systemctl restart guestregister # Sometimes registration fails on first boot. sudo zypper ref sudo zypper install -y -t pattern devel_basis -sudo zypper install -y open-iscsi nfs-client cryptsetup +sudo zypper install -y open-iscsi nfs-client cryptsetup device-mapper sudo systemctl -q enable iscsid sudo systemctl start iscsid @@ -13,12 +13,14 @@ modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/ubuntu/main.tf b/test_framework/terraform/aws/ubuntu/main.tf index 1d38dc22b1..f80a079df3 100644 --- a/test_framework/terraform/aws/ubuntu/main.tf +++ b/test_framework/terraform/aws/ubuntu/main.tf @@ -77,6 +77,13 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + description = "Allow longhorn-ui nodeport" + from_port = 30000 + to_port = 30000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } ingress { description = "Allow k8s API server port" diff --git a/test_framework/terraform/aws/ubuntu/output.tf b/test_framework/terraform/aws/ubuntu/output.tf index 9e865901a5..59dfb3b093 100644 --- a/test_framework/terraform/aws/ubuntu/output.tf +++ b/test_framework/terraform/aws/ubuntu/output.tf @@ -47,6 +47,38 @@ output "load_balancer_url" { value = var.create_load_balancer ? aws_lb.lh_aws_lb[0].dns_name : null } +output "instance_mapping" { + depends_on = [ + aws_instance.lh_aws_instance_controlplane_k3s, + aws_instance.lh_aws_instance_worker_k3s, + ] + + value = jsonencode( + concat( + [ + for controlplane_instance in aws_instance.lh_aws_instance_controlplane_k3s : { + "name": controlplane_instance.private_dns, + "id": controlplane_instance.id + } + + ], + [ + for worker_instance in aws_instance.lh_aws_instance_worker_k3s : { + "name": worker_instance.private_dns, + "id": worker_instance.id + } + ] + ) + ) +} + +output "controlplane_public_ip" { + depends_on = [ + aws_eip.lh_aws_eip_controlplane + ] + value = aws_eip.lh_aws_eip_controlplane[0].public_ip +} + output "resource_suffix" { depends_on = [ random_string.random_suffix diff --git a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl index ead6ed46e4..cb13a443c8 100755 --- a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl @@ -1,18 +1,20 @@ #!/bin/bash apt-get update -apt-get install -y nfs-common cryptsetup linux-modules-extra-`uname -r` +apt-get install -y nfs-common cryptsetup dmsetup linux-modules-extra-`uname -r` modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl index c3be94323e..5de3d402aa 100755 --- a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl @@ -1,18 +1,20 @@ #!/bin/bash apt-get update -apt-get install -y nfs-common cryptsetup linux-modules-extra-`uname -r` +apt-get install -y nfs-common cryptsetup dmsetup linux-modules-extra-`uname -r` modprobe uio modprobe uio_pci_generic modprobe vfio_pci modprobe nvme-tcp +modprobe dm_crypt touch /etc/modules-load.d/modules.conf cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages