Merge branch 'main' into terraform-to-opentofu

.github/workflows/test_aws_integration.yaml

@@ -5,11 +5,6 @@ on:
     - cron: "0 0 * * MON"
   workflow_dispatch:
     inputs:
-      branch:
-        description: 'Nebari branch to deploy, test, destroy'
-        required: true
-        default: main
-        type: string
       image-tag:
         description: 'Nebari image tag created by the nebari-docker-images repo'
         required: true
@@ -30,7 +25,6 @@ on:
 
 env:
   AWS_DEFAULT_REGION: "us-west-2"
-  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'main' }}
   NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
   TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}∏
 
@@ -45,7 +39,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: ${{ env.NEBARI_GH_BRANCH }}
           fetch-depth: 0
 
       - name: Set up Python

.github/workflows/test_azure_integration.yaml

@@ -5,11 +5,6 @@ on:
     - cron: "0 0 * * MON"
   workflow_dispatch:
     inputs:
-      branch:
-        description: 'Nebari branch to deploy, test, destroy'
-        required: true
-        default: main
-        type: string
       image-tag:
         description: 'Nebari image tag created by the nebari-docker-images repo'
         required: true
@@ -28,7 +23,6 @@ on:
         - error
 
 env:
-  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'main' }}
   NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
   TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
 
@@ -43,7 +37,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          ref: ${{ env.NEBARI_GH_BRANCH }}
           fetch-depth: 0
 
       - name: Set up Python

.github/workflows/test_gcp_integration.yaml

@@ -5,11 +5,6 @@ on:
     - cron: "0 0 * * MON"
   workflow_dispatch:
     inputs:
-      branch:
-        description: 'Nebari branch to deploy, test, destroy'
-        required: true
-        default: main
-        type: string
       image-tag:
         description: 'Nebari image tag created by the nebari-docker-images repo'
         required: true
@@ -28,7 +23,6 @@ on:
         - error
 
 env:
-  NEBARI_GH_BRANCH: ${{ github.event.inputs.branch || 'main' }}
   NEBARI_IMAGE_TAG: ${{ github.event.inputs.image-tag || 'main' }}
   TF_LOG: ${{ github.event.inputs.tf-log-level || 'info' }}
 
@@ -44,7 +38,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: ${{ env.NEBARI_GH_BRANCH }}
           fetch-depth: 0
 
       - name: Set up Python

src/_nebari/provider/cloud/google_cloud.py

@@ -57,13 +57,23 @@ def kubernetes_versions(region: str) -> List[str]:
     credentials, project_id = load_credentials()
     client = container_v1.ClusterManagerClient(credentials=credentials)
     response = client.get_server_config(
-        name=f"projects/{project_id}/locations/{region}"
+        name=f"projects/{project_id}/locations/{region}", timeout=300
     )
     supported_kubernetes_versions = response.valid_master_versions
 
     return filter_by_highest_supported_k8s_version(supported_kubernetes_versions)
 
 
+def get_patch_version(full_version: str) -> str:
+    return full_version.split("-")[0]
+
+
+def get_minor_version(full_version: str) -> str:
+    patch_version = get_patch_version(full_version)
+    parts = patch_version.split(".")
+    return f"{parts[0]}.{parts[1]}"
+
+
 def cluster_exists(cluster_name: str, region: str) -> bool:
     """Check if a GKE cluster exists."""
     credentials, project_id = load_credentials()

src/_nebari/stages/infrastructure/__init__.py

@@ -8,7 +8,7 @@
 import tempfile
 from typing import Annotated, Any, Dict, List, Literal, Optional, Tuple, Type, Union
 
-from pydantic import Field, field_validator, model_validator
+from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator
 
 from _nebari import constants
 from _nebari.provider import opentofu
@@ -136,7 +136,7 @@ class AWSAmiTypes(str, enum.Enum):
 
 class AWSNodeLaunchTemplate(schema.Base):
     pre_bootstrap_command: Optional[str] = None
-    ami_id: Optional[str] = None
+    _ami_id: Optional[str] = PrivateAttr(default=None)
 
 
 class AWSNodeGroupInputVars(schema.Base):
@@ -155,7 +155,7 @@ class AWSNodeGroupInputVars(schema.Base):
 def construct_aws_ami_type(gpu_enabled: bool, launch_template: AWSNodeLaunchTemplate):
     """Construct the AWS AMI type based on the provided parameters."""
 
-    if launch_template and launch_template.ami_id:
+    if launch_template and launch_template._ami_id:
         return "CUSTOM"
 
     if gpu_enabled:
@@ -359,6 +359,9 @@ class GCPNodeGroup(schema.Base):
 
 
 class GoogleCloudPlatformProvider(schema.Base):
+    # If you pass a major and minor version without a patch version
+    # yaml will pass it as a float, so we need to coerce it to a string
+    model_config = ConfigDict(coerce_numbers_to_str=True)
     region: str
     project: str
     kubernetes_version: str
@@ -373,6 +376,12 @@ class GoogleCloudPlatformProvider(schema.Base):
     master_authorized_networks_config: Optional[Union[GCPCIDRBlock, None]] = None
     private_cluster_config: Optional[Union[GCPPrivateClusterConfig, None]] = None
 
+    @field_validator("kubernetes_version", mode="before")
+    @classmethod
+    def transform_version_to_str(cls, value) -> str:
+        """Transforms the version to a string if it is not already."""
+        return str(value)
+
     @model_validator(mode="before")
     @classmethod
     def _check_input(cls, data: Any) -> Any:
@@ -383,8 +392,10 @@ def _check_input(cls, data: Any) -> Any:
             )
 
         available_kubernetes_versions = google_cloud.kubernetes_versions(data["region"])
-        print(available_kubernetes_versions)
-        if data["kubernetes_version"] not in available_kubernetes_versions:
+        if not any(
+            v.startswith(str(data["kubernetes_version"]))
+            for v in available_kubernetes_versions
+        ):
             raise ValueError(
                 f"\nInvalid `kubernetes-version` provided: {data['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available."
             )

src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf

@@ -55,13 +55,15 @@ resource "aws_security_group" "main" {
   vpc_id = aws_vpc.main.id
 
   ingress {
+    description = "Allow all ports and protocols to enter the security group"
     from_port   = 0
     to_port     = 0
     protocol    = "-1"
     cidr_blocks = [var.vpc_cidr_block]
   }
 
   egress {
+    description = "Allow all ports and protocols to exit the security group"
     from_port   = 0
     to_port     = 0
     protocol    = "-1"

src/_nebari/subcommands/init.py

@@ -409,13 +409,15 @@ def check_cloud_provider_kubernetes_version(
         versions = google_cloud.kubernetes_versions(region)
 
         if not kubernetes_version or kubernetes_version == LATEST:
-            kubernetes_version = get_latest_kubernetes_version(versions)
+            kubernetes_version = google_cloud.get_patch_version(
+                get_latest_kubernetes_version(versions)
+            )
             rich.print(
                 DEFAULT_KUBERNETES_VERSION_MSG.format(
                     kubernetes_version=kubernetes_version
                 )
             )
-        if kubernetes_version not in versions:
+        if not any(v.startswith(kubernetes_version) for v in versions):
             raise ValueError(
                 f"Invalid Kubernetes version `{kubernetes_version}`. Please refer to the GCP docs for a list of valid versions: {versions}"
             )

src/_nebari/upgrade.py

@@ -6,6 +6,7 @@
 
 import json
 import logging
+import os
 import re
 import secrets
 import string
@@ -1297,11 +1298,38 @@ def _version_specific_upgrade(
 
             urllib3.disable_warnings()
 
-            keycloak_admin = get_keycloak_admin(
-                server_url=f"https://{config['domain']}/auth/",
-                username="root",
-                password=config["security"]["keycloak"]["initial_root_password"],
+            keycloak_username = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "root")
+            keycloak_password = os.environ.get(
+                "KEYCLOAK_ADMIN_PASSWORD",
+                config["security"]["keycloak"]["initial_root_password"],
             )
+
+            try:
+                # Quick test to connect to Keycloak
+                keycloak_admin = get_keycloak_admin(
+                    server_url=f"https://{config['domain']}/auth/",
+                    username=keycloak_username,
+                    password=keycloak_password,
+                )
+            except ValueError as e:
+                if "invalid_grant" in str(e):
+                    rich.print(
+                        textwrap.dedent(
+                            """
+                            [red bold]Failed to connect to the Keycloak server.[/red bold]\n
+                            [yellow]Please set the [bold]KEYCLOAK_ADMIN_USERNAME[/bold] and [bold]KEYCLOAK_ADMIN_PASSWORD[/bold]
+                            environment variables with the Keycloak root credentials and try again.[/yellow]
+                            """
+                        )
+                    )
+                    exit()
+                else:
+                    # Handle other exceptions
+                    rich.print(
+                        f"[red bold]An unexpected error occurred: {repr(e)}[/red bold]"
+                    )
+                    exit()
+
             # Get client ID as role is bound to the JupyterHub client
             client_id = keycloak_admin.get_client_id("jupyterhub")
             role_name = "legacy-group-directory-creation-role"