Skip to content

Commit

Permalink
Merge branch 'develop' into workload_identity
Browse files Browse the repository at this point in the history
  • Loading branch information
Adam-D-Lewis authored May 17, 2024
2 parents 92a27f7 + fa07566 commit 240b6dc
Show file tree
Hide file tree
Showing 13 changed files with 267 additions and 19 deletions.
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ ci:
repos:
# general
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
rev: v4.6.0
hooks:
- id: end-of-file-fixer
exclude: "^docs-sphinx/cli.html"
Expand Down Expand Up @@ -51,13 +51,13 @@ repos:

# python
- repo: https://github.com/psf/black
rev: 24.3.0
rev: 24.4.2
hooks:
- id: black
args: ["--line-length=88", "--exclude=/src/_nebari/template/"]

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.3.5
rev: v0.4.3
hooks:
- id: ruff
args: ["--fix"]
Expand All @@ -73,7 +73,7 @@ repos:

# terraform
- repo: https://github.com/antonbabenko/pre-commit-terraform
rev: v1.88.4
rev: v1.89.1
hooks:
- id: terraform_fmt
args:
Expand Down
15 changes: 15 additions & 0 deletions RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,21 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->

---

### Release 2024.5.1 - May 13, 2024

## What's Changed

* make userscheduler run on general node group by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2415>
* Upgrade to Pydantic V2 by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2348>
* Pydantic2 PR fix by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2421>
* remove redundant pydantic class, fix bug by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2426>
* Update `python-keycloak` version pins constraints by @viniciusdc in <https://github.com/nebari-dev/nebari/pull/2435>
* add HERA_TOKEN env var to user pods by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2438>
* fix docs link by @Adam-D-Lewis in <https://github.com/nebari-dev/nebari/pull/2443>
* Update allowed admin groups by @aktech in <https://github.com/nebari-dev/nebari/pull/2429>

**Full Changelog**: <https://github.com/nebari-dev/nebari/compare/2024.4.1...2024.5.1>

## Release 2024.4.1 - April 20, 2024

### What's Changed
Expand Down
2 changes: 1 addition & 1 deletion src/_nebari/constants.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
CURRENT_RELEASE = "2024.4.1"
CURRENT_RELEASE = "2024.5.1"

# NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
# implemented in August 2023.
Expand Down
6 changes: 3 additions & 3 deletions src/_nebari/stages/infrastructure/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,9 +315,9 @@ class GCPNodeGroup(schema.Base):


DEFAULT_GCP_NODE_GROUPS = {
"general": GCPNodeGroup(instance="n1-standard-8", min_nodes=1, max_nodes=1),
"user": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5),
"worker": GCPNodeGroup(instance="n1-standard-4", min_nodes=0, max_nodes=5),
"general": GCPNodeGroup(instance="e2-highmem-4", min_nodes=1, max_nodes=1),
"user": GCPNodeGroup(instance="e2-standard-4", min_nodes=0, max_nodes=5),
"worker": GCPNodeGroup(instance="e2-standard-4", min_nodes=0, max_nodes=5),
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ def service_for_jhub_apps(name, url):
"url": url,
"external": True,
},
"oauth_no_confirm": True,
}

c.JupyterHub.services.extend(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
import json
import os
import urllib
from functools import reduce

from jupyterhub.traitlets import Callable
from oauthenticator.generic import GenericOAuthenticator
from traitlets import Bool, Unicode, Union


class KeyCloakOAuthenticator(GenericOAuthenticator):
"""
Since `oauthenticator` 16.3 `GenericOAuthenticator` supports group management.
This subclass adds role management on top of it, building on the new `manage_roles`
feature added in JupyterHub 5.0 (https://github.com/jupyterhub/jupyterhub/pull/4748).
"""

claim_roles_key = Union(
[Unicode(os.environ.get("OAUTH2_ROLES_KEY", "groups")), Callable()],
config=True,
help="""As `claim_groups_key` but for roles.""",
)

realm_api_url = Unicode(
config=True, help="""The keycloak REST API URL for the realm."""
)

reset_managed_roles_on_startup = Bool(True)

async def update_auth_model(self, auth_model):
auth_model = await super().update_auth_model(auth_model)
user_info = auth_model["auth_state"][self.user_auth_state_key]
user_roles = self._get_user_roles(user_info)
auth_model["roles"] = [{"name": role_name} for role_name in user_roles]
# note: because the roles check is comprehensive, we need to re-add the admin and user roles
if auth_model["admin"]:
auth_model["roles"].append({"name": "admin"})
if self.check_allowed(auth_model["name"], auth_model):
auth_model["roles"].append({"name": "user"})
return auth_model

async def load_managed_roles(self):
if not self.manage_roles:
raise ValueError(
"Managed roles can only be loaded when `manage_roles` is True"
)
token = await self._get_token()

# Get the clients list to find the "id" of "jupyterhub" client.
clients_data = await self._fetch_api(endpoint="clients/", token=token)
jupyterhub_clients = [
client for client in clients_data if client["clientId"] == "jupyterhub"
]
assert len(jupyterhub_clients) == 1
jupyterhub_client_id = jupyterhub_clients[0]["id"]

# Includes roles like "jupyterhub_admin", "jupyterhub_developer", "dask_gateway_developer"
client_roles = await self._fetch_api(
endpoint=f"clients/{jupyterhub_client_id}/roles", token=token
)
# Includes roles like "default-roles-nebari", "offline_access", "uma_authorization"
realm_roles = await self._fetch_api(endpoint="roles", token=token)
roles = {
role["name"]: {"name": role["name"], "description": role["description"]}
for role in [*realm_roles, *client_roles]
}
# we could use either `name` (e.g. "developer") or `path` ("/developer");
# since the default claim key returns `path`, it seems preferable.
group_name_key = "path"
for realm_role in realm_roles:
role_name = realm_role["name"]
role = roles[role_name]
# fetch role assignments to groups
groups = await self._fetch_api(f"roles/{role_name}/groups", token=token)
role["groups"] = [group[group_name_key] for group in groups]
# fetch role assignments to users
users = await self._fetch_api(f"roles/{role_name}/users", token=token)
role["users"] = [user["username"] for user in users]
for client_role in client_roles:
role_name = client_role["name"]
role = roles[role_name]
# fetch role assignments to groups
groups = await self._fetch_api(
f"clients/{jupyterhub_client_id}/roles/{role_name}/groups", token=token
)
role["groups"] = [group[group_name_key] for group in groups]
# fetch role assignments to users
users = await self._fetch_api(
f"clients/{jupyterhub_client_id}/roles/{role_name}/users", token=token
)
role["users"] = [user["username"] for user in users]

return list(roles.values())

def _get_user_roles(self, user_info):
if callable(self.claim_roles_key):
return set(self.claim_roles_key(user_info))
try:
return set(reduce(dict.get, self.claim_roles_key.split("."), user_info))
except TypeError:
self.log.error(
f"The claim_roles_key {self.claim_roles_key} does not exist in the user token"
)
return set()

async def _get_token(self) -> str:
http = self.http_client

body = urllib.parse.urlencode(
{
"client_id": self.client_id,
"client_secret": self.client_secret,
"grant_type": "client_credentials",
}
)
response = await http.fetch(
self.token_url,
method="POST",
body=body,
)
data = json.loads(response.body)
return data["access_token"] # type: ignore[no-any-return]

async def _fetch_api(self, endpoint: str, token: str):
response = await self.http_client.fetch(
f"{self.realm_api_url}/{endpoint}",
method="GET",
headers={"Authorization": f"Bearer {token}"},
)
return json.loads(response.body)


c.JupyterHub.authenticator_class = KeyCloakOAuthenticator
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ resource "helm_release" "jupyterhub" {

repository = "https://jupyterhub.github.io/helm-chart/"
chart = "jupyterhub"
version = "3.2.1"
version = "4.0.0-0.dev.git.6586.h0a16e5a0"

values = concat([
file("${path.module}/values.yaml"),
Expand Down Expand Up @@ -130,6 +130,7 @@ resource "helm_release" "jupyterhub" {
"01-theme.py" = file("${path.module}/files/jupyterhub/01-theme.py")
"02-spawner.py" = file("${path.module}/files/jupyterhub/02-spawner.py")
"03-profiles.py" = file("${path.module}/files/jupyterhub/03-profiles.py")
"04-auth.py" = file("${path.module}/files/jupyterhub/04-auth.py")
}

services = {
Expand All @@ -143,25 +144,25 @@ resource "helm_release" "jupyterhub" {
# for simple key value configuration with jupyterhub traitlets
# this hub.config property should be used
config = {
JupyterHub = {
authenticator_class = "generic-oauth"
}
Authenticator = {
enable_auth_state = true
}
GenericOAuthenticator = {
KeyCloakOAuthenticator = {
client_id = module.jupyterhub-openid-client.config.client_id
client_secret = module.jupyterhub-openid-client.config.client_secret
oauth_callback_url = "https://${var.external-url}/hub/oauth_callback"
authorize_url = module.jupyterhub-openid-client.config.authentication_url
token_url = module.jupyterhub-openid-client.config.token_url
userdata_url = module.jupyterhub-openid-client.config.userinfo_url
realm_api_url = module.jupyterhub-openid-client.config.realm_api_url
login_service = "Keycloak"
username_claim = "preferred_username"
claim_groups_key = "groups"
claim_roles_key = "roles"
allowed_groups = ["/analyst", "/developer", "/admin", "jupyterhub_admin", "jupyterhub_developer"]
admin_groups = ["/admin", "jupyterhub_admin"]
manage_groups = true
manage_roles = true
refresh_pre_spawn = true
validate_server_cert = false

Expand Down Expand Up @@ -283,6 +284,10 @@ module "jupyterhub-openid-client" {
var.jupyterhub-logout-redirect-url
]
jupyterlab_profiles_mapper = true
service-accounts-enabled = true
service-account-roles = [
"view-realm", "view-users", "view-clients"
]
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ resource "keycloak_openid_client" "main" {
access_type = "CONFIDENTIAL"
standard_flow_enabled = true

valid_redirect_uris = var.callback-url-paths
valid_redirect_uris = var.callback-url-paths
service_accounts_enabled = var.service-accounts-enabled
}


Expand Down Expand Up @@ -62,6 +63,33 @@ resource "keycloak_openid_user_attribute_protocol_mapper" "jupyterlab_profiles"
aggregate_attributes = true
}

data "keycloak_realm" "master" {
realm = "nebari"
}

data "keycloak_openid_client" "realm_management" {
realm_id = var.realm_id
client_id = "realm-management"
}

data "keycloak_role" "main-service" {
for_each = toset(var.service-account-roles)

realm_id = data.keycloak_realm.master.id
client_id = data.keycloak_openid_client.realm_management.id
name = each.key
}

resource "keycloak_openid_client_service_account_role" "main" {
for_each = toset(var.service-account-roles)

realm_id = var.realm_id
service_account_user_id = keycloak_openid_client.main.service_account_user_id
client_id = data.keycloak_openid_client.realm_management.id
role = data.keycloak_role.main-service[each.key].name
}


resource "keycloak_role" "main" {
for_each = toset(flatten(values(var.role_mapping)))

Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
output "config" {
description = "configuration credentials for connecting to openid client"
value = {
client_id = keycloak_openid_client.main.client_id
client_secret = keycloak_openid_client.main.client_secret
client_id = keycloak_openid_client.main.client_id
client_secret = keycloak_openid_client.main.client_secret
service_account_user_id = keycloak_openid_client.main.service_account_user_id

authentication_url = "https://${var.external-url}/auth/realms/${var.realm_id}/protocol/openid-connect/auth"
token_url = "https://${var.external-url}/auth/realms/${var.realm_id}/protocol/openid-connect/token"
userinfo_url = "https://${var.external-url}/auth/realms/${var.realm_id}/protocol/openid-connect/userinfo"
realm_api_url = "https://${var.external-url}/auth/admin/realms/${var.realm_id}"
callback_urls = var.callback-url-paths
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,19 @@ variable "external-url" {
}


variable "service-accounts-enabled" {
description = "Whether the client should have a service account created"
type = bool
default = false
}

variable "service-account-roles" {
description = "Roles to be granted to the service account. Requires setting service-accounts-enabled to true."
type = list(string)
default = []
}


variable "role_mapping" {
description = "Group to role mapping to establish for client"
type = map(list(string))
Expand Down
11 changes: 11 additions & 0 deletions src/_nebari/upgrade.py
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,17 @@ def _version_specific_upgrade(
return config


class Upgrade_2024_5_1(UpgradeStep):
version = "2024.5.1"

def _version_specific_upgrade(
self, config, start_version, config_filename: Path, *args, **kwargs
):
rich.print("Ready to upgrade to Nebari version [green]2024.5.1[/green].")

return config


__rounded_version__ = str(rounded_ver_parse(__version__))

# Manually-added upgrade steps must go above this line
Expand Down
Loading

0 comments on commit 240b6dc

Please sign in to comment.