diff --git a/molecule/mini-setup-ldap-mtls/molecule.yml b/molecule/mini-setup-ldap-mtls/molecule.yml new file mode 100644 index 000000000..a23f416af --- /dev/null +++ b/molecule/mini-setup-ldap-mtls/molecule.yml @@ -0,0 +1,244 @@ +--- +### Installs Confluent Platform Cluster on ubi9. +### RBAC enabled. +### MTLS enabled. +### Kafka Broker Customer Listener. +### SSO authentication using OIDC in Control center using Okta IdP + +driver: + name: docker +platforms: + - name: ldap1 + hostname: ldap1.confluent + groups: + - ldap_server + image: centos:centos8 + dockerfile: ../Dockerfile-centos8-base.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}1 + hostname: ${KRAFT_CONTROLLER:-zookeeper}1.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}2 + hostname: ${KRAFT_CONTROLLER:-zookeeper}2.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}3 + hostname: ${KRAFT_CONTROLLER:-zookeeper}3.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker1 + hostname: kafka-broker1.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker2 + hostname: kafka-broker2.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker3 + hostname: kafka-broker3.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: schema-registry1 + hostname: schema-registry1.confluent + groups: + - schema_registry + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-rest1 + hostname: kafka-rest1.confluent + groups: + - kafka_rest + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-connect1 + hostname: kafka-connect1.confluent + groups: + - kafka_connect + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ksql1 + hostname: ksql1.confluent + groups: + - ksql + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: control-center1 + hostname: control-center1.confluent + groups: + - control_center + image: redhat/ubi9-minimal + published_ports: + - "9021:9021" + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent +provisioner: + playbooks: + converge: ${MIGRATION_CONVERGE:-../collections_converge.yml} + inventory: + group_vars: + all: + mask_secrets: false + ssl_enabled: true + ssl_mutual_auth_enabled: true + ssl_client_authentication: required + + rbac_enabled: true + auth_mode: ldap + + mds_super_user: mds + mds_super_user_password: password + + schema_registry_ldap_user: schema-registry1 + schema_registry_ldap_password: password + + kafka_connect_ldap_user: kafka-connect1 + kafka_connect_ldap_password: password + + ksql_ldap_user: ksql1 + ksql_ldap_password: password + + kafka_rest_ldap_user: kafka-rest1 + kafka_rest_ldap_password: password + + control_center_ldap_user: control-center1 + control_center_ldap_password: password + + kafka_broker_custom_properties: + ldap.java.naming.factory.initial: com.sun.jndi.ldap.LdapCtxFactory + ldap.com.sun.jndi.ldap.read.timeout: 3000 + ldap.java.naming.provider.url: ldap://ldap1:389 + ldap.java.naming.security.principal: uid=mds,OU=rbac,DC=example,DC=com + ldap.java.naming.security.credentials: password + ldap.java.naming.security.authentication: simple + ldap.user.search.base: OU=rbac,DC=example,DC=com + ldap.group.search.base: OU=rbac,DC=example,DC=com + ldap.user.name.attribute: uid + ldap.user.memberof.attribute.pattern: CN=(.*),OU=rbac,DC=example,DC=com + ldap.group.name.attribute: cn + ldap.group.member.attribute.pattern: CN=(.*),OU=rbac,DC=example,DC=com + ldap.user.object.class: account + + ldap_server: + ldaps_enabled: false + ldaps_custom_certs: false + + ldap_admin_password: ldppassword + + ldap_rbac_group: rbac + ldap_dc: example + ldap_dc_extension: com + + ldap_users: + - username: "{{schema_registry_ldap_user}}" + password: "{{schema_registry_ldap_password}}" + uid: 9998 + guid: 98 + - username: "{{kafka_connect_ldap_user}}" + password: "{{kafka_connect_ldap_password}}" + uid: 9997 + guid: 97 + - username: "{{ksql_ldap_user}}" + password: "{{ksql_ldap_password}}" + uid: 9996 + guid: 96 + - username: "{{control_center_ldap_user}}" + password: "{{control_center_ldap_password}}" + uid: 9995 + guid: 95 + - username: "{{kafka_rest_ldap_user}}" + password: "{{kafka_rest_ldap_password}}" + uid: 9994 + guid: 94 + - username: "{{mds_super_user}}" + password: "{{mds_super_user_password}}" + uid: 9993 + guid: 93 \ No newline at end of file diff --git a/molecule/mini-setup-ldap-mtls/prepare.yml b/molecule/mini-setup-ldap-mtls/prepare.yml new file mode 100644 index 000000000..c1237c36a --- /dev/null +++ b/molecule/mini-setup-ldap-mtls/prepare.yml @@ -0,0 +1,9 @@ +--- +- name: Install and configure OpenLDAP + hosts: ldap_server + tasks: + - import_role: + name: confluent.test.ldap +- name: Install Zookeeper Cluster + import_playbook: confluent.platform.all + when: lookup('env', 'MIGRATION')|default('false') == 'true' diff --git a/molecule/mini-setup-ldap-mtls/verify.yml b/molecule/mini-setup-ldap-mtls/verify.yml new file mode 100644 index 000000000..ff4028d00 --- /dev/null +++ b/molecule/mini-setup-ldap-mtls/verify.yml @@ -0,0 +1,67 @@ +--- +### Validates that SSL Protocol is set. +### Validates ssl.client.authentication is set to REQUIRED. + +- name: Verify - kafka_controller + hosts: kafka_controller + gather_facts: false + tasks: + - import_role: + name: variables + - import_role: + name: confluent.test + tasks_from: check_property.yml + vars: + file_path: /etc/controller/server.properties + property: controller.quorum.voters + expected_value: "{{ kafka_controller_quorum_voters }}" + +- name: Verify - kafka_broker + hosts: kafka_broker + gather_facts: false + tasks: + - set_fact: + kraft_mode: "{{ ('kafka_controller' in groups.keys() and groups['kafka_controller'] | length > 0) }}" + + - import_role: + name: confluent.test + tasks_from: check_property.yml + vars: + file_path: /etc/kafka/server.properties + property: confluent.metadata.server.ssl.client.authentication + expected_value: REQUIRED + + - name: Get current time in milliseconds + command: date +%s%3N + register: current_time + + - name: Set current time in milliseconds + set_fact: + topic_name: "test-topic-{{current_time.stdout}}" + + - name: Display milliseconds since Unix epoch + debug: + msg: + - "topic_name: {{ topic_name }} will be created " + + - name: Create Kafka topic + shell: kafka-topics --create --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --command-config /etc/kafka/client.properties \ + --replication-factor 1 --partitions 6 + run_once: true + register: output + + - name: Create Topic Data + shell: | + seq 10 | kafka-console-producer --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --producer.config /etc/kafka/client.properties + run_once: true + + - name: Read Topic Data + shell: kafka-console-consumer --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --timeout-ms 10000 \ + --from-beginning --consumer.config /etc/kafka/client.properties + run_once: true + register: consumer_output + failed_when: + - "'1\n2\n3\n4\n5\n6\n7\n8\n9\n10' not in consumer_output.stdout" diff --git a/molecule/mini-setup-mtls/credentials b/molecule/mini-setup-mtls/credentials new file mode 100644 index 000000000..15a038bff --- /dev/null +++ b/molecule/mini-setup-mtls/credentials @@ -0,0 +1,2 @@ +user1: password1 +user2: password2 \ No newline at end of file diff --git a/molecule/mini-setup-mtls/molecule.yml b/molecule/mini-setup-mtls/molecule.yml new file mode 100644 index 000000000..c472310bd --- /dev/null +++ b/molecule/mini-setup-mtls/molecule.yml @@ -0,0 +1,178 @@ +--- +### Installs Confluent Platform Cluster on ubi9. +### RBAC enabled. +### MTLS enabled. +### Kafka Broker Customer Listener. +### SSO authentication using OIDC in Control center using Okta IdP + +driver: + name: docker +platforms: + - name: ${KRAFT_CONTROLLER:-zookeeper}1 + hostname: ${KRAFT_CONTROLLER:-zookeeper}1.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}2 + hostname: ${KRAFT_CONTROLLER:-zookeeper}2.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}3 + hostname: ${KRAFT_CONTROLLER:-zookeeper}3.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker1 + hostname: kafka-broker1.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker2 + hostname: kafka-broker2.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker3 + hostname: kafka-broker3.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: schema-registry1 + hostname: schema-registry1.confluent + groups: + - schema_registry + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-rest1 + hostname: kafka-rest1.confluent + groups: + - kafka_rest + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-connect1 + hostname: kafka-connect1.confluent + groups: + - kafka_connect + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ksql1 + hostname: ksql1.confluent + groups: + - ksql + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: control-center1 + hostname: control-center1.confluent + groups: + - control_center + image: redhat/ubi9-minimal + published_ports: + - "9021:9021" + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent +provisioner: + playbooks: + converge: ${MIGRATION_CONVERGE:-../collections_converge.yml} + inventory: + group_vars: + all: + mask_secrets: false + ssl_enabled: true + ssl_mutual_auth_enabled: true + ssl_client_authentication: required + + rbac_enabled: true + auth_mode: mtls + + principal_mapping_rules: + - "RULE:.*CN=([a-zA-Z0-9.-_]*).*$$/$$1/" + - "DEFAULT" + + kafka_broker_custom_properties: + confluent.metadata.server.user.store: FILE + confluent.metadata.server.user.store.file.path: /tmp/credentials + + kafka_broker_custom_listeners: + client_listener: + name: CLIENT + port: 9093 + principal_mapping_rules: + - "DEFAULT" diff --git a/molecule/mini-setup-mtls/prepare.yml b/molecule/mini-setup-mtls/prepare.yml new file mode 100644 index 000000000..847862ddb --- /dev/null +++ b/molecule/mini-setup-mtls/prepare.yml @@ -0,0 +1,12 @@ +--- +- name: File Based login in C3/CLI configs via override + hosts: kafka_broker + tasks: + - name: Copy Credentials file to MDS + copy: + src: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/credentials" + dest: "/tmp/credentials" + +- name: Install Zookeeper Cluster + import_playbook: confluent.platform.all + when: lookup('env', 'MIGRATION')|default('false') == 'true' diff --git a/molecule/mini-setup-mtls/verify.yml b/molecule/mini-setup-mtls/verify.yml new file mode 100644 index 000000000..ff4028d00 --- /dev/null +++ b/molecule/mini-setup-mtls/verify.yml @@ -0,0 +1,67 @@ +--- +### Validates that SSL Protocol is set. +### Validates ssl.client.authentication is set to REQUIRED. + +- name: Verify - kafka_controller + hosts: kafka_controller + gather_facts: false + tasks: + - import_role: + name: variables + - import_role: + name: confluent.test + tasks_from: check_property.yml + vars: + file_path: /etc/controller/server.properties + property: controller.quorum.voters + expected_value: "{{ kafka_controller_quorum_voters }}" + +- name: Verify - kafka_broker + hosts: kafka_broker + gather_facts: false + tasks: + - set_fact: + kraft_mode: "{{ ('kafka_controller' in groups.keys() and groups['kafka_controller'] | length > 0) }}" + + - import_role: + name: confluent.test + tasks_from: check_property.yml + vars: + file_path: /etc/kafka/server.properties + property: confluent.metadata.server.ssl.client.authentication + expected_value: REQUIRED + + - name: Get current time in milliseconds + command: date +%s%3N + register: current_time + + - name: Set current time in milliseconds + set_fact: + topic_name: "test-topic-{{current_time.stdout}}" + + - name: Display milliseconds since Unix epoch + debug: + msg: + - "topic_name: {{ topic_name }} will be created " + + - name: Create Kafka topic + shell: kafka-topics --create --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --command-config /etc/kafka/client.properties \ + --replication-factor 1 --partitions 6 + run_once: true + register: output + + - name: Create Topic Data + shell: | + seq 10 | kafka-console-producer --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --producer.config /etc/kafka/client.properties + run_once: true + + - name: Read Topic Data + shell: kafka-console-consumer --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --timeout-ms 10000 \ + --from-beginning --consumer.config /etc/kafka/client.properties + run_once: true + register: consumer_output + failed_when: + - "'1\n2\n3\n4\n5\n6\n7\n8\n9\n10' not in consumer_output.stdout" diff --git a/molecule/mini-setup-oauth-mtls/molecule.yml b/molecule/mini-setup-oauth-mtls/molecule.yml new file mode 100644 index 000000000..8717f46d8 --- /dev/null +++ b/molecule/mini-setup-oauth-mtls/molecule.yml @@ -0,0 +1,240 @@ +--- +### Installs Confluent Platform Cluster on ubi9. +### RBAC enabled. +### MTLS enabled. +### Kafka Broker Customer Listener. +### SSO authentication using OIDC in Control center using Okta IdP + +driver: + name: docker +platforms: + - name: oauth1 + hostname: oauth1.confluent + groups: + - oauth_server + image: quay.io/keycloak/keycloak:latest + env: + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KC_HOSTNAME: oauth1 + KC_HTTPS_CERTIFICATE_FILE: /idp-cert.pem + KC_HTTPS_CERTIFICATE_KEY_FILE: /idp-key.pem + KEYCLOAK_HTTP_PORT: "8080" + KEYCLOAK_HTTPS_PORT: "8443" + dockerfile: ../Dockerfile-oauth.j2 + published_ports: + - "8080:8080" + - "8443:8443" + command: start-dev + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}1 + hostname: ${KRAFT_CONTROLLER:-zookeeper}1.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}2 + hostname: ${KRAFT_CONTROLLER:-zookeeper}2.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ${KRAFT_CONTROLLER:-zookeeper}3 + hostname: ${KRAFT_CONTROLLER:-zookeeper}3.confluent + groups: + - ${CONTROLLER_HOSTGROUP:-zookeeper} + - ${CONTROLLER_HOSTGROUP:-zookeeper}_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker1 + hostname: kafka-broker1.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker2 + hostname: kafka-broker2.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-broker3 + hostname: kafka-broker3.confluent + groups: + - kafka_broker + - kafka_broker_migration + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: schema-registry1 + hostname: schema-registry1.confluent + groups: + - schema_registry + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-rest1 + hostname: kafka-rest1.confluent + groups: + - kafka_rest + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: kafka-connect1 + hostname: kafka-connect1.confluent + groups: + - kafka_connect + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: ksql1 + hostname: ksql1.confluent + groups: + - ksql + image: redhat/ubi9-minimal + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent + - name: control-center1 + hostname: control-center1.confluent + groups: + - control_center + image: redhat/ubi9-minimal + published_ports: + - "9021:9021" + dockerfile: ../Dockerfile-rhel-java17.j2 + command: "" + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + networks: + - name: confluent +provisioner: + playbooks: + converge: ${MIGRATION_CONVERGE:-../collections_converge.yml} + inventory: + group_vars: + all: + mask_secrets: false + ssl_enabled: true + ssl_mutual_auth_enabled: true + ssl_client_authentication: requested + + rbac_enabled: true + auth_mode: oauth + + #impersonation_super_users: + # - 'C=US,ST=Ca,L=PaloAlto,O=CONFLUENT,OU=TEST,CN=kafka_broker' + # - 'C=US,ST=Ca,L=PaloAlto,O=CONFLUENT,OU=TEST,CN=kafka_rest' + # - 'C=US,ST=Ca,L=PaloAlto,O=CONFLUENT,OU=TEST,CN=schema_registry' + # - 'C=US,ST=Ca,L=PaloAlto,O=CONFLUENT,OU=TEST,CN=kafka_connect' + + kafka_broker_custom_listeners: + client_listener: + name: CLIENT + port: 9093 + + keycloak_oauth_server_port: 8443 + keycloak_http_protocol: https + + oauth_idp_cert_path: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/oauthcertfile.pem" + oauth_superuser_client_id: superuser + oauth_superuser_client_password: my-secret + + oauth_sub_claim: client_id + oauth_groups_claim: groups + oauth_token_uri: https://oauth1:8443/realms/cp-ansible-realm/protocol/openid-connect/token + oauth_issuer_url: https://oauth1:8443/realms/cp-ansible-realm + oauth_jwks_uri: https://oauth1:8443/realms/cp-ansible-realm/protocol/openid-connect/certs + oauth_expected_audience: Confluent,account,api://default + + schema_registry_oauth_user: schema_registry + schema_registry_oauth_password: my-secret + + kafka_rest_oauth_user: kafka_rest + kafka_rest_oauth_password: my-secret + + kafka_connect_oauth_user: kafka_connect + kafka_connect_oauth_password: my-secret + + ksql_oauth_user: ksql + ksql_oauth_password: my-secret + + control_center_oauth_user: control_center + control_center_oauth_password: my-secret + + sso_mode: oidc + sso_groups_claim: groups + sso_sub_claim: sub + sso_groups_scope: groups + sso_issuer_url: https://dev-59009577.okta.com/oauth2/aus96p2og3u7Cpwu65d7 + sso_jwks_uri: https://dev-59009577.okta.com/oauth2/aus96p2og3u7Cpwu65d7/v1/keys + sso_authorize_uri: https://dev-59009577.okta.com/oauth2/aus96p2og3u7Cpwu65d7/v1/authorize + sso_token_uri: https://dev-59009577.okta.com/oauth2/aus96p2og3u7Cpwu65d7/v1/token + sso_device_authorization_uri: https://dev-59009577.okta.com/oauth2/aus96p2og3u7Cpwu65d7/v1/device/authorize + sso_cli: true + sso_client_id: ${OKTA_CLIENT:-user} + sso_client_password: ${OKTA_PASSWORD:-pass} + sso_idp_cert_path: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/ssocertfile.pem" diff --git a/molecule/mini-setup-oauth-mtls/prepare.yml b/molecule/mini-setup-oauth-mtls/prepare.yml new file mode 100644 index 000000000..1db3b0149 --- /dev/null +++ b/molecule/mini-setup-oauth-mtls/prepare.yml @@ -0,0 +1,17 @@ +--- +- name: Provision Oauth Server + import_playbook: ../oauth.yml + +- name: Download SSO IDP cert # temporary, till we are using OKTA IDP + hosts: localhost + tasks: + - shell: openssl s_client -showcerts -connect dev-59009577.okta.com:443 /dev/null|openssl x509 -outform PEM > ssocertfile.pem + +- name: Download OAuth IDP cert + hosts: localhost + tasks: + - shell: openssl s_client -showcerts -connect localhost:8443 /dev/null|openssl x509 -outform PEM > oauthcertfile.pem + +- name: Install Zookeeper Cluster + import_playbook: confluent.platform.all + when: lookup('env', 'MIGRATION')|default('false') == 'true' diff --git a/molecule/mini-setup-oauth-mtls/verify.yml b/molecule/mini-setup-oauth-mtls/verify.yml new file mode 100644 index 000000000..f575e5440 --- /dev/null +++ b/molecule/mini-setup-oauth-mtls/verify.yml @@ -0,0 +1,67 @@ +--- +### Validates that SSL Protocol is set. +### Validates ssl.client.authentication is set to REQUIRED. + +- name: Verify - kafka_controller + hosts: kafka_controller + gather_facts: false + tasks: + - import_role: + name: variables + - import_role: + name: confluent.test + tasks_from: check_property.yml + vars: + file_path: /etc/controller/server.properties + property: controller.quorum.voters + expected_value: "{{ kafka_controller_quorum_voters }}" + +- name: Verify - kafka_broker + hosts: kafka_broker + gather_facts: false + tasks: + - set_fact: + kraft_mode: "{{ ('kafka_controller' in groups.keys() and groups['kafka_controller'] | length > 0) }}" + + - import_role: + name: confluent.test + tasks_from: check_property.yml + vars: + file_path: /etc/kafka/server.properties + property: confluent.metadata.server.ssl.client.authentication + expected_value: REQUESTED + + - name: Get current time in milliseconds + command: date +%s%3N + register: current_time + + - name: Set current time in milliseconds + set_fact: + topic_name: "test-topic-{{current_time.stdout}}" + + - name: Display milliseconds since Unix epoch + debug: + msg: + - "topic_name: {{ topic_name }} will be created " + + - name: Create Kafka topic + shell: kafka-topics --create --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --command-config /etc/kafka/client.properties \ + --replication-factor 1 --partitions 6 + run_once: true + register: output + + - name: Create Topic Data + shell: | + seq 10 | kafka-console-producer --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --producer.config /etc/kafka/client.properties + run_once: true + + - name: Read Topic Data + shell: kafka-console-consumer --topic "{{ topic_name }}" \ + --bootstrap-server kafka-broker1:9091 --timeout-ms 10000 \ + --from-beginning --consumer.config /etc/kafka/client.properties + run_once: true + register: consumer_output + failed_when: + - "'1\n2\n3\n4\n5\n6\n7\n8\n9\n10' not in consumer_output.stdout" diff --git a/roles/common/tasks/config_validations.yml b/roles/common/tasks/config_validations.yml index bc9b2f22c..6e7d88d57 100644 --- a/roles/common/tasks/config_validations.yml +++ b/roles/common/tasks/config_validations.yml @@ -161,13 +161,13 @@ - assert: that: - - auth_mode in ['ldap', 'ldap_with_oauth', 'oauth', 'none'] + - auth_mode in ['ldap', 'ldap_with_oauth', 'oauth', 'mtls', 'none'] fail_msg: "auth_mode must be one of 'ldap', 'ldap_with_oauth', 'oauth', 'none'" tags: validate - assert: that: - - auth_mode in ['ldap', 'ldap_with_oauth', 'oauth'] + - auth_mode in ['ldap', 'ldap_with_oauth', 'oauth', 'mtls'] fail_msg: "When RBAC is enabled, auth_mode must be one of 'ldap', 'ldap_with_oauth', 'oauth'" when: - rbac_enabled|bool