Merge branch 'develop'

charts/confluent/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: lsdmesp-confluent
-version: "0.6.0"
-appVersion: "0.6.0"
+version: "0.6.1"
+appVersion: "0.6.1"
 description: 'LSDMESP CFK: LSD Event Streaming Platform with Confluent Operator'
 keywords:
   - lsdmesp

charts/confluent/templates/020.schemaregistry.yaml

@@ -156,6 +156,17 @@ spec:
             client_type: $1
   oneReplicaPerNode: true
   podTemplate:
+    probe:
+      liveness:
+        failureThreshold: 3
+        periodSeconds: 30
+        timeoutSeconds: 5
+      readiness:
+        failureThreshold: 10
+        initialDelaySeconds: 120
+        periodSeconds: 45
+        successThreshold: 1
+        timeoutSeconds: 60
     podSecurityContext:
       fsGroup: 1000
       runAsUser: 1000

charts/confluent/templates/030.connect.yaml

@@ -33,6 +33,9 @@ spec:
         - name: debezium-connector-mysql
           owner: debezium
           version: 2.2.1
+        - name: debezium-connector-postgresql
+          owner: debezium
+          version: 2.5.4
         - name: kafka-connect-elasticsearch-source
           owner: dariobalinzo
           version: 1.5.5

charts/confluent/templates/050.controlcenter-ingress.yaml

@@ -17,7 +17,7 @@ spec:
         paths:
           - backend:
               service:
-                name: controlcenter-0-internal
+                name: controlcenter
                 port:
                   number: 9021
             path: /

charts/confluent/values.yaml

@@ -276,7 +276,7 @@ lsdmesp:
         # -- Annotations for the Control Center ingress
         annotations: {}
         # -- Base domain configuration for the Control Center ingress
-        baseDomain: "example.com"
+        baseDomain: "apps.mesp.lsdopen.io"
         # -- TLS host configuration will be dynamically built based on the base domain. You can change the secret name here.
         tls:
           secretName: controlcenter-tls

charts/demos/README.md

@@ -0,0 +1,221 @@
+# Create Kind cluster
+`kind create cluster --config ~/.kindconf/lsdmesp.conf`
+
+# Create Ingress controller
+`kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml`
+
+# Deploy Postgres
+`cd ./demos/bin`
+`./deploy_postgres.sh`
+
+# Port-forward for Postgres
+`kubectl port-forward svc/postgres 5432:5432 --namespace postgres`
+
+# Startup Confluent MESP cluster
+`cd ./confluent`
+`./install.sh`
+
+# Deploy Monolith app and Webapp, as well as scaled down Webapp 2, Webapp 3 and Microservice app
+`cd ./demos/bin`
+`./deploy_appmod.sh`
+
+# Prep Army Knife for ksql cli
+Exec into the Army Knife and start the KSQL CLI, but first create a properties file (/root/ksql.client.properties) with the following:
+```
+ssl.truststore.location=/root/lsdmesp.truststore.jks
+ssl.truststore.password=112233
+```
+
+# Port-forward for Monolith
+`kubectl port-forward svc/monolith-service 8000:8000 --namespace app-modernisation`
+
+# Port-forward for Control Center
+`kubectl port-forward controlcenter-0 9021:9021 --namespace lsdmesp-confluent`
+
+# Add CFRBs
+`cd ./demos/templates`
+`kubectl apply -f 010-cf-additional-cfrb.yaml -n lsdmesp-confluent`
+
+# Create topics
+Create topics in CCC with user `peter`
+```
+postgres.bank.transactions
+postgres.bank.accounts
+postgres.bank.customers
+express.bank.transactions
+```
+
+Or from army knife:
+
+```
+kafka-topics --create --topic postgres.bank.transactions --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --partitions 6 --replication-factor 3 --command-config /root/etc/client.properties
+kafka-topics --create --topic postgres.bank.accounts --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --partitions 6 --replication-factor 3 --command-config /root/etc/client.properties
+kafka-topics --create --topic postgres.bank.customers --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --partitions 6 --replication-factor 3 --command-config /root/etc/client.properties
+kafka-topics --create --topic express.bank.transactions --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --partitions 6 --replication-factor 3 --command-config /root/etc/client.properties
+```
+
+# Add transactions
+Navigate to http://webapp.apps.mesp.lsdopen.io
+
+Add the following transactions:
+```
+5208-0272-3184-5035:5
+5188-6083-8011-0307:100
+5188-6083-8011-0307:250
+5188-6083-8011-0307:400
+5588-6461-5550-9705:120
+```
+
+# Create source connector
+Upload `postgres_source.json`
+Show data in topics.
+
+# Scale up Microservice deployment. Scale down webapp-mf1 and scale up webapp-mf2
+```
+kubectl scale deployment microservice-deployment -n app-modernisation --replicas=1
+kubectl scale deployment webapp-mf1-deployment -n app-modernisation --replicas=0
+kubectl scale deployment webapp-mf2-deployment -n app-modernisation --replicas=1
+```
+
+# Add additional port forwarding for microservice
+```
+kubectl port-forward svc/microservice-service 8001:8001 --namespace app-modernisation
+```
+
+# Add KSQL Streams
+Start KSQL CLI:
+`ksql --config-file /root/ksql.client.properties https://ksqldb:8088 -u peter -p peter-secret`
+`SET 'auto.offset.reset' = 'earliest';`
+
+```
+CREATE STREAM postgres_bank_transactions WITH (KAFKA_TOPIC='postgres.bank.transactions', KEY_FORMAT ='AVRO', VALUE_FORMAT='AVRO');
+SELECT * FROM postgres_bank_transactions EMIT CHANGES;
+CREATE TABLE balances WITH (kafka_topic='balances') AS
+    SELECT card_number, SUM(transaction_amount) AS balance
+    FROM postgres_bank_transactions
+    GROUP BY card_number
+EMIT CHANGES;
+SELECT * FROM balances EMIT CHANGES;
+```
+
+# Create some new transactions
+```
+5188-6083-8011-0307:99
+5208-0272-3184-5035:140
+5208-0272-3184-5035:410
+5588-6461-5550-9705:118
+5588-6461-5550-9705:203
+```
+
+# Part 3: setup
+
+# Create a new stream for the transactions received from the "microservice"
+```
+CREATE STREAM express_bank_transactions (
+  `key` VARCHAR KEY,
+  `transaction_id` VARCHAR,
+  `card_number` VARCHAR,
+  `transaction_amount` INTEGER,
+  `transaction_time` VARCHAR)
+WITH (kafka_topic='express.bank.transactions', value_format='JSON');
+```
+
+# Create a new stream for sinking back to Postgres
+```
+CREATE STREAM jdbc_bank_transactions WITH (KAFKA_TOPIC='jdbc.bank.transactions', PARTITIONS=6, REPLICAS=3, VALUE_FORMAT='AVRO') AS
+  SELECT `key`, `transaction_id`,`card_number`, `transaction_amount`, `transaction_time`
+  FROM express_bank_transactions
+EMIT CHANGES;
+```
+
+# Create a sink connector
+Upload `postgres_sink.json`
+
+# Cutover to using only microservice
+```
+kubectl scale deployment webapp-mf2-deployment -n app-modernisation --replicas=0
+kubectl scale deployment webapp-mf3-deployment -n app-modernisation --replicas=1
+kubectl scale deployment monolith-deployment -n app-modernisation --replicas=0
+```
+
+# Check port-forwarding
+
+# Create some new transactions
+```
+5188-6083-8011-0307:2500
+5208-0272-3184-5035:80
+5588-6461-5550-9705:60
+5588-6461-5550-9705:900
+5588-6461-5550-9705:45
+```
+
+# Run queries in ksqlDB
+```
+SELECT * FROM express_bank_transactions EMIT CHANGES;
+SELECT * FROM jdbc_bank_transactions EMIT CHANGES;
+```
+
+# Show data in database
+
+# Demo Notification Service
+
+## Create the advanced streams
+
+- Copy ./streams/advanced-streams.ksql to the army knife and then run:
+
+`http --verify=false --auth=peter:peter-secret POST https://ksqldb:8088/ksql [email protected] --check-status`
+
+## Build and Push if there are any new changes to the app
+
+```
+cd apps/notification-service
+mvn clean install -Pdocker
+docker push lsdtrip/demo-notification-service
+```
+
+## Deploy
+
+`kc apply -f kube/demo-notification-service-deployment.yaml`
+
+## Clean-up
+
+Undeploy app:
+
+`kc delete -f kube/demo-notification-service-deployment.yaml`
+
+From ksql drop streams and tables:
+
+`ksql --config-file /root/ksql.client.properties https://ksqldb:8088 -u peter -p peter-secret`
+
+```
+drop stream jdbc_bank_transactions_enriched;
+drop stream jdbc_bank_transactions_rekeyed;
+drop table customers_accounts;
+drop table accounts;
+drop table customers;
+drop stream accounts_stream;
+drop stream customers_stream;
+```
+
+From army knife delete topics:
+
+```
+kafka-topics --delete --topic customers --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --command-config /root/etc/client.properties
+kafka-topics --delete --topic accounts --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --command-config /root/etc/client.properties
+kafka-topics --delete --topic customers_accounts --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --command-config /root/etc/client.properties
+kafka-topics --delete --topic jdbc_bank_transactions_rekeyed --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --command-config /root/etc/client.properties
+kafka-topics --delete --topic jdbc_bank_transactions_enriched --bootstrap-server $LSDMESP_BOOTSTRAP_SERVERS --command-config /root/etc/client.properties
+```
+
+```
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/customers-key
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/customers-value
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/accounts-key
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/accounts-value
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/customers_accounts-key
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/customers_accounts-value
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/jdbc_bank_transactions_rekeyed-key
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/jdbc_bank_transactions_rekeyed-value
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/jdbc_bank_transactions_enriched-key
+http --verify=false --auth=peter:peter-secret DELETE https://schemaregistry:8081/subjects/jdbc_bank_transactions_enriched-value
+```

charts/demos/apps/notification-service/Dockerfile

@@ -0,0 +1,25 @@
+FROM openjdk:11-jdk-slim
+
+# install required packages
+RUN apt-get -yq update
+RUN apt-get -yqq install wget
+RUN apt-get -yqq install curl
+
+WORKDIR /opt
+VOLUME /opt/etc
+
+# add app jar
+ARG JAR_FILE
+ADD target/${JAR_FILE} app.jar
+ADD target/lib lib
+ADD docker/container_entry.sh container_entry.sh
+
+# add user
+RUN useradd -d /home/kafka -m kafka -s /bin/bash -c "Kafka User"
+USER kafka
+
+# setup environment
+ENV JAVA_OPTS=""
+ENV ARGS=""
+
+ENTRYPOINT ["/opt/container_entry.sh"]

charts/demos/apps/notification-service/docker/container_entry.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+echo "Starting container with JAVA_OPTS: $JAVA_OPTS and ARGS: $ARGS"
+exec java $JAVA_OPTS -jar app.jar $ARGS
+echo "Exiting shell..."

charts/demos/apps/notification-service/kube/demo-notification-service-deployment.yaml

@@ -0,0 +1,83 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: demo-notification-service-client-properties
+stringData:
+  client.properties: |-
+    security.protocol=SASL_SSL
+    sasl.mechanism=PLAIN
+    sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="peter" password="peter-secret";
+    ssl.truststore.location=/home/kafka/lsdmesp.truststore.jks
+    ssl.truststore.password=112233
+    basic.auth.credentials.source=USER_INFO
+    basic.auth.user.info=peter:peter-secret
+    schema.registry.ssl.truststore.location=/home/kafka/lsdmesp.truststore.jks
+    schema.registry.ssl.truststore.password=112233
+  mail.properties: |-
+    smtp.host=smtp.sendgrid.net
+    smtp.port=587
+    smtp.username=apikey
+    smtp.password=<apipassword>
+    [email protected]
+    [email protected]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: demo-notification-service
+  labels:
+    app: demo-notification-service
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: demo-notification-service
+  template:
+    metadata:
+      labels:
+        app: demo-notification-service
+    spec:
+      containers:
+        - image: lsdtrip/demo-notification-service:latest
+          command:
+            - java
+            - -jar
+            - app.jar
+            - --bootstrap-servers
+            - kafka.lsdmesp-confluent.svc.cluster.local:9092
+            - --schema-registry-url
+            - https://schemaregistry.lsdmesp-confluent.svc.cluster.local:8081
+            - --source-topic
+            - jdbc_bank_transactions_enriched
+            - --command-config
+            - /opt/etc/client.properties
+            - --mail-config
+            - /opt/etc/mail.properties
+          imagePullPolicy: Always
+          name: demo-notification-service
+          lifecycle:
+            postStart:
+              exec:
+                command: [ 'sh', '-c', "keytool -keystore /home/kafka/lsdmesp.truststore.jks -alias CARoot -importcert -file /opt/certs/ca.pem -storepass 112233 -noprompt" ]
+          volumeMounts:
+            - mountPath: /opt/etc
+              name: demo-notification-service-client-properties
+            - mountPath: /opt/certs
+              name: ca-pair-sslcerts-tls-crt
+      volumes:
+        - secret:
+            defaultMode: 420
+            secretName: demo-notification-service-client-properties
+          name: demo-notification-service-client-properties
+        - secret:
+            defaultMode: 420
+            secretName: ca-pair-sslcerts
+            items:
+              - key: tls.crt
+                path: ca.pem
+          name: ca-pair-sslcerts-tls-crt