Merge branch 'develop' into cordaent-2586

docs/source/guides/fabric/add-new-org.md

@@ -28,16 +28,16 @@ Refer [this guide](../networkyaml-fabric.md) for details on editing the configur
 While modifying the configuration file(`network.yaml`) for adding new organization, all the existing organizations should have `org_status` tag as `existing` and the new organization should have `org_status` tag as `new` under `network.channels` e.g.
 
 ```yaml
---8<-- "platforms/hyperledger-fabric/configuration/samples/network-fabric-add-organization.yaml:65:139"
+--8<-- "platforms/hyperledger-fabric/configuration/samples/network-fabric-add-organization.yaml:62:151"
 ```
 
 and under `network.organizations` as
 
 ```yaml
---8<-- "platforms/hyperledger-fabric/configuration/samples/network-fabric-add-organization.yaml:144:155"
+--8<-- "platforms/hyperledger-fabric/configuration/samples/network-fabric-add-organization.yaml:153:168"
       ..
       ..
---8<-- "platforms/hyperledger-fabric/configuration/samples/network-fabric-add-organization.yaml:406:414"
+--8<-- "platforms/hyperledger-fabric/configuration/samples/network-fabric-add-organization.yaml:448:458"
       ..
       ..
 

platforms/hyperledger-fabric/charts/fabric-channel-join/templates/anchorpeer.yaml

@@ -3,7 +3,7 @@
 #
 #  SPDX-License-Identifier: Apache-2.0
 ##############################################################################################
-{{- if and (eq .Values.peer.type "anchor") (not .Values.peer.addPeerValue) }}
+{{- if and (eq .Values.peer.type "anchor") (not .Values.peer.addPeerValue) (not .Values.peer.addnewOrgValue) }}
 apiVersion: batch/v1
 kind: Job
 metadata:

platforms/hyperledger-fabric/charts/fabric-genesis/templates/job.yaml

@@ -233,6 +233,16 @@ spec:
                 fi
               }
   {{- end }} 
+
+              function writeNewOrg {
+                key=$1
+                file=$2
+
+                kubectl get configmap --namespace ${COMPONENT_NAME} ${key}
+                if [ $? -ne 0 ]; then
+                  kubectl create configmap --namespace ${COMPONENT_NAME} ${key} --from-file=${key}=${file}
+                fi
+              }
 
               OUTPUT_PATH_PEER="/templates/crypto-config/organizations"
             {{- range $org := $.Values.organizations }}
@@ -310,6 +320,13 @@ spec:
                   {{- end }}
                 fi
               {{- end }}
+              {{- if .Values.add_new_org }}
+                {{- range $org := $.Values.newOrgs }}
+                  export FABRIC_CFG_PATH=/templates
+                  configtxgen -printOrg {{ $org.name | lower }}MSP > {{ $org.name | lower }}_new.json
+                  writeNewOrg {{ $org.name | lower }}-new-data {{ $org.name | lower }}_new.json
+                {{- end }}
+              {{- end }}
           volumeMounts:
           - name: scripts-volume
             mountPath: /scripts/bevel-vault.sh

platforms/hyperledger-fabric/configuration/add-organization.yaml

@@ -26,85 +26,52 @@
         path: "./build"
         state: absent
 
-    # Create Namespaces and Vault-rbac kubernetes-value files for new organization
-    - include_role:
-        name: "create/namespace"
-      vars:
-        component_name: "{{ item.name | lower }}-net"
-        component_type_name: "{{ item.type | lower }}"
-        kubernetes: "{{ item.k8s }}"
-        release_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}"
-      loop: "{{ network['organizations'] }}"
-      when: item.org_status == 'new'
-
-    # Setup script for Vault and OS Package Manager
-    - name: "Setup script for Vault and OS Package Manager"
+    # Create Namespaces and Vault-rbac kubernetes-value files
+    - name: Create namespace and Vault auth for each organization
       include_role:
-        name: "{{ playbook_dir }}/../../shared/configuration/roles/setup/scripts"
+        name: "create/namespace"
       vars:
-        namespace: "{{ org.name | lower }}-net"
+        component_name: "{{ org.name | lower }}-net"
         kubernetes: "{{ org.k8s }}"
+        release_dir: "{{playbook_dir}}/../../../{{org.gitops.release_dir}}/{{ org.name | lower }}"
       loop: "{{ network['organizations'] }}"
       loop_control:
         loop_var: org
-      when: org.org_status == 'new'
 
-    # Setup Vault-Kubernetes accesses and Regcred for docker registry for new organization
-    - include_role: 
-        name: "{{ playbook_dir }}/../../shared/configuration/roles/setup/vault_kubernetes"
+    # Create necessary secrets
+    - name: "Create k8s secrets"
+      include_role:
+        name: create/secrets
       vars:
-        name: "{{ org.name | lower }}"
-        component_name: "{{ org.name | lower }}-vaultk8s-job"
-        component_type: "{{ org.type | lower }}"
         component_ns: "{{ org.name | lower }}-net"
-        component_auth: "{{ network.env.type }}{{ name }}"
         kubernetes: "{{ org.k8s }}"
         vault: "{{ org.vault }}"
-        gitops: "{{ org.gitops }}"
-        reset_path: "platforms/hyperledger-fabric/configuration"
-      loop: "{{ network['organizations'] }}"
-      loop_control:
-        loop_var: org
-      when: org.org_status == 'new'
-
-    # Create Storageclass for new organization
-    - include_role:
-        name: "{{ playbook_dir }}/../../../platforms/shared/configuration/roles/setup/storageclass"  
-      vars:
-        org_name: "{{ org.name | lower }}"
-        sc_name: "{{ org_name }}-bevel-storageclass"
-        region: "{{ org.k8s.region | default('eu-west-1') }}"
       loop: "{{ network['organizations'] }}"
       loop_control:
         loop_var: org
-      when: org.org_status == 'new'
+      when: 
+      - org.org_status is not defined or org.org_status == 'new'
 
-    # Create CA Server value files and check-in for new organization
-    - include_role:
+    # Create CA Server helm-value files and check-in
+    - name: Create CA server for each organization
+      include_role:
         name: "create/ca_server"
       vars:
-        component_name: "{{ item.name | lower}}-net"
-        component: "{{ item.name | lower}}"
-        component_type: "{{ item.type | lower}}"
-        component_services: "{{ item.services }}"
-        sc_name: "{{ component }}-bevel-storageclass"
-        kubernetes: "{{ item.k8s }}"
-        vault: "{{ item.vault }}"
-        ca: "{{ item.services.ca }}"
+        component_ns: "{{ org.name | lower}}-net"
+        component: "{{ org.name | lower}}"
+        component_services: "{{ org.services }}"
+        kubernetes: "{{ org.k8s }}"
+        vault: "{{ org.vault }}"
+        ca: "{{ org.services.ca }}"
         docker_url: "{{ network.docker.url }}"
-        gitops: "{{ item.gitops }}"
-        values_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}"
-      loop: "{{ network['organizations'] }}"
-      when: item.services.ca is defined and item.org_status == 'new'
-
-    # Create generate_crypto script for new organization
-    - include_role:
-        name: "create/crypto_script"
-      vars:
-        component_type: "{{ item.type | lower}}"
-        orderers: "{{ item.services.orderers }}"
+        gitops: "{{ org.gitops }}"
+        values_dir: "{{playbook_dir}}/../../../{{org.gitops.release_dir}}/{{ org.name | lower }}"
       loop: "{{ network['organizations'] }}"
-      when: item.org_status == 'new'
+      loop_control:
+        loop_var: org
+      when: 
+        - org.services.ca is defined
+        - org.org_status is not defined or org.org_status == 'new'
 
     #Creating a pause so that the client certificates are valid
     # You can continue if the CA-server pods are running for more than 5 minutes
@@ -122,64 +89,41 @@
       loop: "{{ network['channels'] }}"
       loop_control:
         loop_var: channel
-
-    # Create CA Tools value files and check-in for new organization
-    - include_role:
-        name: "create/ca_tools/peer"
-      vars:
-        component_name: "{{ item.name | lower}}-net"
-        component: "{{ item.name | lower}}"
-        component_type: "{{ item.type | lower}}"
-        component_services: "{{ item.services }}"
-        orderer_org: "{{ item.orderer_org | lower }}"
-        sc_name: "{{ component }}-bevel-storageclass"
-        kubernetes: "{{ item.k8s }}"
-        vault: "{{ item.vault }}"
-        ca: "{{ item.services.ca }}"
-        docker_url: "{{ network.docker.url }}"
-        gitops: "{{ item.gitops }}"
-        values_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}"
-      loop: "{{ network['organizations'] }}"
-      when: item.type == 'peer' and item.org_status == 'new'
-
-    # Creating channel artifacts and putting them in vault
-    # This role creates configtx.yaml file as the requirements mentioned in network.yaml
-    # which is then consumed by configtxgen tool
-    - include_role:
-        name: "create/configtx"
-      vars:
-        config_file: "./build/configtx.yaml"
-
-    # This role generate genesis block and channeltx
-    # Generates the config.json from the configtx.yaml for new organization
-    - include_role:
-        name: "create/channel_artifacts"
-      vars:
-        build_path: "./build"
-        genesis: "{{ item.genesis }}"
-        channel_name: "{{ item.channel_name | lower}}"
-        fetch_certs: "false"
-        profile_name: "{{ item.channel_name }}"
-      loop: "{{ network['channels'] }}"
 
     # This role creates the value file for peers of organisations and write couch db credentials
     # to the vault.
-    - include_role:
+    - name: Create all peers
+      include_role:
         name: "create/peers"
       vars:
         build_path: "./build"
-        namespace: "{{ item.name | lower}}-net"
-        component_type: "{{ item.type | lower}}"
-        component_services: "{{ item.services }}"
-        vault: "{{ item.vault }}"
-        git_protocol: "{{ item.gitops.git_protocol }}"
-        git_url: "{{ item.gitops.git_url }}"
-        git_branch: "{{ item.gitops.branch }}"
+        namespace: "{{ org.name | lower}}-net"
+        component_type: "{{ org.type | lower}}"
+        component_services: "{{ org.services }}"
+        kubernetes: "{{ org.k8s }}"
+        vault: "{{ org.vault }}"
+        git_protocol: "{{ org.gitops.git_protocol }}"
+        git_url: "{{ org.gitops.git_url }}"
+        git_branch: "{{ org.gitops.branch }}"
         docker_url: "{{ network.docker.url }}"
-        charts_dir: "{{ item.gitops.chart_source }}"
-        values_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}"
+        charts_dir: "{{ org.gitops.chart_source }}"
+        values_dir: "{{playbook_dir}}/../../../{{org.gitops.release_dir}}/{{ org.name | lower }}"
       loop: "{{ network['organizations'] }}"
-      when: item.type == 'peer' and item.org_status == 'new'
+      loop_control:
+        loop_var: org
+      when: 
+        - org.services.peers is defined and org.services.peers | length > 0
+        - org.org_status is not defined or org.org_status == 'new'
+
+    - name: "Create genesis block"
+      include_role: 
+        name: "create/genesis"
+      vars:
+        org: "{{ network['organizations'] | first }}"
+        docker_url: "{{ network.docker.url }}"
+        kubernetes: "{{ org.k8s  }}"
+        generateGenisis: false
+      when: genererate_configtx is defined and genererate_configtx == 'true'
 
     # This role is to start the existing peer cli
     # Fetch the configuration block and modify the block
@@ -207,23 +151,12 @@
         build_path: "./build"
         participants: "{{ item.participants }}"
         docker_url: "{{ network.docker.url }}"
-      loop: "{{ network['channels'] }}"  
-
-    # Create CLI pod for peers with cli option enabled
-    - name: Create CLI pod for each peer with it enabled
-      include_role:
-        name: "create/cli_pod"
-      vars:
-        peers: "{{ org.services.peers }}"
-        docker_url: "{{ network.docker.url }}"
-      loop: "{{ network.organizations }}"
-      loop_control:
-        loop_var: org
-      when: org.type == 'peer' and org.org_status == 'new'
+      loop: "{{ network['channels'] }}"
 
   vars: # These variables can be overriden from the command line
     privilege_escalate: false           # Default to NOT escalate to root privledges
     install_os: "linux"                 # Default to linux OS
     install_arch:  "amd64"              # Default to amd64 architecture
     bin_install_dir:  "~/bin"           # Default to /bin install directory for binaries
     add_new_org: "true"                 # Default to true
+    genererate_configtx: "true" 

platforms/hyperledger-fabric/configuration/roles/create/channels_join/tasks/nested_channel_join.yaml

@@ -79,6 +79,7 @@
     vault: "{{ org.vault }}"
     k8s: "{{ org.k8s }}"
     add_peer_value: "{{ add_peer | default('false') }}"
+    add_new_org_value: "{{ add_new_org | default('false') }}"
     values_dir: "{{playbook_dir}}/../../../{{org.gitops.component_dir}}/{{ org.name | lower }}"
   loop: "{{ participant.peers }}"
   loop_control:

platforms/hyperledger-fabric/configuration/roles/create/genesis/tasks/get_peer_msp_config.yaml

@@ -13,20 +13,6 @@
     path: "{{ playbook_dir }}/../../../{{ charts_dir }}/fabric-genesis/files"
     state: directory
 
-# Find old .json files
-- name: Find .json files
-  find:
-    paths: "{{ playbook_dir }}/../../../{{ charts_dir }}/fabric-genesis/files"
-    patterns: "*.json"
-  register: json_files
-
-# Delete old .json files
-- name: Delete .json files
-  file:
-    path: "{{ item.path }}"
-    state: absent
-  loop: "{{ json_files.files }}"
-
 - name: Save config peer msp config locally for genesis
   copy:
     content: "{{ config_file_data.resources[0] | to_nice_json }}"

platforms/hyperledger-fabric/configuration/roles/create/genesis/tasks/main.yaml

@@ -6,6 +6,19 @@
 
 ############################################################################################
 
+# Find old .json files
+- name: Find .json files
+  find:
+    paths: "{{ playbook_dir }}/../../../{{ org.gitops.chart_source }}/fabric-genesis/files"
+    patterns: "*.json"
+  register: json_files
+
+# Delete old .json files
+- name: Delete .json files
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ json_files.files }}"
 
 # Remove all Helm releases of organization except genesis
 - name: Delete Helm releases
@@ -20,7 +33,7 @@
 - name: Fetch peers cetificates
   include_tasks: get_certificates.yaml
   vars:
-    charts_dir: "{{ organization.gitops.chart_source }}"
+    charts_dir: "{{ org.gitops.chart_source }}"
     org_name: "{{ organization.name | lower }}"
     peers: "{{ organization.services.peers }}"
   loop: "{{ network['organizations'] }}"

platforms/hyperledger-fabric/configuration/roles/create/genesis/tasks/valuefile.yaml

@@ -13,6 +13,7 @@
     charts_dir: "{{ org.gitops.chart_source }}"
     values_dir: "{{playbook_dir}}/../../../{{org.gitops.component_dir}}/{{ org.name | lower }}"
     generateGenisisBLock: "{{ generateGenisis }}"
+    add_org: "{{ add_new_org | default(false) }}"
 
 # Git Push: Push the above generated files to git directory 
 - name: Git Push

platforms/hyperledger-fabric/configuration/roles/setup/config_block/fetch/tasks/get_new_org_data.yaml

@@ -0,0 +1,18 @@
+- name: Get new org data 
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ genesis_org_kubeconfig }}"
+    kind: ConfigMap
+    name: "{{ org_name }}-new-data"
+    namespace: "{{ genesis_org_namespace}}"
+  vars:
+    genesis_org: "{{ network['organizations'] | first }}"
+    genesis_org_namespace: "{{ genesis_org.name | lower }}-net"
+    genesis_org_kubeconfig: "{{ genesis_org.k8s.config_file }}"
+    genesis_org_context: "{{ genesis_org.k8s.context }}"
+  register: org_data
+
+- name: Save new org data file locally
+  shell: |
+     echo '{{ org_data.resources[0].data[org_name ~ "-new-data"]  }}' > {{ files_loc }}
+  vars:
+    files_loc: "{{ build_path }}/channel-artifacts/{{ channel_name | lower }}.json"

platforms/hyperledger-fabric/configuration/roles/setup/config_block/fetch/tasks/main.yaml

@@ -16,7 +16,7 @@
     channel_name: "{{ item.channel_name | lower }}"
     org_query: "organizations[?name=='{{participant.name}}']"
     org: "{{ network | json_query(org_query) | first }}"
-    peer: "{{ participant.peers | first }}"
+    peer: "{{ org.services.peers | first | default() }}"
   loop: "{{ participants }}"
   loop_control:
     loop_var: participant