diff --git a/automation/terraform/modules/kubernetes/testnet/locals.tf b/automation/terraform/modules/kubernetes/testnet/locals.tf index 974361e1bac..fad6cddfb25 100644 --- a/automation/terraform/modules/kubernetes/testnet/locals.tf +++ b/automation/terraform/modules/kubernetes/testnet/locals.tf @@ -8,25 +8,7 @@ provider "helm" { locals { mina_helm_repo = "https://coda-charts.storage.googleapis.com" - peers = var.additional_peers - - daemon = { - runtimeConfig = var.runtime_config - image = var.mina_image - useCustomEntrypoint = var.use_custom_entrypoint - customEntrypoint = var.custom_entrypoint - privkeyPass = var.block_producer_key_pass - seedPeers = local.peers - logLevel = var.log_level - logSnarkWorkGossip = var.log_snark_work_gossip - logPrecomputedBlocks = var.log_precomputed_blocks - logTxnPoolGossip = var.log_txn_pool_gossip - uploadBlocksToGCloud = var.upload_blocks_to_gcloud - seedPeersURL = var.seed_peers_url - exposeGraphql = var.expose_graphql - cpuRequest = var.cpu_request - memRequest= var.mem_request - } + # peers = var.additional_peers healthcheck_vars = { enabled = var.healthcheck_enabled @@ -38,13 +20,12 @@ locals { seed_vars = { testnetName = var.testnet_name mina = { - runtimeConfig = local.daemon.runtimeConfig + runtimeConfig = var.runtime_config image = var.mina_image useCustomEntrypoint = var.use_custom_entrypoint customEntrypoint = var.custom_entrypoint - privkeyPass = var.block_producer_key_pass // TODO: Change this to a better name - seedPeers = local.peers + seedPeers = var.additional_peers logLevel = var.log_level logSnarkWorkGossip = var.log_snark_work_gossip logTxnPoolGossip = var.log_txn_pool_gossip @@ -52,27 +33,47 @@ locals { client = "8301" graphql = "3085" metrics = "8081" - p2p = var.seed_port + p2p = var.seed_external_port } - seedPeersURL = var.seed_peers_url + # seedPeersURL = var.seed_peers_url uploadBlocksToGCloud = var.upload_blocks_to_gcloud exposeGraphql = var.expose_graphql } - healthcheck = local.healthcheck_vars - seedConfigs = [ for index, config in var.seed_configs : { name = config.name class = config.class libp2pSecret = config.libp2p_secret + libp2pSecretPassword = config.libp2p_secret_pw # privateKeySecret = config.private_key_secret - externalPort = config.external_port + # externalPort = config.external_port externalIp = config.external_ip enableArchive = config.enableArchive archiveAddress = config.archiveAddress } ] + + healthcheck = local.healthcheck_vars + + } + + daemon = { + runtimeConfig = var.runtime_config + image = var.mina_image + useCustomEntrypoint = var.use_custom_entrypoint + customEntrypoint = var.custom_entrypoint + # privkeyPass = var.block_producer_key_pass + seedPeers = var.additional_peers + logLevel = var.log_level + logSnarkWorkGossip = var.log_snark_work_gossip + logPrecomputedBlocks = var.log_precomputed_blocks + logTxnPoolGossip = var.log_txn_pool_gossip + uploadBlocksToGCloud = var.upload_blocks_to_gcloud + # seedPeersURL = var.seed_peers_url + exposeGraphql = var.expose_graphql + cpuRequest = var.cpu_request + memRequest= var.mem_request } block_producer_vars = { @@ -109,7 +110,10 @@ locals { runWithUserAgent = config.run_with_user_agent runWithBots = config.run_with_bots enableGossipFlooding = config.enable_gossip_flooding - privateKeySecret = config.private_key_secret + keypairName = config.keypair_name + # privateKey = config.private_key + # publicKey = config.private_key + privateKeyPW = config.privkey_password # libp2pSecret = config.libp2p_secret enablePeerExchange = config.enable_peer_exchange isolated = config.isolated @@ -125,9 +129,9 @@ locals { image = var.mina_image useCustomEntrypoint = var.use_custom_entrypoint customEntrypoint = var.custom_entrypoint - seedPeers = local.peers - runtimeConfig = local.daemon.runtimeConfig - seedPeersURL = var.seed_peers_url + seedPeers = var.additional_peers + runtimeConfig = var.runtime_config + # seedPeersURL = var.seed_peers_url } healthcheck = local.healthcheck_vars archive = item @@ -166,10 +170,12 @@ locals { mina = local.daemon healthcheck = local.healthcheck_vars - coordinatorName = "snark-coordinator-${lower(substr(snark.snark_worker_public_key,-6,-1))}" - workerName = "snark-worker-${lower(substr(snark.snark_worker_public_key,-6,-1))}" + # coordinatorName = "snark-coordinator-${lower(substr(snark.snark_worker_public_key,-6,-1))}" + coordinatorName = snark.snark_coordinator_name + # workerName = "snark-worker-${lower(substr(snark.snark_worker_public_key,-6,-1))}" + workerName = "${snark.snark_coordinator_name}-worker" workerReplicas = snark.snark_worker_replicas - coordinatorHostName = "snark-coordinator-${lower(substr(snark.snark_worker_public_key,-6,-1))}.${var.testnet_name}" + coordinatorHostName = "${snark.snark_coordinator_name}.${var.testnet_name}" coordinatorRpcPort = 8301 coordinatorHostPort = snark.snark_coordinators_host_port publicKey =snark.snark_worker_public_key @@ -204,6 +210,6 @@ locals { makeReportEveryMins = var.make_report_every_mins makeReportDiscordWebhookUrl = var.make_report_discord_webhook_url makeReportAccounts = var.make_report_accounts - seedPeersURL = var.seed_peers_url + seedPeersURL = var.additional_peers } } diff --git a/automation/terraform/modules/kubernetes/testnet/variables.tf b/automation/terraform/modules/kubernetes/testnet/variables.tf index a3cf60705e7..d09018cb65b 100644 --- a/automation/terraform/modules/kubernetes/testnet/variables.tf +++ b/automation/terraform/modules/kubernetes/testnet/variables.tf @@ -146,11 +146,6 @@ variable "mem_request" { # Seed Vars -variable "seed_port" { - type = string - default = "10001" -} - variable "seed_region" { type = string default = "us-west1" @@ -161,12 +156,34 @@ variable "seed_zone" { default = "us-west1-a" } -variable "seed_discovery_keypairs" { - type = list(any) - default = [ - "CAESQNf7ldToowe604aFXdZ76GqW/XVlDmnXmBT+otorvIekBmBaDWu/6ZwYkZzqfr+3IrEh6FLbHQ3VSmubV9I9Kpc=,CAESIAZgWg1rv+mcGJGc6n6/tyKxIehS2x0N1Uprm1fSPSqX,12D3KooWAFFq2yEQFFzhU5dt64AWqawRuomG9hL8rSmm5vxhAsgr", - "CAESQKtOnmYHQacRpNvBZDrGLFw/tVB7V4I14Y2xtGcp1sEsEyfcsNoFi7NnUX0T2lQDGQ31KvJRXJ+u/f9JQhJmLsI=,CAESIBMn3LDaBYuzZ1F9E9pUAxkN9SryUVyfrv3/SUISZi7C,12D3KooWB79AmjiywL1kMGeKHizFNQE9naThM2ooHgwFcUzt6Yt1" - ] +# variable "seed_discovery_keypairs" { +# type = list(any) +# default = [ +# "CAESQNf7ldToowe604aFXdZ76GqW/XVlDmnXmBT+otorvIekBmBaDWu/6ZwYkZzqfr+3IrEh6FLbHQ3VSmubV9I9Kpc=,CAESIAZgWg1rv+mcGJGc6n6/tyKxIehS2x0N1Uprm1fSPSqX,12D3KooWAFFq2yEQFFzhU5dt64AWqawRuomG9hL8rSmm5vxhAsgr", +# "CAESQKtOnmYHQacRpNvBZDrGLFw/tVB7V4I14Y2xtGcp1sEsEyfcsNoFi7NnUX0T2lQDGQ31KvJRXJ+u/f9JQhJmLsI=,CAESIBMn3LDaBYuzZ1F9E9pUAxkN9SryUVyfrv3/SUISZi7C,12D3KooWB79AmjiywL1kMGeKHizFNQE9naThM2ooHgwFcUzt6Yt1" +# ] +# } + +variable "seed_external_port" { + type = string + default = "10001" +} + +variable "seed_configs" { + type = list( + object({ + name = string, + class = string, + libp2p_secret = string, + libp2p_secret_pw = string + # external_port = number, + external_ip = string, + # private_key_secret = string, + enableArchive = bool, + archiveAddress = string + }) + ) + default = [] } # Block Producer Vars @@ -176,16 +193,19 @@ variable "log_level" { default = "Trace" } -variable "block_producer_key_pass" { - type = string -} +# variable "block_producer_key_pass" { +# type = string +# } variable "block_producer_configs" { type = list( object({ name = string, class = string, - private_key_secret = string, + keypair_name = string, + # private_key = string, + # public_key = string, + privkey_password = string, external_port = number, libp2p_secret = string, enable_gossip_flooding = bool, @@ -200,31 +220,22 @@ variable "block_producer_configs" { default = [] } -variable "seed_configs" { - type = list( - object({ - name = string, - class = string, - libp2p_secret = string, - external_port = number, - external_ip = string, - # private_key_secret = string, - enableArchive = bool, - archiveAddress = string - }) - ) - default = [] -} - variable "plain_node_configs" { default = null } # Snark Worker Vars - variable "snark_coordinators" { - type = list(any) + type = list( + object({ + + snark_coordinator_name = string, + snark_worker_replicas = number + snark_worker_fee = number + snark_worker_public_key = string + snark_coordinators_host_port = number + })) default = [] } @@ -337,7 +348,7 @@ variable "upload_blocks_to_gcloud" { default = false } -variable "seed_peers_url" { - type = string - default = "" -} +# variable "seed_peers_url" { +# type = string +# default = "" +# } diff --git a/automation/terraform/modules/o1-integration/graphql_ingress.tf b/automation/terraform/modules/o1-integration/graphql_ingress.tf index d1b9c45c258..4db56c2b7a3 100644 --- a/automation/terraform/modules/o1-integration/graphql_ingress.tf +++ b/automation/terraform/modules/o1-integration/graphql_ingress.tf @@ -19,13 +19,13 @@ resource "kubernetes_ingress" "testnet_graphql_ingress" { spec { rule { - host = "${local.graphql_ingress_dns}" + host = local.graphql_ingress_dns http { dynamic "path" { for_each = concat( [local.seed_config.name], [for config in var.block_producer_configs : config.name], - var.snark_worker_replicas > 0 ? [local.snark_coordinator_name] : [] + var.snark_coordinator_config != null ? [var.snark_coordinator_config.name] : [] ) content { diff --git a/automation/terraform/modules/o1-integration/inputs.tf b/automation/terraform/modules/o1-integration/inputs.tf index 5c0269f0dce..74a67069892 100644 --- a/automation/terraform/modules/o1-integration/inputs.tf +++ b/automation/terraform/modules/o1-integration/inputs.tf @@ -50,18 +50,23 @@ variable "runtime_config" { type = string } -variable "snark_worker_replicas" { - type = number -} - variable "snark_worker_fee" { type = string } -variable "snark_worker_public_key" { - type = string - default = "4vsRCVadXwWMSGA9q81reJRX3BZ5ZKRtgZU7PtGsNq11w2V9tUNf4urZAGncZLUiP4SfWqur7AZsyhJKD41Ke7rJJ8yDibL41ePBeATLUnwNtMTojPDeiBfvTfgHzbAVFktD65vzxMNCvvAJ" +# variable "snark_worker_public_key" { +# type = string +# default = "4vsRCVadXwWMSGA9q81reJRX3BZ5ZKRtgZU7PtGsNq11w2V9tUNf4urZAGncZLUiP4SfWqur7AZsyhJKD41Ke7rJJ8yDibL41ePBeATLUnwNtMTojPDeiBfvTfgHzbAVFktD65vzxMNCvvAJ" +# } +variable "snark_coordinator_config" { + description = "configurations for the snark coordinator and its workers" + type = object({ + name = string, + public_key = string, + worker_nodes = number + }) + default = null } variable "log_precomputed_blocks" { @@ -110,10 +115,12 @@ variable "block_producer_configs" { type = list( object({ name = string, - id = string, - public_key = string, - private_key = string, - keypair_secret = string, + keypair = object({ + keypair_name = string + public_key = string + private_key = string, + privkey_password = string + }), libp2p_secret = string }) ) diff --git a/automation/terraform/modules/o1-integration/locals.tf b/automation/terraform/modules/o1-integration/locals.tf index 5b9bdce732d..47d2525087b 100644 --- a/automation/terraform/modules/o1-integration/locals.tf +++ b/automation/terraform/modules/o1-integration/locals.tf @@ -3,27 +3,29 @@ locals { snark_worker_host_port = 10001 block_producer_starting_host_port = 10010 - seed_peer = { - multiaddr = "/dns4/seed.${var.testnet_name}/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf", - peerid = "2D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf", - private_key_secretbox = "{\"box_primitive\":\"xsalsa20poly1305\",\"pw_primitive\":\"argon2i\",\"nonce\":\"7YSQmeRbo4fGd2nYh9aS4WzNWzSXsFmmkTqKy93\",\"pwsalt\":\"9QsEJdSkFbF8PUwLPo2ZLHpgT7ja\",\"pwdiff\":[134217728,6],\"ciphertext\":\"7o8WU4cBiuUqGPaF2fNA815XqhZM5j95dhns5zztCiSehb3xVzTUSbCj1nDXG5rAteu67pvMnaGbQ57cQw1HEPB2DDdrtAAWbCt7qYjmP6cNm2L7H9DC8NKHs1LYuWvthfjDvxBDHnVidpRCyqtMBg9TPWtMPkZy1UCVRFokAA5HaA2xkh4WFgy2SCVrAeWNP5BeUGq9u779KcM9BAtg9n6rqbKDTybX4h1aSZ9qA72tg1LHzENfHLAgzJXZQcxhjvw6b8H51m9rVhwFTKPCRRCNXfcyQqjEpH4fm\"}", - secret = "seed-discovery-keys", - port = 10401 - } + # seed_peer = { + # multiaddr = "/dns4/seed.${var.testnet_name}/tcp/10401/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf", + # peerid = "12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf", + # private_key_secretbox = "{\"box_primitive\":\"xsalsa20poly1305\",\"pw_primitive\":\"argon2i\",\"nonce\":\"7YSQmeRbo4fGd2nYh9aS4WzNWzSXsFmmkTqKy93\",\"pwsalt\":\"9QsEJdSkFbF8PUwLPo2ZLHpgT7ja\",\"pwdiff\":[134217728,6],\"ciphertext\":\"7o8WU4cBiuUqGPaF2fNA815XqhZM5j95dhns5zztCiSehb3xVzTUSbCj1nDXG5rAteu67pvMnaGbQ57cQw1HEPB2DDdrtAAWbCt7qYjmP6cNm2L7H9DC8NKHs1LYuWvthfjDvxBDHnVidpRCyqtMBg9TPWtMPkZy1UCVRFokAA5HaA2xkh4WFgy2SCVrAeWNP5BeUGq9u779KcM9BAtg9n6rqbKDTybX4h1aSZ9qA72tg1LHzENfHLAgzJXZQcxhjvw6b8H51m9rVhwFTKPCRRCNXfcyQqjEpH4fm\"}", + # secret = "seed-discovery-keys", + # port = 10401 + # } + + seed_external_port = 10001 seed_config = { name = "seed", class = "seed", - libp2p_secret = local.seed_peer.secret, - external_port = 10401, - node_port = null, + libp2p_secret = "seed-discovery-keys", + libp2p_secret_pw = "naughty blue worm" external_ip = null, - private_key_secret = null, + # private_key_secret = null, enableArchive = false, archiveAddress = null } + - snark_coordinator_name = "snark-coordinator-${lower(substr(var.snark_worker_public_key, -6, -1))}" + #snark_coordinator_name = "snark-coordinator-${lower(substr(var.snark_worker_public_key, -6, -1))}" default_archive_node = { image = var.mina_archive_image diff --git a/automation/terraform/modules/o1-integration/secrets.tf b/automation/terraform/modules/o1-integration/secrets.tf index 629c41acb7f..d8e8dfdb1e0 100644 --- a/automation/terraform/modules/o1-integration/secrets.tf +++ b/automation/terraform/modules/o1-integration/secrets.tf @@ -3,26 +3,27 @@ resource "kubernetes_secret" "mina_account_keypairs" { for_each = {for config in var.block_producer_configs : config.name => config} metadata { - name = each.value.keypair_secret + name = each.value.keypair.keypair_name namespace = var.testnet_name } data = { - pub = each.value.public_key - key = each.value.private_key + pub = each.value.keypair.public_key + key = each.value.keypair.private_key } } +#this is entire for the seed peer resource "kubernetes_secret" "libp2p_discovery_keys" { depends_on = [module.kubernetes_testnet.testnet_namespace] metadata { - name = local.seed_peer.secret + name = local.seed_config.libp2p_secret namespace = var.testnet_name } data = { - key = local.seed_peer.private_key_secretbox, - pub = local.seed_peer.peerid + key = "{\"box_primitive\":\"xsalsa20poly1305\",\"pw_primitive\":\"argon2i\",\"nonce\":\"7YSQmeRbo4fGd2nYh9aS4WzNWzSXsFmmkTqKy93\",\"pwsalt\":\"9QsEJdSkFbF8PUwLPo2ZLHpgT7ja\",\"pwdiff\":[134217728,6],\"ciphertext\":\"7o8WU4cBiuUqGPaF2fNA815XqhZM5j95dhns5zztCiSehb3xVzTUSbCj1nDXG5rAteu67pvMnaGbQ57cQw1HEPB2DDdrtAAWbCt7qYjmP6cNm2L7H9DC8NKHs1LYuWvthfjDvxBDHnVidpRCyqtMBg9TPWtMPkZy1UCVRFokAA5HaA2xkh4WFgy2SCVrAeWNP5BeUGq9u779KcM9BAtg9n6rqbKDTybX4h1aSZ9qA72tg1LHzENfHLAgzJXZQcxhjvw6b8H51m9rVhwFTKPCRRCNXfcyQqjEpH4fm\"}" + pub = "12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf" #this is also the peer-id } } diff --git a/automation/terraform/modules/o1-integration/testnet.tf b/automation/terraform/modules/o1-integration/testnet.tf index 69bf2ec181e..6fcb9931867 100644 --- a/automation/terraform/modules/o1-integration/testnet.tf +++ b/automation/terraform/modules/o1-integration/testnet.tf @@ -23,11 +23,13 @@ module "kubernetes_testnet" { log_level = "Trace" log_snark_work_gossip = true - additional_peers = [local.seed_peer.multiaddr] + #make sure everyone has the seed peer's multiaddress + additional_peers = ["/dns4/seed.${var.testnet_name}/tcp/${local.seed_external_port}/p2p/12D3KooWCoGWacXE4FRwAX8VqhnWVKhz5TTEecWEuGmiNrDt2XLf"] runtime_config = var.runtime_config seed_zone = "us-west1-a" seed_region = "us-west1" + seed_external_port = local.seed_external_port seed_configs = [local.seed_config] archive_configs = local.archive_node_configs @@ -37,23 +39,27 @@ module "kubernetes_testnet" { archive_node_count = var.archive_node_count - snark_coordinators = var.snark_worker_replicas <= 0 ? [] : [ + snark_coordinators = var.snark_coordinator_config == null ? [] :[ { - snark_worker_replicas = var.snark_worker_replicas + snark_coordinator_name = var.snark_coordinator_config.name + snark_worker_replicas = var.snark_coordinator_config.worker_nodes snark_worker_fee = var.snark_worker_fee - snark_worker_public_key = var.snark_worker_public_key + snark_worker_public_key = var.snark_coordinator_config.public_key snark_coordinators_host_port = local.snark_worker_host_port } ] - block_producer_key_pass = "naughty blue worm" + # block_producer_key_pass = "naughty blue worm" block_producer_configs = [ for index, config in var.block_producer_configs : { name = config.name - id = config.id + # id = config.id class = "test", external_port = local.block_producer_starting_host_port + index - private_key_secret = config.keypair_secret + keypair_name = config.keypair.keypair_name + # private_key = config.keypair.private_key + # public_key = config.keypair.public_key + privkey_password = config.keypair.privkey_password libp2p_secret = config.libp2p_secret isolated = false enable_gossip_flooding = false diff --git a/automation/terraform/modules/o1-testnet/inputs.tf b/automation/terraform/modules/o1-testnet/inputs.tf index ddaedf27c1e..0ad90a58203 100644 --- a/automation/terraform/modules/o1-testnet/inputs.tf +++ b/automation/terraform/modules/o1-testnet/inputs.tf @@ -176,7 +176,15 @@ variable "seed_starting_host_port" { variable "snark_coordinators" { description = "configurations for not just the snark coordinators but also the snark workers they coordinate" - type = list(any) + type = list( + object({ + + snark_coordinator_name = string, + snark_worker_replicas = number + snark_worker_fee = number + snark_worker_public_key = string + snark_coordinators_host_port = number + })) default = [] } diff --git a/automation/terraform/modules/o1-testnet/locals.tf b/automation/terraform/modules/o1-testnet/locals.tf index e9bb0a70ad7..66521e2b01c 100644 --- a/automation/terraform/modules/o1-testnet/locals.tf +++ b/automation/terraform/modules/o1-testnet/locals.tf @@ -60,8 +60,9 @@ locals { seed_static_peers = [ for index, name in keys(data.local_file.libp2p_seed_peers) : { - full_peer = "/dns4/${name}.${var.testnet_name}/tcp/${var.seed_starting_host_port + index}/p2p/${trimspace(data.local_file.libp2p_seed_peers[name].content)}", - port = var.seed_starting_host_port + index + # i don't think the seeds need to have different ports + full_peer = "/dns4/${name}.${var.testnet_name}/tcp/${var.seed_external_port}/p2p/${trimspace(data.local_file.libp2p_seed_peers[name].content)}", + # port = var.seed_starting_host_port + index name = local.seed_names[index] unique_node_index= -1 total_node_index= -1 diff --git a/automation/terraform/modules/o1-testnet/testnet.tf b/automation/terraform/modules/o1-testnet/testnet.tf index 78c7f7bebb5..f8aa0c7938c 100644 --- a/automation/terraform/modules/o1-testnet/testnet.tf +++ b/automation/terraform/modules/o1-testnet/testnet.tf @@ -66,9 +66,10 @@ module "kubernetes_testnet" { { name = bp.name class = bp.class - id = bp.total_node_index + # id = bp.total_node_index + keypair_name = "${bp.class}-${bp.unique_node_index}-key" + privkey_password = "naughty blue worm" external_port = bp.port - private_key_secret = "${bp.class}-${bp.unique_node_index}-key" libp2p_secret = "" enable_gossip_flooding = false run_with_user_agent = bp.class =="whale" ? false : ( var.nodes_with_user_agent == [] ? true : contains(var.nodes_with_user_agent, bp.name) ) @@ -81,15 +82,18 @@ module "kubernetes_testnet" { } ] + seed_external_port = 10001 + seed_configs = [ for i in range(var.seed_count) : { name = local.seed_static_peers[i].name class = "seed" - id = i + 1 - external_port = local.seed_static_peers[i].port + # id = i + 1 + # external_port = local.seed_static_peers[i].port + libp2p_secret = "seed-${i + 1}-key" + libp2p_secret_pw = "naughty blue worm" external_ip = google_compute_address.seed_static_ip[i].address # private_key_secret = "online-seeds-account-${i + 1}-key" - libp2p_secret = "seed-${i + 1}-key" enableArchive = length(local.archive_node_configs) > 0 archiveAddress = length(local.archive_node_configs) > 0 ? "${element(local.archive_node_configs, i)["name"]}:${element(local.archive_node_configs, i)["serverPort"]}" : "" } diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 8f425bd6ff4..5d1595dce51 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -32,6 +32,7 @@ in Pipeline.build Pipeline.Config::{ TestExecutive.execute "gossip-consis" dependsOn, TestExecutive.execute "opt-block-prod" dependsOn, TestExecutive.execute "medium-bootstrap" dependsOn, + TestExecutive.execute "snark" dependsOn, TestExecutive.execute "archive-node" dependsOn ] diff --git a/helm/block-producer/templates/block-producer.yaml b/helm/block-producer/templates/block-producer.yaml index c8ab35ea335..34fb2cb51eb 100644 --- a/helm/block-producer/templates/block-producer.yaml +++ b/helm/block-producer/templates/block-producer.yaml @@ -60,7 +60,7 @@ spec: mountPath: /root/.mina-config env: - name: MINA_PRIVKEY_PASS - value: {{ $.Values.mina.privkeyPass | quote }} + value: {{ $config.privateKeyPW | quote }} {{ if $config.libp2pSecret -}} - name: libp2p-perms image: {{ $.Values.mina.image | quote }} @@ -112,10 +112,10 @@ spec: - name: CODA_PUBLIC_KEY valueFrom: secretKeyRef: - name: {{ $config.privateKeySecret }} + name: {{ $config.keypairName }} key: pub - name: MINA_PRIVKEY_PASS - value: {{ $.Values.mina.privkeyPass | quote }} + value: {{ $config.privateKeyPW | quote }} - name: PYTHONUNBUFFERED value: "1" ports: @@ -142,14 +142,14 @@ spec: name: echo-service-key key: pub - name: ECHO_PASSWORD - value: {{ $.Values.mina.privkeyPass | quote }} + value: {{ $config.privateKeyPW | quote }} - name: FAUCET_PUBLICKEY valueFrom: secretKeyRef: name: faucet-service-key key: pub - name: FAUCET_PASSWORD - value: {{ $.Values.mina.privkeyPass | quote }} + value: {{ $config.privateKeyPW | quote }} - name: DISCORD_API_KEY valueFrom: secretKeyRef: @@ -243,10 +243,10 @@ spec: - name: GCLOUD_BLOCK_UPLOAD_BUCKET value: "mina_network_block_data" - name: MINA_PRIVKEY_PASS - value: {{ $.Values.mina.privkeyPass | quote }} + value: {{ $config.privateKeyPW | quote }} {{- if $config.libp2pSecret }} - name: MINA_LIBP2P_PASS - value: {{ $.Values.mina.privkeyPass | quote }} + value: {{ $config.privateKeyPW | quote }} {{- end }} - name: MINA_CLIENT_TRUSTLIST value: "10.0.0.0/8" @@ -288,7 +288,7 @@ spec: volumes: - name: private-keys secret: - secretName: {{ $config.privateKeySecret }} + secretName: {{ $config.keypairName }} defaultMode: 256 items: - key: key diff --git a/helm/block-producer/values.yaml b/helm/block-producer/values.yaml index 1dec0fa0ac4..9f1f61b7a82 100644 --- a/helm/block-producer/values.yaml +++ b/helm/block-producer/values.yaml @@ -10,7 +10,7 @@ mina: image: gcr.io/o1labs-192920/mina-daemon:1.2.0beta8-5b35b27-devnet useCustomEntrypoint: false customEntrypoint: "" - privkeyPass: "naughty blue worm" + # privkeyPass: "naughty blue worm" seedPeers: - /dns4/seed-one.genesis-redux.o1test.net/tcp/10002/p2p/12D3KooWP7fTKbyiUcYJGajQDpCFo2rDexgTHFJTxCH8jvcL1eAH - /dns4/seed-two.genesis-redux.o1test.net/tcp/10002/p2p/12D3KooWL9ywbiXNfMBqnUKHSB1Q1BaHFNUzppu6JLMVn9TTPFSA @@ -47,7 +47,11 @@ blockProducerConfigs: - name: "test-1" runWithUserAgent: false runWithBots: false - privateKeySecret: "yolo" + # privateKeySecret: "yolo" + keypairName: "" + # privateKey : "" + # publicKey : "" + privateKeyPW : "" isolated: false enableGossipFlooding: false enablePeerExchange: false @@ -57,7 +61,11 @@ blockProducerConfigs: - name: "test-2" runWithUserAgent: true runWithBots: false - privateKeySecret: "yolo" + # privateKeySecret: "yolo" + keypairName: "" + # privateKey : "" + # publicKey : "" + privateKeyPW : "" isolated: false enableGossipFlooding: false enablePeerExchange: false @@ -67,7 +75,11 @@ blockProducerConfigs: - name: "test-3" runWithUserAgent: false runWithBots: true - privateKeySecret: "yolo" + # privateKeySecret: "yolo" + keypairName: "" + # privateKey : "" + # publicKey : "" + privateKeyPW : "" isolated: false enableGossipFlooding: false enablePeerExchange: false @@ -77,7 +89,11 @@ blockProducerConfigs: - name: "test-4" runWithUserAgent: true runWithBots: true - privateKeySecret: "yolo" + # privateKeySecret: "yolo" + keypairName: "" + # privateKey : "" + # publicKey : "" + privateKeyPW : "" isolated: false enableGossipFlooding: false enablePeerExchange: false diff --git a/helm/seed-node/templates/seed-node.yaml b/helm/seed-node/templates/seed-node.yaml index 6d05b74451c..ca6482f9bf2 100644 --- a/helm/seed-node/templates/seed-node.yaml +++ b/helm/seed-node/templates/seed-node.yaml @@ -110,10 +110,10 @@ spec: - name: GCLOUD_BLOCK_UPLOAD_BUCKET value: "mina_network_block_data" - name: DAEMON_EXTERNAL_PORT - value: {{ default $.Values.mina.ports.p2p $config.externalPort | quote }} + value: {{ $.Values.mina.ports.p2p | quote }} {{- if $config.libp2pSecret }} - name: MINA_LIBP2P_PASS - value: {{ $.Values.mina.privkeyPass | quote }} + value: {{ $config.libp2pSecretPassword | quote }} {{- end }} ports: - name: client-port @@ -127,8 +127,8 @@ spec: containerPort: {{ $.Values.mina.ports.metrics }} - name: external-port protocol: TCP - containerPort: {{ default $.Values.mina.ports.p2p $config.externalPort }} - hostPort: {{ default $.Values.mina.ports.external $config.externalPort }} + containerPort: {{ $.Values.mina.ports.p2p }} + hostPort: {{ $.Values.mina.ports.p2p }} {{$data := dict "name" $config.name "healthcheck" $.Values.healthcheck }} {{- include "healthcheck.seed.allChecks" $data | indent 8 }} imagePullPolicy: Always @@ -205,7 +205,7 @@ spec: app: {{ $config.name }} ports: - name: tcp-p2p - port: {{ default $.Values.mina.ports.p2p $config.externalPort }} + port: {{ $.Values.mina.ports.p2p }} targetPort: external-port --- {{- if $.Values.mina.exposeGraphql }} diff --git a/helm/seed-node/templates/seed-service.yaml b/helm/seed-node/templates/seed-service.yaml index 58ec7f7aeb2..e24f9bf400c 100644 --- a/helm/seed-node/templates/seed-service.yaml +++ b/helm/seed-node/templates/seed-service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ tpl .Values.seed.fullname . }} + name: "seed-service" spec: type: ClusterIP # We want all pods to have their addresses published for the sake of the @@ -10,9 +10,9 @@ spec: # have to be able to talk to each other in order to become ready. publishNotReadyAddresses: true selector: - app: {{ tpl .Values.seed.fullname . }} + app: "seed-service" ports: - name: tcp-p2p - port: {{ .Values.mina.ports.p2p }} + port: {{ $.Values.mina.ports.p2p }} targetPort: external-port ---- +--- \ No newline at end of file diff --git a/helm/seed-node/values.yaml b/helm/seed-node/values.yaml index c75a4f0db32..1032b738e3f 100644 --- a/helm/seed-node/values.yaml +++ b/helm/seed-node/values.yaml @@ -14,25 +14,26 @@ mina: ports: client: "8301" graphql: "3085" - metrics: "10001" - p2p: "10909" + metrics: "8081" + p2p: "10001" exposeGraphql: false cpuRequest: 6 memRequest: "12.0Gi" -seed: - active: true - fullname: "seed-node" - hostPort: 10001 - rpcPort: 8301 - discoveryKeypair: "CAESQNf7ldToowe604aFXdZ76GqW/XVlDmnXmBT+otorvIekBmBaDWu/6ZwYkZzqfr+3IrEh6FLbHQ3VSmubV9I9Kpc=,CAESIAZgWg1rv+mcGJGc6n6/tyKxIehS2x0N1Uprm1fSPSqX,12D3KooWAFFq2yEQFFzhU5dt64AWqawRuomG9hL8rSmm5vxhAsgr" +# seed: +# active: true +# fullname: "seed-node" +# hostPort: 10001 +# rpcPort: 8301 +# discoveryKeypair: "CAESQNf7ldToowe604aFXdZ76GqW/XVlDmnXmBT+otorvIekBmBaDWu/6ZwYkZzqfr+3IrEh6FLbHQ3VSmubV9I9Kpc=,CAESIAZgWg1rv+mcGJGc6n6/tyKxIehS2x0N1Uprm1fSPSqX,12D3KooWAFFq2yEQFFzhU5dt64AWqawRuomG9hL8rSmm5vxhAsgr" seedConfigs: - name: test-1 class: "seed" - external_port: 1111 - external_ip: 1.1.1.1 - node_port: 1111 + libp2pSecret: "seed-discovery-keys" + libp2pSecretPassword: "naughty blue worm" + # externalPort: 10001 + externalIp: 1.1.1.1 enableArchive: false archiveAddress: archive-1:3086 diff --git a/src/app/test_executive/archive_node_test.ml b/src/app/test_executive/archive_node_test.ml index 0cac7629948..9cee3488f3b 100644 --- a/src/app/test_executive/archive_node_test.ml +++ b/src/app/test_executive/archive_node_test.ml @@ -17,15 +17,20 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in { default with - requires_graphql = true (* a few block producers, where few = 4 *) + requires_graphql = true + ; genesis_ledger = + [ { account_name = "node-a-key"; balance = "4000"; timing = Untimed } + ; { account_name = "node-b-key"; balance = "9000"; timing = Untimed } + ; { account_name = "node-c-key"; balance = "8000"; timing = Untimed } + ; { account_name = "node-d-key"; balance = "17000"; timing = Untimed } + ] ; block_producers = - [ { balance = "4000"; timing = Untimed } - ; { balance = "9000"; timing = Untimed } - ; { balance = "8000"; timing = Untimed } - ; { balance = "17000"; timing = Untimed } + [ { node_name = "node-a"; account_name = "node-a-key" } + ; { node_name = "node-b"; account_name = "node-b-key" } + ; { node_name = "node-c"; account_name = "node-c-key" } + ; { node_name = "node-d"; account_name = "node-d-key" } ] ; num_archive_nodes = 1 - ; num_snark_workers = 0 ; log_precomputed_blocks = true } @@ -35,11 +40,16 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let run network t = let open Malleable_error.Let_syntax in let logger = Logger.create () in - let archive_node = List.hd_exn @@ Network.archive_nodes network in + let archive_node = + List.hd_exn @@ Core.String.Map.data (Network.archive_nodes network) + in let all_nodes = Network.all_nodes network in (* waiting for archive_node does not seem to work *) [%log info] "archive node test: waiting for block producers to initialize" ; - let%bind () = wait_for t (Wait_condition.nodes_to_initialize all_nodes) in + let%bind () = + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) + in [%log info] "archive node test: waiting for archive node to initialize" ; let%bind () = wait_for t (Wait_condition.node_to_initialize archive_node) in [%log info] "archive node test: running network for %0.1f minutes" @@ -52,7 +62,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct in [%log info] "archive node test: collecting block logs" ; let%map () = - Malleable_error.List.iter all_nodes ~f:(fun bp -> + Malleable_error.List.iter (Core.String.Map.data all_nodes) ~f:(fun bp -> Network.Node.dump_precomputed_blocks ~logger bp ) in [%log info] "archive node test: succesfully completed" diff --git a/src/app/test_executive/block_production_priority.ml b/src/app/test_executive/block_production_priority.ml index 54b49493b16..648bd85e622 100644 --- a/src/app/test_executive/block_production_priority.ml +++ b/src/app/test_executive/block_production_priority.ml @@ -19,13 +19,34 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with requires_graphql = true + ; genesis_ledger = + [ { Test_Account.account_name = "receiver-key" + ; balance = "9999999" + ; timing = Untimed + } + ; { account_name = "empty-bp-key"; balance = "0"; timing = Untimed } + ; { account_name = "snark-node-key"; balance = "0"; timing = Untimed } + ] + @ List.init 1000 ~f:(fun i -> + let i_str = Int.to_string i in + { Test_Account.account_name = + String.concat [ "sender-account"; i_str ] + ; balance = "10000" + ; timing = Untimed + } ) ; block_producers = - { Wallet.balance = "9999999"; timing = Untimed } - :: List.init 4 ~f:(const { Wallet.balance = "0"; timing = Untimed }) - ; num_snark_workers = 25 - ; extra_genesis_accounts = - [ { balance = "1000"; timing = Untimed } ] - (* ; aux_account_balance = Some "1000" *) + [ { node_name = "receiver"; account_name = "receiver-key" } + ; { node_name = "empty_node-1"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-2"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-3"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-4"; account_name = "empty-bp-key" } + ] + ; snark_coordinator = + Some + { node_name = "snark-node" + ; account_name = "snark-node-key" + ; worker_nodes = 25 + } ; txpool_max_size = 10_000_000 ; snark_worker_fee = "0.0001" } @@ -43,34 +64,44 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let run network t = let open Malleable_error.Let_syntax in let logger = Logger.create () in - let%bind receiver, senders = - match Network.block_producers network with - | [] -> - Malleable_error.hard_error_string "no block producers" - | [ r ] -> - (* Sender and receiver are the same node *) - return (r, [ r ]) - | r :: rs -> - return (r, rs) + let receiver = + Core.String.Map.find_exn (Network.block_producers network) "receiver" in let%bind receiver_pub_key = pub_key_of_node receiver in + let empty_bps = + Core.String.Map.remove (Network.block_producers network) "receiver" + |> Core.String.Map.data + in + let rec map_remove_keys map ~(keys : string list) = + match keys with + | [] -> + map + | hd :: tl -> + map_remove_keys (Core.String.Map.remove map hd) ~keys:tl + in + let sender_kps = + map_remove_keys + (Network.genesis_keypairs network) + ~keys:[ "receiver-key"; "empty-bp-key"; "snark-node-key" ] + |> Core.String.Map.data + in + let sender_priv_keys = + List.map sender_kps ~f:(fun kp -> kp.keypair.private_key) + in let pk_to_string = Signature_lib.Public_key.Compressed.to_base58_check in [%log info] "receiver: %s" (pk_to_string receiver_pub_key) ; let%bind () = - Malleable_error.List.iter senders ~f:(fun s -> - let%map pk = pub_key_of_node s in - [%log info] "sender: %s" (pk_to_string pk) ) + Malleable_error.List.iter sender_kps ~f:(fun s -> + let pk = s.keypair.public_key |> Signature_lib.Public_key.compress in + return ([%log info] "sender: %s" (pk_to_string pk)) ) in let window_ms = (Network.constraint_constants network).block_window_duration_ms in + let all_nodes = Network.all_nodes network in let%bind () = - section_hard "wait for nodes to initialize" - (Malleable_error.List.iter - ( Network.seeds network - @ Network.block_producers network - @ Network.snark_coordinators network ) - ~f:(Fn.compose (wait_for t) Wait_condition.node_to_initialize) ) + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) in let%bind () = section_hard "wait for 3 blocks to be produced (warm-up)" @@ -82,26 +113,16 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct in let%bind () = section_hard "spawn transaction sending" - (let num_senders = List.length senders in - let sender_keys = - List.map ~f:snd - @@ List.drop - ( Array.to_list @@ Lazy.force - @@ Mina_base.Sample_keypairs.keypairs ) - (num_senders + 2) - in - let num_sender_keys = List.length sender_keys in - let keys_per_sender = num_sender_keys / num_senders in - let%bind () = - Malleable_error.ok_if_true ~error_type:`Hard - ~error:(Error.of_string "not enough sender keys") - (keys_per_sender > 0) - in - let num_payments = num_slots * window_ms / tx_delay_ms in + (let num_payments = num_slots * window_ms / tx_delay_ms in let repeat_count = Unsigned.UInt32.of_int num_payments in let repeat_delay_ms = Unsigned.UInt32.of_int tx_delay_ms in - [%log info] "will now send %d payments" num_payments ; - Malleable_error.List.fold ~init:sender_keys senders + let num_sender_keys = List.length sender_priv_keys in + let keys_per_sender = num_sender_keys / List.length empty_bps in + [%log info] + "will now send %d payments from as many accounts. %d nodes will \ + send %d payments each from distinct keys" + num_payments (List.length empty_bps) keys_per_sender ; + Malleable_error.List.fold ~init:sender_priv_keys empty_bps ~f:(fun keys node -> let keys0, rest = List.split_n keys keys_per_sender in Network.Node.must_send_test_payments ~repeat_count ~repeat_delay_ms @@ -171,9 +192,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ok_if_true "block production was delayed" (rcv_delay_rest <= 2) ) in section "retrieve metrics of tx sender nodes" - (* We omit the result because we just want to query senders to see some useful + (* We omit the result because we just want to query the txn sending nodes to see some useful output in test logs *) - (Malleable_error.List.iter senders + (Malleable_error.List.iter empty_bps ~f: (Fn.compose Malleable_error.soften_error (Fn.compose Malleable_error.ignore_m get_metrics) ) ) diff --git a/src/app/test_executive/chain_reliability_test.ml b/src/app/test_executive/chain_reliability_test.ml index 4109b45110d..3711ff5ec14 100644 --- a/src/app/test_executive/chain_reliability_test.ml +++ b/src/app/test_executive/chain_reliability_test.ml @@ -17,15 +17,18 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in - let open Test_config.Wallet in { default with requires_graphql = true + ; genesis_ledger = + [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-c-key"; balance = "0"; timing = Untimed } + ] ; block_producers = - [ { balance = "1000"; timing = Untimed } - ; { balance = "1000"; timing = Untimed } - ; { balance = "0"; timing = Untimed } + [ { node_name = "node-a"; account_name = "node-a-key" } + ; { node_name = "node-b"; account_name = "node-b-key" } + ; { node_name = "node-c"; account_name = "node-c-key" } ] - ; num_snark_workers = 0 } let run network t = @@ -33,9 +36,18 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Malleable_error.Let_syntax in let logger = Logger.create () in let all_nodes = Network.all_nodes network in - let%bind () = wait_for t (Wait_condition.nodes_to_initialize all_nodes) in - let[@warning "-8"] [ node_a; node_b; node_c ] = - Network.block_producers network + let%bind () = + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) + in + let node_a = + Core.String.Map.find_exn (Network.block_producers network) "node-a" + in + let node_b = + Core.String.Map.find_exn (Network.block_producers network) "node-b" + in + let node_c = + Core.String.Map.find_exn (Network.block_producers network) "node-c" in let%bind _ = section "blocks are produced" @@ -92,7 +104,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct section "common prefix of all nodes is no farther back than 1 block" (* the common prefix test relies on at least 4 blocks having been produced. previous sections altogether have already produced 4, so no further block production is needed. if previous sections change, then this may need to be re-adjusted*) (let%bind (labeled_chains : (string * string list) list) = - Malleable_error.List.map all_nodes ~f:(fun node -> + Malleable_error.List.map (Core.String.Map.data all_nodes) + ~f:(fun node -> let%map chain = Network.Node.must_get_best_chain ~logger node in (Node.id node, List.map ~f:(fun b -> b.state_hash) chain) ) in diff --git a/src/app/test_executive/delegation_test.ml b/src/app/test_executive/delegation_test.ml index 9e9a5d7877f..fb0cb973fac 100644 --- a/src/app/test_executive/delegation_test.ml +++ b/src/app/test_executive/delegation_test.ml @@ -16,14 +16,16 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in - let open Test_config.Wallet in { default with requires_graphql = true + ; genesis_ledger = + [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } + ] ; block_producers = - [ { balance = "1000"; timing = Untimed } - ; { balance = "1000"; timing = Untimed } + [ { node_name = "node-a"; account_name = "node-a-key" } + ; { node_name = "node-b"; account_name = "node-b-key" } ] - ; num_snark_workers = 0 } let run network t = @@ -32,8 +34,16 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct (* fee for user commands *) let fee = Currency.Fee.of_int 10_000_000 in let all_nodes = Network.all_nodes network in - let%bind () = wait_for t (Wait_condition.nodes_to_initialize all_nodes) in - let[@warning "-8"] [ node_a; node_b ] = Network.block_producers network in + let%bind () = + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) + in + let node_a = + Core.String.Map.find_exn (Network.block_producers network) "node-a" + in + let node_b = + Core.String.Map.find_exn (Network.block_producers network) "node-b" + in let%bind () = section "delegate all mina currency from node_b to node_a" (let delegation_receiver = node_a in diff --git a/src/app/test_executive/gossip_consistency.ml b/src/app/test_executive/gossip_consistency.ml index 1628b86d8b4..be471d477db 100644 --- a/src/app/test_executive/gossip_consistency.ml +++ b/src/app/test_executive/gossip_consistency.ml @@ -15,16 +15,18 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct type dsl = Dsl.t - let block_producer_balance = "1000" (* 1_000_000_000_000 *) - let config = - let n = 2 in let open Test_config in { default with requires_graphql = true + ; genesis_ledger = + [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } + ] ; block_producers = - List.init n ~f:(fun _ -> - { Wallet.balance = block_producer_balance; timing = Untimed } ) + [ { node_name = "node-a"; account_name = "node-a-key" } + ; { node_name = "node-b"; account_name = "node-b-key" } + ] } let run network t = @@ -33,13 +35,16 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct [%log info] "gossip_consistency test: starting..." ; let%bind () = wait_for t - (Wait_condition.nodes_to_initialize (Network.all_nodes network)) + (Wait_condition.nodes_to_initialize + (Core.String.Map.data (Network.all_nodes network)) ) in [%log info] "gossip_consistency test: done waiting for initializations" ; - let receiver_bp = Caml.List.nth (Network.block_producers network) 0 in + let receiver_bp = + Core.String.Map.find_exn (Network.block_producers network) "node-a" + in let%bind receiver_pub_key = pub_key_of_node receiver_bp in let sender_bp = - Core_kernel.List.nth_exn (Network.block_producers network) 1 + Core.String.Map.find_exn (Network.block_producers network) "node-b" in let%bind sender_pub_key = pub_key_of_node sender_bp in let num_payments = 3 in diff --git a/src/app/test_executive/medium_bootstrap.ml b/src/app/test_executive/medium_bootstrap.ml index a40f8a28351..74a6b66185d 100644 --- a/src/app/test_executive/medium_bootstrap.ml +++ b/src/app/test_executive/medium_bootstrap.ml @@ -18,16 +18,19 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in - let open Test_config.Wallet in { default with k = 2 ; requires_graphql = true + ; genesis_ledger = + [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-c-key"; balance = "0"; timing = Untimed } + ] ; block_producers = - [ { balance = "1000"; timing = Untimed } - ; { balance = "1000"; timing = Untimed } - ; { balance = "0"; timing = Untimed } + [ { node_name = "node-a"; account_name = "node-a-key" } + ; { node_name = "node-b"; account_name = "node-b-key" } + ; { node_name = "node-c"; account_name = "node-c-key" } ] - ; num_snark_workers = 0 } (* @@ -45,9 +48,18 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Malleable_error.Let_syntax in let logger = Logger.create () in let all_nodes = Network.all_nodes network in - let%bind () = wait_for t (Wait_condition.nodes_to_initialize all_nodes) in - let[@warning "-8"] [ node_a; node_b; node_c ] = - Network.block_producers network + let%bind () = + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) + in + let node_a = + Core.String.Map.find_exn (Network.block_producers network) "node-a" + in + let node_b = + Core.String.Map.find_exn (Network.block_producers network) "node-b" + in + let node_c = + Core.String.Map.find_exn (Network.block_producers network) "node-c" in let%bind () = section "blocks are produced" @@ -70,7 +82,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct section "network is fully connected after one node was restarted" (let%bind () = Malleable_error.lift (after (Time.Span.of_sec 240.0)) in let%bind final_connectivity_data = - fetch_connectivity_data ~logger all_nodes + fetch_connectivity_data ~logger (Core.String.Map.data all_nodes) in assert_peers_completely_connected final_connectivity_data ) end diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index 37bc4e254b8..84e9e3c2be4 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -22,7 +22,6 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct (* TODO: test snark work *) let config = let open Test_config in - let open Test_config.Wallet in let make_timing ~min_balance ~cliff_time ~cliff_amount ~vesting_period ~vesting_increment : Mina_base.Account_timing.t = let open Currency in @@ -36,24 +35,40 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct in { default with requires_graphql = true - ; block_producers = - [ { balance = "400000"; timing = Untimed } (* 400_000_000_000_000 *) - ; { balance = "300000"; timing = Untimed } (* 300_000_000_000_000 *) - ; { balance = "30000" + ; genesis_ledger = + [ { account_name = "untimed-node-a-key" + ; balance = "400000" + ; timing = Untimed (* 400_000_000_000_000 *) + } + ; { account_name = "untimed-node-b-key" + ; balance = "300000" + ; timing = Untimed (* 300_000_000_000_000 *) + } + ; { account_name = "timed-node-c-key" + ; balance = "30000" ; timing = make_timing ~min_balance:10_000_000_000_000 ~cliff_time:8 ~cliff_amount:0 ~vesting_period:4 ~vesting_increment:5_000_000_000_000 + (* 30_000_000_000_000 mina is the total. initially, the balance will be 10k mina. after 8 global slots, the cliff is hit, although the cliff amount is 0. 4 slots after that, 5_000_000_000_000 mina will vest, and 4 slots after that another 5_000_000_000_000 will vest, and then twice again, for a total of 30k mina all fully liquid and unlocked at the end of the schedule*) } - (* 30_000_000_000_000 mina is the total. initially, the balance will be 10k mina. after 8 global slots, the cliff is hit, although the cliff amount is 0. 4 slots after that, 5_000_000_000_000 mina will vest, and 4 slots after that another 5_000_000_000_000 will vest, and then twice again, for a total of 30k mina all fully liquid and unlocked at the end of the schedule*) + ; { account_name = "snark-node-key"; balance = "100"; timing = Untimed } + ; { account_name = "fish1"; balance = "100"; timing = Untimed } + ; { account_name = "fish2"; balance = "100"; timing = Untimed } ] - ; extra_genesis_accounts = - [ { balance = "1000"; timing = Untimed } - ; { balance = "1000"; timing = Untimed } + ; block_producers = + [ { node_name = "untimed-node-a"; account_name = "untimed-node-a-key" } + ; { node_name = "untimed-node-b"; account_name = "untimed-node-b-key" } + ; { node_name = "timed-node-c"; account_name = "timed-node-c-key" } ] - ; num_archive_nodes = 1 - ; num_snark_workers = 4 + ; snark_coordinator = + Some + { node_name = "snark-node" + ; account_name = "snark-node-key" + ; worker_nodes = 8 + } ; snark_worker_fee = "0.0001" + ; num_archive_nodes = 1 ; proof_config = { proof_config_default with work_delay = Some 1 @@ -62,50 +77,52 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct } } - (* Call [f] [n] times in sequence *) - let repeat_seq ~n ~f = - let open Malleable_error.Let_syntax in - let rec go n = - if n = 0 then return () - else - let%bind () = f () in - go (n - 1) - in - go n - let run network t = let open Network in let open Malleable_error.Let_syntax in let logger = Logger.create () in let all_nodes = Network.all_nodes network in - let%bind () = wait_for t (Wait_condition.nodes_to_initialize all_nodes) in - let[@warning "-8"] [ untimed_node_a; untimed_node_b; timed_node_c ] = - Network.block_producers network + let%bind () = + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) + in + let untimed_node_a = + Core.String.Map.find_exn + (Network.block_producers network) + "untimed-node-a" + in + let untimed_node_b = + Core.String.Map.find_exn + (Network.block_producers network) + "untimed-node-b" + in + let timed_node_c = + Core.String.Map.find_exn (Network.block_producers network) "timed-node-c" + in + let fish1 = + Core.String.Map.find_exn (Network.genesis_keypairs network) "fish1" + in + let fish2 = + Core.String.Map.find_exn (Network.genesis_keypairs network) "fish2" in [%log info] "extra genesis keypairs: %s" - (List.to_string (Network.extra_genesis_keypairs network) + (List.to_string [ fish1.keypair; fish2.keypair ] ~f:(fun { Signature_lib.Keypair.public_key; _ } -> public_key |> Signature_lib.Public_key.to_bigstring |> Bigstring.to_string ) ) ; - let[@warning "-8"] [ fish1; fish2 ] = - Network.extra_genesis_keypairs network - in (* create a signed txn which we'll use to make a successfull txn, and then a replay attack *) let amount = Currency.Amount.of_formatted_string "10" in let fee = Currency.Fee.of_formatted_string "1" in let test_constants = Engine.Network.constraint_constants network in let receiver_pub_key = - fish1.public_key |> Signature_lib.Public_key.compress + fish1.keypair.public_key |> Signature_lib.Public_key.compress in - let sender_kp = fish2 in let sender_pub_key = - sender_kp.public_key |> Signature_lib.Public_key.compress + fish2.keypair.public_key |> Signature_lib.Public_key.compress in (* hardcoded copy of extra_genesis_accounts[0] and extra_genesis_accounts[1], update here if they change *) - let receiver_original_balance = - Currency.Amount.of_formatted_string "1000" - in - let sender_original_balance = Currency.Amount.of_formatted_string "1000" in + let receiver_original_balance = Currency.Amount.of_formatted_string "100" in + let sender_original_balance = Currency.Amount.of_formatted_string "100" in let txn_body = Signed_command_payload.Body.Payment { source_pk = sender_pub_key @@ -124,7 +141,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ~fee_payer_pk:sender_pub_key ~valid_until:None ~memo:(Signed_command_memo.create_from_string_exn "") ~body:txn_body ~signer:sender_pub_key - ~sign_choice:(User_command_input.Sign_choice.Keypair sender_kp) () + ~sign_choice:(User_command_input.Sign_choice.Keypair fish2.keypair) () in [%log info] "user_command_input: $user_command" ~metadata: @@ -151,7 +168,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct in (* setup complete *) let%bind () = - section "send a single payment between 2 untimed accounts" + section "send a single signed payment between 2 fish accounts" (let%bind { hash; _ } = Network.Node.must_send_payment_with_raw_sig untimed_node_b ~logger ~sender_pub_key: @@ -399,7 +416,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let%bind receiver_pub_key = pub_key_of_node receiver in let sender = untimed_node_b in let%bind sender_pub_key = pub_key_of_node sender in - let%bind () = + let%bind _ = (* To fill up a `small` transaction capacity with work delay of 1, there needs to be 12 total txns sent. @@ -415,10 +432,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct 2 successful txn are sent in the prior course of this test, so spamming out at least 10 more here will trigger a ledger proof to be emitted *) - repeat_seq ~n:10 ~f:(fun () -> - Network.Node.must_send_payment ~logger sender ~sender_pub_key - ~receiver_pub_key ~amount:Currency.Amount.one ~fee - >>| ignore ) + send_payments ~logger ~sender_pub_key ~receiver_pub_key + ~amount:Currency.Amount.one ~fee ~node:sender 10 in wait_for t (Wait_condition.ledger_proofs_emitted_since_genesis ~num_proofs:1) ) @@ -426,7 +441,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct section_hard "running replayer" (let%bind logs = Network.Node.run_replayer ~logger - (List.hd_exn @@ Network.archive_nodes network) + (List.hd_exn @@ (Network.archive_nodes network |> Core.Map.data)) in check_replayer_logs ~logger logs ) end diff --git a/src/app/test_executive/peers_reliability_test.ml b/src/app/test_executive/peers_reliability_test.ml index 505544419ef..f42ad3a7a87 100644 --- a/src/app/test_executive/peers_reliability_test.ml +++ b/src/app/test_executive/peers_reliability_test.ml @@ -18,15 +18,18 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in - let open Test_config.Wallet in { default with requires_graphql = true + ; genesis_ledger = + [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } + ; { account_name = "node-c-key"; balance = "0"; timing = Untimed } + ] ; block_producers = - [ { balance = "1000"; timing = Untimed } - ; { balance = "1000"; timing = Untimed } - ; { balance = "0"; timing = Untimed } + [ { node_name = "node-a"; account_name = "node-a-key" } + ; { node_name = "node-b"; account_name = "node-b-key" } + ; { node_name = "node-c"; account_name = "node-c-key" } ] - ; num_snark_workers = 0 } let run network t = @@ -36,14 +39,26 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let all_nodes = Network.all_nodes network in [%log info] "peers_list" ~metadata: - [ ("peers", `List (List.map all_nodes ~f:(fun n -> `String (Node.id n)))) + [ ( "peers" + , `List + (List.map (Core.String.Map.data all_nodes) ~f:(fun n -> + `String (Node.id n) ) ) ) ] ; - let%bind () = wait_for t (Wait_condition.nodes_to_initialize all_nodes) in - let[@warning "-8"] [ node_a; node_b; node_c ] = - Network.block_producers network + let%bind () = + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) + in + let node_a = + Core.String.Map.find_exn (Network.block_producers network) "node-a" + in + let node_b = + Core.String.Map.find_exn (Network.block_producers network) "node-b" + in + let node_c = + Core.String.Map.find_exn (Network.block_producers network) "node-c" in let%bind initial_connectivity_data = - fetch_connectivity_data ~logger all_nodes + fetch_connectivity_data ~logger (Core.String.Map.data all_nodes) in let%bind () = section "network is fully connected upon initialization" @@ -81,7 +96,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct section "network is fully connected after one node was restarted" (let%bind () = Malleable_error.lift (after (Time.Span.of_sec 240.0)) in let%bind final_connectivity_data = - fetch_connectivity_data ~logger all_nodes + fetch_connectivity_data ~logger (Core.String.Map.data all_nodes) in assert_peers_completely_connected final_connectivity_data ) end diff --git a/src/app/test_executive/snark_test.ml b/src/app/test_executive/snark_test.ml new file mode 100644 index 00000000000..1ae6180c3a5 --- /dev/null +++ b/src/app/test_executive/snark_test.ml @@ -0,0 +1,205 @@ +open Core +open Integration_test_lib +open Mina_base +open Async + +module Make (Inputs : Intf.Test.Inputs_intf) = struct + open Inputs + open Engine + open Dsl + + open Test_common.Make (Inputs) + + (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) + type network = Network.t + + type node = Network.Node.t + + type dsl = Dsl.t + + (* TODO: refactor all currency values to decimal represenation *) + (* TODO: test account creation fee *) + (* TODO: test snark work *) + let config = + let open Test_config in + { default with + requires_graphql = true + ; genesis_ledger = + [ { account_name = "node-a-key"; balance = "400000"; timing = Untimed } + ; { account_name = "node-b-key"; balance = "300000"; timing = Untimed } + ; { account_name = "fish1"; balance = "100"; timing = Untimed } + ; { account_name = "fish2"; balance = "100"; timing = Untimed } + ; { account_name = "snark-node-key"; balance = "100"; timing = Untimed } + ] + ; block_producers = + [ { node_name = "node-a"; account_name = "node-a-key" } + (* 400_000_000_000_000 *) + ; { node_name = "node-b"; account_name = "node-b-key" } + (* 300_000_000_000_000 *) + ] + ; num_archive_nodes = 0 + ; snark_coordinator = + Some + { node_name = "snark-node" + ; account_name = "snark-node-key" + ; worker_nodes = 8 + } + ; snark_worker_fee = "0.0001" + ; proof_config = + { proof_config_default with + work_delay = Some 1 + ; transaction_capacity = + Some Runtime_config.Proof_keys.Transaction_capacity.small + } + } + + let run network t = + let open Malleable_error.Let_syntax in + let logger = Logger.create () in + let all_nodes = Network.all_nodes network in + let%bind () = + wait_for t + (Wait_condition.nodes_to_initialize (Core.String.Map.data all_nodes)) + in + let node_a = + Core.String.Map.find_exn (Network.block_producers network) "node-a" + in + let node_b = + Core.String.Map.find_exn (Network.block_producers network) "node-b" + in + let fish1 = + Core.String.Map.find_exn (Network.genesis_keypairs network) "fish1" + in + let fish2 = + Core.String.Map.find_exn (Network.genesis_keypairs network) "fish2" + in + [%log info] "extra genesis keypairs: %s" + (List.to_string [ fish1.keypair; fish2.keypair ] + ~f:(fun { Signature_lib.Keypair.public_key; _ } -> + public_key |> Signature_lib.Public_key.to_bigstring + |> Bigstring.to_string ) ) ; + let amount = Currency.Amount.of_formatted_string "10" in + let fee = Currency.Fee.of_formatted_string "1" in + let test_constants = Engine.Network.constraint_constants network in + let receiver_pub_key = + fish1.keypair.public_key |> Signature_lib.Public_key.compress + in + let sender_pub_key = + fish2.keypair.public_key |> Signature_lib.Public_key.compress + in + let txn_body = + Signed_command_payload.Body.Payment + { source_pk = sender_pub_key + ; receiver_pk = receiver_pub_key + ; token_id = Token_id.default + ; amount + } + in + let%bind { nonce = sender_current_nonce; _ } = + Network.Node.must_get_account_data ~logger node_b + ~public_key:sender_pub_key + in + let user_command_input = + User_command_input.create ~fee ~nonce:sender_current_nonce + ~fee_token:(Signed_command_payload.Body.token txn_body) + ~fee_payer_pk:sender_pub_key ~valid_until:None + ~memo:(Signed_command_memo.create_from_string_exn "") + ~body:txn_body ~signer:sender_pub_key + ~sign_choice:(User_command_input.Sign_choice.Keypair fish2.keypair) () + in + [%log info] "user_command_input: $user_command" + ~metadata: + [ ( "user_command" + , User_command_input.Stable.Latest.to_yojson user_command_input ) + ] ; + let%bind txn_signed = + User_command_input.to_user_command + ~get_current_nonce:(fun _ -> failwith "get_current_nonce, don't call me") + ~nonce_map: + (Account_id.Map.of_alist_exn + [ ( Account_id.create sender_pub_key + (Signed_command_payload.Body.token txn_body) + , (sender_current_nonce, sender_current_nonce) ) + ] ) + ~get_account:(fun _ -> `Bootstrapping) + ~constraint_constants:test_constants ~logger user_command_input + |> Deferred.bind ~f:Malleable_error.or_hard_error + in + let (signed_cmmd, _) + : Signed_command.t + * (Unsigned.uint32 * Unsigned.uint32) Account_id.Map.t = + txn_signed + in + (* let snark_worker_pk = Test_config.default.snark_worker_public_key in *) + let%bind () = + section "send a single signed payment between 2 fish accounts" + (let%bind { hash; _ } = + Network.Node.must_send_payment_with_raw_sig node_b ~logger + ~sender_pub_key: + (Signed_command_payload.Body.source_pk signed_cmmd.payload.body) + ~receiver_pub_key: + (Signed_command_payload.Body.receiver_pk signed_cmmd.payload.body) + ~amount: + ( Signed_command_payload.amount signed_cmmd.payload + |> Option.value_exn ) + ~fee:(Signed_command_payload.fee signed_cmmd.payload) + ~nonce:signed_cmmd.payload.common.nonce + ~memo: + (Signed_command_memo.to_raw_bytes_exn + signed_cmmd.payload.common.memo ) + ~token:(Signed_command_payload.token signed_cmmd.payload) + ~valid_until:signed_cmmd.payload.common.valid_until + ~raw_signature: + (Mina_base.Signature.Raw.encode signed_cmmd.signature) + in + wait_for t + (Wait_condition.signed_command_to_be_included_in_frontier + ~txn_hash:hash ~node_included_in:(`Node node_b) ) ) + in + + (* let%bind () = + section_hard "check snark worker's account balance" + (let%bind { total_balance = snark_worker_balance; _ } = + Network.Node.must_get_account_data ~logger untimed_node_b + ~public_key:snark_worker_pk + in + if + Currency.Amount.( >= ) + (Currency.Balance.to_amount snark_worker_balance) + snark_worker_expected + then Malleable_error.return () + else + Malleable_error.soft_error_format ~value:() + "Error with snark_worker_balance. snark_worker_balance is %d and \ + should be %d. snark fee is %d" + (Currency.Balance.to_int snark_worker_balance) + (Currency.Amount.to_int snark_worker_expected) + (Currency.Fee.to_int fee) ) + in *) + section_hard + "send out a bunch more txns to fill up the snark ledger, then wait for \ + proofs to be emitted" + (let receiver = node_a in + let%bind receiver_pub_key = pub_key_of_node receiver in + let sender = node_b in + let%bind sender_pub_key = pub_key_of_node sender in + let%bind _ = + (* + To fill up a `small` transaction capacity with work delay of 1, + there needs to be 12 total txns sent. + + Calculation is as follows: + Max number trees in the scan state is + `(transaction_capacity_log+1) * (work_delay+1)` + and for 2^2 transaction capacity and work delay 1 it is + `(2+1)*(1+1)=6`. + Per block there can be 2 transactions included (other two slots would be for a coinbase and fee transfers). + In the initial state of the network, the scan state waits till all the trees are filled before emitting a proof from the first tree. + Hence, 6*2 = 12 transactions untill we get the first snarked ledger. +*) + send_payments ~logger ~sender_pub_key ~receiver_pub_key + ~amount:Currency.Amount.one ~fee ~node:sender 12 + in + wait_for t + (Wait_condition.ledger_proofs_emitted_since_genesis ~num_proofs:1) ) +end diff --git a/src/app/test_executive/snark_test.mli b/src/app/test_executive/snark_test.mli new file mode 100644 index 00000000000..8664ff022c1 --- /dev/null +++ b/src/app/test_executive/snark_test.mli @@ -0,0 +1 @@ +module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/test_common.ml b/src/app/test_executive/test_common.ml index 796c9b15cb8..c5ecc67f3a6 100644 --- a/src/app/test_executive/test_common.ml +++ b/src/app/test_executive/test_common.ml @@ -8,6 +8,17 @@ open Mina_base module Make (Inputs : Intf.Test.Inputs_intf) = struct open Inputs + (* Call [f] [n] times in sequence *) + let repeat_seq ~n ~f = + let open Malleable_error.Let_syntax in + let rec go n = + if n = 0 then return () + else + let%bind () = f () in + go (n - 1) + in + go n + let send_payments ~logger ~sender_pub_key ~receiver_pub_key ~amount ~fee ~node n = let open Malleable_error.Let_syntax in @@ -19,8 +30,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct Engine.Network.Node.must_send_payment ~logger ~sender_pub_key ~receiver_pub_key ~amount ~fee node in - [%log info] "gossip_consistency test: payment #%d sent with hash %s." - n + [%log info] + "sending multiple payments: payment #%d sent with hash %s." n (Transaction_hash.to_base58_check hash) ; hash in @@ -42,7 +53,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ~txn_hash:hash ~node_included_in:`Any_node ) in [%log info] - "gossip_consistency test: payment #%d with hash %s successfully \ + "wait for multiple payments: payment #%d with hash %s successfully \ included in frontier." n (Transaction_hash.to_base58_check hash) ; diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 05d5eeca393..faf9e708f94 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -61,6 +61,7 @@ let tests : test list = ; ("medium-bootstrap", (module Medium_bootstrap.Make : Intf.Test.Functor_intf)) ; ( "opt-block-prod" , (module Block_production_priority.Make : Intf.Test.Functor_intf) ) + ; ("snark", (module Snark_test.Make : Intf.Test.Functor_intf)) ] let report_test_errors ~log_error_set ~internal_error_set = @@ -360,8 +361,12 @@ let main inputs = [%log info] "%s started" (Engine.Network.Node.id node) ; Malleable_error.return res in - let seed_nodes = network |> Engine.Network.seeds in - let non_seed_pods = network |> Engine.Network.all_non_seed_pods in + let seed_nodes = + network |> Engine.Network.seeds |> Core.String.Map.data + in + let non_seed_pods = + network |> Engine.Network.all_non_seed_pods |> Core.String.Map.data + in (* TODO: parallelize (requires accumlative hard errors) *) let%bind () = Malleable_error.List.iter seed_nodes ~f:start_print in let%bind () = diff --git a/src/lib/integration_test_cloud_engine/kubernetes_network.ml b/src/lib/integration_test_cloud_engine/kubernetes_network.ml index 9d0502a7695..f8ff4b79759 100644 --- a/src/lib/integration_test_cloud_engine/kubernetes_network.ml +++ b/src/lib/integration_test_cloud_engine/kubernetes_network.ml @@ -38,27 +38,36 @@ module Node = struct } type t = - { app_id : string; pod_id : string; pod_info : pod_info; config : config } + { app_id : string + ; pod_ids : string list + ; pod_info : pod_info + ; config : config + } - let id { pod_id; _ } = pod_id + let id { pod_ids; _ } = List.hd_exn pod_ids let network_keypair { pod_info = { network_keypair; _ }; _ } = network_keypair let base_kube_args t = [ "--cluster"; t.cluster; "--namespace"; t.namespace ] - let get_logs_in_container ?container_id { pod_id; config; pod_info; _ } = + let get_logs_in_container ?container_id { pod_ids; config; pod_info; _ } = let container_id = Option.value container_id ~default:pod_info.primary_container_id in let%bind cwd = Unix.getcwd () in Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "kubectl" - (base_kube_args config @ [ "logs"; "-c"; container_id; pod_id ]) + ( base_kube_args config + @ [ "logs"; "-c"; container_id; List.hd_exn pod_ids ] ) let run_in_container ?(exit_code = 10) ?container_id ?override_with_pod_id ~cmd t = let { config; pod_info; _ } = t in let pod_id = - match override_with_pod_id with Some pid -> pid | None -> t.pod_id + match override_with_pod_id with + | Some pid -> + pid + | None -> + List.hd_exn t.pod_ids in let container_id = Option.value container_id ~default:pod_info.primary_container_id @@ -70,7 +79,7 @@ module Node = struct @ cmd ) let cp_string_to_container_file ?container_id ~str ~dest t = - let { pod_id; config; pod_info; _ } = t in + let { pod_ids; config; pod_info; _ } = t in let container_id = Option.value container_id ~default:pod_info.primary_container_id in @@ -81,7 +90,9 @@ module Node = struct Out_channel.output_string oc str ; Out_channel.close oc ; let%bind cwd = Unix.getcwd () in - let dest_file = sprintf "%s/%s:%s" config.namespace pod_id dest in + let dest_file = + sprintf "%s/%s:%s" config.namespace (List.hd_exn pod_ids) dest + in Integration_test_lib.Util.run_cmd_or_error cwd "kubectl" (base_kube_args config @ [ "cp"; "-c"; container_id; tmp_file; dest_file ]) @@ -102,7 +113,7 @@ module Node = struct let logger_metadata node = [ ("namespace", `String node.config.namespace) ; ("app_id", `String node.app_id) - ; ("pod_id", `String node.pod_id) + ; ("pod_id", `String (List.hd_exn node.pod_ids)) ] module Scalars = Graphql_lib.Scalars @@ -635,7 +646,7 @@ module Node = struct let run_replayer ~logger (t : t) = [%log info] "Running replayer on archived data (node: %s, container: %s)" - t.pod_id mina_archive_container_id ; + (List.hd_exn t.pod_ids) mina_archive_container_id ; let open Malleable_error.Let_syntax in let%bind accounts = run_in_container t @@ -666,8 +677,8 @@ module Node = struct let dump_mina_logs ~logger (t : t) ~log_file = let open Malleable_error.Let_syntax in - [%log info] "Dumping container logs from (node: %s, container: %s)" t.pod_id - t.pod_info.primary_container_id ; + [%log info] "Dumping container logs from (node: %s, container: %s)" + (List.hd_exn t.pod_ids) t.pod_info.primary_container_id ; let%map logs = get_logs_in_container t in [%log info] "Dumping container log to file %s" log_file ; Out_channel.with_file log_file ~f:(fun out_ch -> @@ -677,7 +688,7 @@ module Node = struct let open Malleable_error.Let_syntax in [%log info] "Dumping precomputed blocks from logs for (node: %s, container: %s)" - t.pod_id t.pod_info.primary_container_id ; + (List.hd_exn t.pod_ids) t.pod_info.primary_container_id ; let%bind logs = get_logs_in_container t in (* kubectl logs may include non-log output, like "Using password from environment variable" *) let log_lines = @@ -789,7 +800,7 @@ module Node = struct end module Workload_to_deploy = struct - type t = { workload_id : string; pod_info : Node.pod_info list } + type t = { workload_id : string; pod_info : Node.pod_info } let construct_workload workload_id pod_info : t = { workload_id; pod_info } @@ -821,28 +832,24 @@ module Workload_to_deploy = struct |> List.filter ~f:(Fn.compose not String.is_empty) |> List.map ~f:(String.substr_replace_first ~pattern:"pod/" ~with_:"") in - if Stdlib.List.compare_lengths t.pod_info pod_ids <> 0 then - failwithf - "Unexpected number of replicas in kubernetes deployment for workload \ - %s: expected %d, got %d" - t.workload_id (List.length t.pod_info) (List.length pod_ids) () ; - List.zip_exn t.pod_info pod_ids - |> List.map ~f:(fun (pod_info, pod_id) -> - { Node.app_id; pod_id; pod_info; config } ) + (* we have a strict 1 workload to 1 pod setup, except the snark workers. *) + (* elsewhere in the code I'm simply using List.hd_exn which is not ideal but enabled by the fact that in all relevant cases, there's only going to be 1 pod id in pod_ids *) + (* TODO fix this^ and have a more elegant solution *) + let pod_info = t.pod_info in + { Node.app_id; pod_ids; pod_info; config } end type t = { namespace : string ; constants : Test_config.constants - ; seeds : Node.t list - ; block_producers : Node.t list - ; snark_coordinators : Node.t list - ; snark_workers : Node.t list - ; archive_nodes : Node.t list - ; testnet_log_filter : string (* ; keypairs : Signature_lib.Keypair.t list *) - ; block_producer_keypairs : Signature_lib.Keypair.t list - ; extra_genesis_keypairs : Signature_lib.Keypair.t list - ; nodes_by_pod_id : Node.t String.Map.t + ; seeds : Node.t Core.String.Map.t + ; block_producers : Node.t Core.String.Map.t + ; snark_coordinators : Node.t Core.String.Map.t + ; snark_workers : Node.t Core.String.Map.t + ; archive_nodes : Node.t Core.String.Map.t + (* ; nodes_by_pod_id : Node.t Core.String.Map.t *) + ; testnet_log_filter : string + ; genesis_keypairs : Network_keypair.t Core.String.Map.t } let constants { constants; _ } = constants @@ -863,48 +870,59 @@ let archive_nodes { archive_nodes; _ } = archive_nodes (* all_nodes returns all *actual* mina nodes; note that a snark_worker is a pod within the network but not technically a mina node, therefore not included here. snark coordinators on the other hand ARE mina nodes *) let all_nodes { seeds; block_producers; snark_coordinators; archive_nodes; _ } = - List.concat [ seeds; block_producers; snark_coordinators; archive_nodes ] + List.concat + [ Core.String.Map.to_alist seeds + ; Core.String.Map.to_alist block_producers + ; Core.String.Map.to_alist snark_coordinators + ; Core.String.Map.to_alist archive_nodes + ] + |> Core.String.Map.of_alist_exn (* all_pods returns everything in the network. remember that snark_workers will never initialize and will never sync, and aren't supposed to *) -let all_pods - { seeds - ; block_producers - ; snark_coordinators - ; snark_workers - ; archive_nodes - ; _ - } = +(* TODO snark workers and snark coordinators have the same key name, but different workload ids*) +let all_pods t = List.concat - [ seeds; block_producers; snark_coordinators; snark_workers; archive_nodes ] + [ Core.String.Map.to_alist t.seeds + ; Core.String.Map.to_alist t.block_producers + ; Core.String.Map.to_alist t.snark_coordinators + ; Core.String.Map.to_alist t.snark_workers + ; Core.String.Map.to_alist t.archive_nodes + ] + |> Core.String.Map.of_alist_exn (* all_non_seed_pods returns everything in the network except seed nodes *) -let all_non_seed_pods - { block_producers; snark_coordinators; snark_workers; archive_nodes; _ } = +let all_non_seed_pods t = List.concat - [ block_producers; snark_coordinators; snark_workers; archive_nodes ] - -let all_keypairs { block_producer_keypairs; extra_genesis_keypairs; _ } = - block_producer_keypairs @ extra_genesis_keypairs - -let block_producer_keypairs { block_producer_keypairs; _ } = - block_producer_keypairs + [ Core.String.Map.to_alist t.block_producers + ; Core.String.Map.to_alist t.snark_coordinators + ; Core.String.Map.to_alist t.snark_workers + ; Core.String.Map.to_alist t.archive_nodes + ] + |> Core.String.Map.of_alist_exn -let extra_genesis_keypairs { extra_genesis_keypairs; _ } = - extra_genesis_keypairs +let genesis_keypairs { genesis_keypairs; _ } = genesis_keypairs -let lookup_node_by_pod_id t = Map.find t.nodes_by_pod_id +let lookup_node_by_pod_id t id = + let pods = all_pods t |> Core.Map.to_alist in + List.fold pods ~init:None ~f:(fun acc (node_name, node) -> + match acc with + | Some acc -> + Some acc + | None -> + if String.equal id (List.hd_exn node.pod_ids) then + Some (node_name, node) + else None ) -let all_pod_ids t = Map.keys t.nodes_by_pod_id +let all_pod_ids t = + let pods = all_pods t |> Core.Map.to_alist in + List.fold pods ~init:[] ~f:(fun acc (_, node) -> + List.cons (List.hd_exn node.pod_ids) acc ) let initialize_infra ~logger network = let open Malleable_error.Let_syntax in let poll_interval = Time.Span.of_sec 15.0 in let max_polls = 40 (* 10 mins *) in - let all_pods_set = - all_pods network - |> List.map ~f:(fun { pod_id; _ } -> pod_id) - |> String.Set.of_list - in + let all_pods_set = all_pod_ids network |> String.Set.of_list in let kube_get_pods () = Integration_test_lib.Util.run_cmd_or_error_timeout ~timeout_seconds:60 "/" "kubectl" diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index 7dfdf212b00..24e4ff4e70f 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -23,16 +23,16 @@ module Network_config = struct module Cli_inputs = Cli_inputs type block_producer_config = - { name : string - ; id : string + { name : string (* ; id : string *) ; keypair : Network_keypair.t - ; public_key : string - ; private_key : string - ; keypair_secret : string ; libp2p_secret : string } [@@deriving to_yojson] + type snark_coordinator_config = + { name : string; public_key : string; worker_nodes : int } + [@@deriving to_yojson] + type terraform_config = { k8s_context : string ; cluster_name : string @@ -53,9 +53,8 @@ module Network_config = struct ; archive_node_count : int ; mina_archive_schema : string ; mina_archive_schema_aux_files : string list - ; snark_worker_replicas : int + ; snark_coordinator_config : snark_coordinator_config option ; snark_worker_fee : string - ; snark_worker_public_key : string ; cpu_request : int ; mem_request : string ; worker_cpu_request : int @@ -65,12 +64,19 @@ module Network_config = struct type t = { mina_automation_location : string - ; debug_arg : bool (* ; keypairs : Network_keypair.t list *) + ; debug_arg : bool ; check_capacity : bool ; check_capacity_delay : int ; check_capacity_retries : int - ; block_producer_keypairs : Network_keypair.t list - ; extra_genesis_keypairs : Network_keypair.t list + ; genesis_keypairs : + (Network_keypair.t Core.String.Map.t + [@to_yojson + fun map -> + `Assoc + (Core.Map.fold_right ~init:[] + ~f:(fun ~key:k ~data:v accum -> + (k, Network_keypair.to_yojson v) :: accum ) + map )] ) ; constants : Test_config.constants ; terraform : terraform_config } @@ -85,20 +91,19 @@ module Network_config = struct let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = - let { Test_config.k + let { requires_graphql + ; genesis_ledger + ; block_producers + ; snark_coordinator + ; snark_worker_fee + ; num_archive_nodes + ; log_precomputed_blocks (* ; num_plain_nodes *) + ; proof_config + ; Test_config.k ; delta ; slots_per_epoch ; slots_per_sub_window ; txpool_max_size - ; requires_graphql - ; block_producers - ; extra_genesis_accounts - ; num_snark_workers - ; num_archive_nodes - ; log_precomputed_blocks - ; snark_worker_fee - ; snark_worker_public_key - ; proof_config } = test_config in @@ -111,56 +116,46 @@ module Network_config = struct let git_commit = Mina_version.commit_id_short in (* see ./src/app/test_executive/README.md for information regarding the namespace name format and length restrictions *) let testnet_name = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in - (* GENERATE ACCOUNTS AND KEYPAIRS *) - let num_block_producers = List.length block_producers in - let bp_keypairs, extra_keypairs = - List.split_n - (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) - (List.drop (Array.to_list (Lazy.force Sample_keypairs.keypairs)) 1) - num_block_producers + + (* check to make sure the test writer hasn't accidentally created duplicate names of accounts and keys *) + let key_names_list = + List.map genesis_ledger ~f:(fun acct -> acct.account_name) in - if List.length bp_keypairs < num_block_producers then + if List.contains_dup ~compare:String.compare key_names_list then failwith - "not enough sample keypairs for specified number of block producers" ; - assert (List.length bp_keypairs >= num_block_producers) ; - if List.length bp_keypairs < num_block_producers then + "All accounts in genesis ledger must have unique names. Check to make \ + sure you are not using the same account_name more than once" ; + let all_nodes_names_list = + List.map block_producers ~f:(fun acct -> acct.node_name) + @ match snark_coordinator with None -> [] | Some n -> [ n.node_name ] + in + if List.contains_dup ~compare:String.compare all_nodes_names_list then failwith - "not enough sample keypairs for specified number of extra keypairs" ; - assert (List.length extra_keypairs >= List.length extra_genesis_accounts) ; - let extra_keypairs_cut = - List.take extra_keypairs (List.length extra_genesis_accounts) - in - let extra_accounts = - List.map (List.zip_exn extra_genesis_accounts extra_keypairs_cut) - ~f:(fun ({ Test_config.Wallet.balance; timing }, (pk, sk)) -> - let timing = - match timing with - | Account.Timing.Untimed -> - None - | Timed t -> - Some - { Runtime_config.Accounts.Single.Timed.initial_minimum_balance = - t.initial_minimum_balance - ; cliff_time = t.cliff_time - ; cliff_amount = t.cliff_amount - ; vesting_period = t.vesting_period - ; vesting_increment = t.vesting_increment - } + "All nodes in testnet must have unique names. Check to make sure you \ + are not using the same node_name more than once" ; + + (* GENERATE ACCOUNTS AND KEYPAIRS *) + let keypairs = + List.take + (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) + (List.drop (Array.to_list (Lazy.force Sample_keypairs.keypairs)) 1) + (List.length genesis_ledger) + in + let labeled_accounts : + ( Runtime_config.Accounts.single + * (Public_key.Compressed.t * Private_key.t) ) + String.Map.t = + String.Map.empty + in + let rec add_accounts mp zip = + match zip with + | [] -> + mp + | hd :: tl -> + let ( { Test_config.Test_Account.balance; account_name; timing } + , (pk, sk) ) = + hd in - let default = Runtime_config.Accounts.Single.default in - { default with - pk = Some (Public_key.Compressed.to_string pk) - ; sk = Some (Private_key.to_base58_check sk) - ; balance = - Balance.of_formatted_string balance - (* delegation currently unsupported *) - ; delegate = None - ; timing - } ) - in - let bp_accounts = - List.map (List.zip_exn block_producers bp_keypairs) - ~f:(fun ({ Test_config.Wallet.balance; timing }, (pk, _)) -> let timing = match timing with | Account.Timing.Untimed -> @@ -176,15 +171,23 @@ module Network_config = struct } in let default = Runtime_config.Accounts.Single.default in - { default with - pk = Some (Public_key.Compressed.to_string pk) - ; sk = None - ; balance = - Balance.of_formatted_string balance - (* delegation currently unsupported *) - ; delegate = None - ; timing - } ) + let acct = + { default with + pk = Some (Public_key.Compressed.to_string pk) + ; sk = Some (Private_key.to_base58_check sk) + ; balance = + Balance.of_formatted_string balance + (* delegation currently unsupported *) + ; delegate = None + ; timing + } + in + add_accounts + (String.Map.add_exn mp ~key:account_name ~data:(acct, (pk, sk))) + tl + in + let genesis_ledger_accounts = + add_accounts labeled_accounts (List.zip_exn genesis_ledger keypairs) in (* DAEMON CONFIG *) let constraint_constants = @@ -206,7 +209,12 @@ module Network_config = struct ; proof = Some proof_config (* TODO: prebake ledger and only set hash *) ; ledger = Some - { base = Accounts (bp_accounts @ extra_accounts) + { base = + Accounts + (let tuplist = String.Map.data genesis_ledger_accounts in + List.map tuplist ~f:(fun tup -> + let acct, _ = tup in + acct ) ) ; add_genesis_winner = None ; num_accounts = None ; balances = [] @@ -225,43 +233,90 @@ module Network_config = struct { constraints = constraint_constants; genesis = genesis_constants } in (* BLOCK PRODUCER CONFIG *) - let block_producer_config index keypair = - { name = "test-block-producer-" ^ Int.to_string (index + 1) - ; id = Int.to_string index - ; keypair - ; keypair_secret = keypair.secret_name - ; public_key = keypair.public_key_file - ; private_key = keypair.private_key_file - ; libp2p_secret = "" - } + let mk_net_keypair keypair_name (pk, sk) = + let keypair = + { Keypair.public_key = Public_key.decompress_exn pk; private_key = sk } + in + Network_keypair.create_network_keypair ~keypair_name ~keypair + in + let block_producer_config name keypair = + { name; keypair; libp2p_secret = "" } + in + let block_producer_configs = + List.map block_producers ~f:(fun node -> + let _, key_tup = + match String.Map.find genesis_ledger_accounts node.account_name with + | Some acct -> + acct + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial block \ + producers must be in the genesis ledger. name of Node: \ + %s. name of Account which does not exist: %s" + node.node_name node.account_name + in + failwith failstring + in + block_producer_config node.node_name + (mk_net_keypair node.account_name key_tup) ) in let mina_archive_schema = "create_schema.sql" in + let long_commit_id = + if String.is_substring Mina_version.commit_id ~substring:"[DIRTY]" then + String.sub Mina_version.commit_id ~pos:7 + ~len:(String.length Mina_version.commit_id - 7) + else Mina_version.commit_id + in let mina_archive_base_url = - "https://raw.githubusercontent.com/MinaProtocol/mina/" - ^ Mina_version.commit_id ^ "/src/app/archive/" + "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id + ^ "/src/app/archive/" in let mina_archive_schema_aux_files = [ mina_archive_base_url ^ "create_schema.sql" ] in - let mk_net_keypair index (pk, sk) = - let secret_name = "test-keypair-" ^ Int.to_string index in - let keypair = - { Keypair.public_key = Public_key.decompress_exn pk; private_key = sk } - in - Network_keypair.create_network_keypair ~keypair ~secret_name - in - let extra_genesis_net_keypairs = - List.mapi extra_keypairs_cut ~f:mk_net_keypair + let genesis_keypairs = + String.Map.of_alist_exn + (List.map (String.Map.to_alist genesis_ledger_accounts) + ~f:(fun element -> + let kp_name, (_, (pk, sk)) = element in + (kp_name, mk_net_keypair kp_name (pk, sk)) ) ) + in + let snark_coordinator_config = + match snark_coordinator with + | None -> + None + | Some node -> + let network_kp = + match String.Map.find genesis_keypairs node.account_name with + | Some acct -> + acct + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial snark \ + coordinators must be in the genesis ledger. name of \ + Node: %s. name of Account which does not exist: %s" + node.node_name node.account_name + in + failwith failstring + in + Some + { name = node.node_name + ; public_key = + Public_key.Compressed.to_base58_check + (Public_key.compress network_kp.keypair.public_key) + ; worker_nodes = node.worker_nodes + } in - let bp_net_keypairs = List.mapi bp_keypairs ~f:mk_net_keypair in + (* NETWORK CONFIG *) { mina_automation_location = cli_inputs.mina_automation_location ; debug_arg = debug ; check_capacity = cli_inputs.check_capacity ; check_capacity_delay = cli_inputs.check_capacity_delay ; check_capacity_retries = cli_inputs.check_capacity_retries - ; block_producer_keypairs = bp_net_keypairs - ; extra_genesis_keypairs = extra_genesis_net_keypairs + ; genesis_keypairs ; constants ; terraform = { cluster_name @@ -275,14 +330,12 @@ module Network_config = struct ; mina_points_image = images.points ; mina_archive_image = images.archive_node ; runtime_config = Runtime_config.to_yojson runtime_config - ; block_producer_configs = - List.mapi bp_net_keypairs ~f:block_producer_config + ; block_producer_configs ; log_precomputed_blocks ; archive_node_count = num_archive_nodes ; mina_archive_schema ; mina_archive_schema_aux_files - ; snark_worker_replicas = num_snark_workers - ; snark_worker_public_key + ; snark_coordinator_config ; snark_worker_fee ; aws_route53_zone_id ; cpu_request = 6 @@ -340,15 +393,19 @@ module Network_manager = struct ; testnet_dir : string ; testnet_log_filter : string ; constants : Test_config.constants - ; seed_workloads : Kubernetes_network.Workload_to_deploy.t list - ; block_producer_workloads : Kubernetes_network.Workload_to_deploy.t list - ; snark_coordinator_workloads : Kubernetes_network.Workload_to_deploy.t list - ; snark_worker_workloads : Kubernetes_network.Workload_to_deploy.t list - ; archive_workloads : Kubernetes_network.Workload_to_deploy.t list - ; workloads_by_id : Kubernetes_network.Workload_to_deploy.t String.Map.t - ; mutable deployed : bool (* ; keypairs : Keypair.t list *) - ; block_producer_keypairs : Keypair.t list - ; extra_genesis_keypairs : Keypair.t list + ; seed_workloads : Kubernetes_network.Workload_to_deploy.t Core.String.Map.t + ; block_producer_workloads : + Kubernetes_network.Workload_to_deploy.t Core.String.Map.t + ; snark_coordinator_workloads : + Kubernetes_network.Workload_to_deploy.t Core.String.Map.t + ; snark_worker_workloads : + Kubernetes_network.Workload_to_deploy.t Core.String.Map.t + ; archive_workloads : + Kubernetes_network.Workload_to_deploy.t Core.String.Map.t + ; workloads_by_id : + Kubernetes_network.Workload_to_deploy.t Core.String.Map.t + ; mutable deployed : bool + ; genesis_keypairs : Network_keypair.t Core.String.Map.t } let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args @@ -410,9 +467,9 @@ module Network_manager = struct let nodes_available = max_nodes - num_kube_nodes in let cpus_needed_estimate = 6 - * ( List.length t.seed_workloads - + List.length t.block_producer_keypairs - + List.length t.snark_coordinator_workloads ) + * ( Core.Map.length t.seed_workloads + + Core.Map.length t.block_producer_workloads + + Core.Map.length t.snark_coordinator_workloads ) (* as of 2022/07, the seed, bps, and the snark coordinator use 6 cpus. this is just a rough heuristic so we're not bothering to calculate memory needed *) in let cluster_nodes_needed = @@ -488,70 +545,98 @@ module Network_manager = struct else return () in (* TODO: prebuild genesis proof and ledger *) - (* - let%bind inputs = - Genesis_ledger_helper.Genesis_proof.generate_inputs ~proof_level ~ledger - ~constraint_constants ~genesis_constants - in - let%bind (_, genesis_proof_filename) = - Genesis_ledger_helper.Genesis_proof.load_or_generate ~logger ~genesis_dir - inputs - in - *) let testnet_log_filter = Network_config.testnet_log_filter network_config in (* we currently only deploy 1 seed and coordinator per deploy (will be configurable later) *) + (* seed node keyname and workload name hardcoded as "seed" *) let seed_workloads = - [ Kubernetes_network.Workload_to_deploy.construct_workload "seed" - [ Kubernetes_network.Workload_to_deploy.cons_pod_info "mina" ] - ] - in - let snark_coordinator_id = - String.lowercase - (String.sub network_config.terraform.snark_worker_public_key - ~pos: - (String.length network_config.terraform.snark_worker_public_key - 6) - ~len:6 ) - in - let snark_coordinator_workloads = - if network_config.terraform.snark_worker_replicas > 0 then - [ Kubernetes_network.Workload_to_deploy.construct_workload - ("snark-coordinator-" ^ snark_coordinator_id) - [ Kubernetes_network.Workload_to_deploy.cons_pod_info "mina" ] - ] - else [] - in - let snark_worker_workloads = - if network_config.terraform.snark_worker_replicas > 0 then - [ Kubernetes_network.Workload_to_deploy.construct_workload - ("snark-worker-" ^ snark_coordinator_id) - (List.init network_config.terraform.snark_worker_replicas - ~f:(fun _i -> - Kubernetes_network.Workload_to_deploy.cons_pod_info "worker" ) - ) - ] - else [] + Core.String.Map.add_exn Core.String.Map.empty ~key:"seed" + ~data: + (Kubernetes_network.Workload_to_deploy.construct_workload "seed" + (Kubernetes_network.Workload_to_deploy.cons_pod_info "mina") ) in + + let snark_coordinator_workloads, snark_worker_workloads = + match network_config.terraform.snark_coordinator_config with + | Some config -> + let snark_coordinator_workloads = + if config.worker_nodes > 0 then + Core.String.Map.of_alist_exn + [ ( config.name + , Kubernetes_network.Workload_to_deploy.construct_workload + config.name + (Kubernetes_network.Workload_to_deploy.cons_pod_info + "mina" ) ) + ] + else Core.String.Map.of_alist_exn [] + in + let snark_worker_workloads = + if config.worker_nodes > 0 then + Core.String.Map.of_alist_exn + [ ( config.name ^ "-worker" + , Kubernetes_network.Workload_to_deploy.construct_workload + (config.name ^ "-worker") + (Kubernetes_network.Workload_to_deploy.cons_pod_info + "worker" ) ) + ] + else Core.String.Map.of_alist_exn [] + in + (snark_coordinator_workloads, snark_worker_workloads) + | None -> + (Core.String.Map.of_alist_exn [], Core.String.Map.of_alist_exn []) + in + (* + let snark_coordinator_id = + String.lowercase + (String.sub network_config.terraform.snark_worker_public_key + ~pos: + (String.length network_config.terraform.snark_worker_public_key - 6) + ~len:6 ) + in + let snark_coordinator_workloads = + if network_config.terraform.snark_worker_replicas > 0 then + [ Kubernetes_network.Workload_to_deploy.construct_workload + ("snark-coordinator-" ^ snark_coordinator_id) + [ Kubernetes_network.Workload_to_deploy.cons_pod_info "mina" ] + ] + else [] + in + let snark_worker_workloads = + if network_config.terraform.snark_worker_replicas > 0 then + [ Kubernetes_network.Workload_to_deploy.construct_workload + ("snark-worker-" ^ snark_coordinator_id) + (List.init network_config.terraform.snark_worker_replicas + ~f:(fun _i -> + Kubernetes_network.Workload_to_deploy.cons_pod_info "worker" ) + ) + ] + else [] + in *) let block_producer_workloads = List.map network_config.terraform.block_producer_configs ~f:(fun bp_config -> - Kubernetes_network.Workload_to_deploy.construct_workload - bp_config.name - [ Kubernetes_network.Workload_to_deploy.cons_pod_info - ~network_keypair:bp_config.keypair "mina" - ] ) + ( bp_config.name + , Kubernetes_network.Workload_to_deploy.construct_workload + bp_config.name + (Kubernetes_network.Workload_to_deploy.cons_pod_info + ~network_keypair:bp_config.keypair "mina" ) ) ) + |> Core.String.Map.of_alist_exn in let archive_workloads = List.init network_config.terraform.archive_node_count ~f:(fun i -> - Kubernetes_network.Workload_to_deploy.construct_workload - (sprintf "archive-%d" (i + 1)) - [ Kubernetes_network.Workload_to_deploy.cons_pod_info - ~has_archive_container:true "mina" - ] ) + ( sprintf "archive-%d" (i + 1) + , Kubernetes_network.Workload_to_deploy.construct_workload + (sprintf "archive-%d" (i + 1)) + (Kubernetes_network.Workload_to_deploy.cons_pod_info + ~has_archive_container:true "mina" ) ) ) + |> Core.String.Map.of_alist_exn in let workloads_by_id = let all_workloads = - seed_workloads @ snark_coordinator_workloads @ snark_worker_workloads - @ block_producer_workloads @ archive_workloads + Core.String.Map.data seed_workloads + @ Core.String.Map.data snark_coordinator_workloads + @ Core.String.Map.data snark_worker_workloads + @ Core.String.Map.data block_producer_workloads + @ Core.String.Map.data archive_workloads in all_workloads |> List.map ~f:(fun w -> (w.workload_id, w)) @@ -577,12 +662,7 @@ module Network_manager = struct ; archive_workloads ; workloads_by_id ; deployed = false - ; block_producer_keypairs = - List.map network_config.block_producer_keypairs - ~f:(fun { keypair; _ } -> keypair) - ; extra_genesis_keypairs = - List.map network_config.extra_genesis_keypairs - ~f:(fun { keypair; _ } -> keypair) + ; genesis_keypairs = network_config.genesis_keypairs } in (* check capacity *) @@ -609,6 +689,11 @@ module Network_manager = struct Network_config.to_terraform network_config |> Terraform.to_string |> Out_channel.output_string ch ) ; + [%log info] + "Writing out the genesis keys (in case you want to use them manually) to \ + testnet dir %s" + testnet_dir ; + (* TODO: write out the keypair files into the file system *) [%log info] "Initializing terraform" ; let open Malleable_error.Let_syntax in let%bind (_ : string) = run_cmd_or_hard_error t "terraform" [ "init" ] in @@ -631,53 +716,45 @@ module Network_manager = struct ; graphql_enabled = t.graphql_enabled } in + let func_for_fold ~(key : string) ~data accum_M = + let%bind mp = accum_M in + let%map node = + Kubernetes_network.Workload_to_deploy.get_nodes_from_workload data + ~config + in + Core.String.Map.add_exn mp ~key ~data:node + in let%map seeds = - Malleable_error.List.map t.seed_workloads - ~f: - (Kubernetes_network.Workload_to_deploy.get_nodes_from_workload ~config) - >>| List.concat + Core.String.Map.fold t.seed_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold and block_producers = - Malleable_error.List.map t.block_producer_workloads - ~f: - (Kubernetes_network.Workload_to_deploy.get_nodes_from_workload ~config) - >>| List.concat + Core.String.Map.fold t.block_producer_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold and snark_coordinators = - Malleable_error.List.map t.snark_coordinator_workloads - ~f: - (Kubernetes_network.Workload_to_deploy.get_nodes_from_workload ~config) - >>| List.concat + Core.String.Map.fold t.snark_coordinator_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold and snark_workers = - Malleable_error.List.map t.snark_worker_workloads - ~f: - (Kubernetes_network.Workload_to_deploy.get_nodes_from_workload ~config) - >>| List.concat + Core.String.Map.fold t.snark_worker_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold and archive_nodes = - Malleable_error.List.map t.archive_workloads - ~f: - (Kubernetes_network.Workload_to_deploy.get_nodes_from_workload ~config) - >>| List.concat - in - let all_nodes = - seeds @ block_producers @ snark_coordinators @ snark_workers - @ archive_nodes - in - let nodes_by_pod_id = - all_nodes - |> List.map ~f:(fun node -> (node.pod_id, node)) - |> String.Map.of_alist_exn + Core.String.Map.fold t.archive_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold in - let result = + let network = { Kubernetes_network.namespace = t.namespace ; constants = t.constants ; seeds ; block_producers ; snark_coordinators ; snark_workers - ; archive_nodes - ; nodes_by_pod_id + ; archive_nodes (* ; all_nodes *) ; testnet_log_filter = t.testnet_log_filter - ; block_producer_keypairs = t.block_producer_keypairs - ; extra_genesis_keypairs = t.extra_genesis_keypairs + ; genesis_keypairs = t.genesis_keypairs } in let nodes_to_string = @@ -687,11 +764,14 @@ module Network_manager = struct [%log info] "Network deployed" ; [%log info] "testnet namespace: %s" t.namespace ; [%log info] "snark coordinators: %s" - (nodes_to_string result.snark_coordinators) ; - [%log info] "snark workers: %s" (nodes_to_string result.snark_workers) ; - [%log info] "block producers: %s" (nodes_to_string result.block_producers) ; - [%log info] "archive nodes: %s" (nodes_to_string result.archive_nodes) ; - result + (nodes_to_string (Core.String.Map.data network.snark_coordinators)) ; + [%log info] "snark workers: %s" + (nodes_to_string (Core.String.Map.data network.snark_workers)) ; + [%log info] "block producers: %s" + (nodes_to_string (Core.String.Map.data network.block_producers)) ; + [%log info] "archive nodes: %s" + (nodes_to_string (Core.String.Map.data network.archive_nodes)) ; + network let destroy t = [%log' info t.logger] "Destroying network" ; diff --git a/src/lib/integration_test_cloud_engine/stack_driver_log_engine.ml b/src/lib/integration_test_cloud_engine/stack_driver_log_engine.ml index 87c4e1876ef..b6482ddb0bf 100644 --- a/src/lib/integration_test_cloud_engine/stack_driver_log_engine.ml +++ b/src/lib/integration_test_cloud_engine/stack_driver_log_engine.ml @@ -265,7 +265,7 @@ let parse_event_from_log_entry ~logger ~network log_entry = let%bind pod_id = find string log_entry [ "resource"; "labels"; "pod_name" ] in - let%bind node = + let%bind _, node = Kubernetes_network.lookup_node_by_pod_id network pod_id |> Option.value_map ~f:Or_error.return ~default: diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index b9c5ecf068c..bf9cadc162c 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -182,23 +182,19 @@ module Engine = struct val genesis_constants : t -> Genesis_constants.t - val seeds : t -> Node.t list + val seeds : t -> Node.t Core.String.Map.t - val all_non_seed_pods : t -> Node.t list + val all_non_seed_pods : t -> Node.t Core.String.Map.t - val block_producers : t -> Node.t list + val block_producers : t -> Node.t Core.String.Map.t - val snark_coordinators : t -> Node.t list + val snark_coordinators : t -> Node.t Core.String.Map.t - val archive_nodes : t -> Node.t list + val archive_nodes : t -> Node.t Core.String.Map.t - val all_nodes : t -> Node.t list + val all_nodes : t -> Node.t Core.String.Map.t - val all_keypairs : t -> Signature_lib.Keypair.t list - - val block_producer_keypairs : t -> Signature_lib.Keypair.t list - - val extra_genesis_keypairs : t -> Signature_lib.Keypair.t list + val genesis_keypairs : t -> Network_keypair.t Core.String.Map.t val initialize_infra : logger:Logger.t -> t -> unit Malleable_error.t end diff --git a/src/lib/integration_test_lib/network_keypair.ml b/src/lib/integration_test_lib/network_keypair.ml index 7ddc3724990..4a46f7e432a 100644 --- a/src/lib/integration_test_lib/network_keypair.ml +++ b/src/lib/integration_test_lib/network_keypair.ml @@ -3,25 +3,26 @@ open Core_kernel type t = { keypair : Keypair.t - ; secret_name : string - ; public_key_file : string - ; private_key_file : string + ; keypair_name : string + ; privkey_password : string + ; public_key : string + ; private_key : string } [@@deriving to_yojson] -let create_network_keypair ~keypair ~secret_name = +let create_network_keypair ~keypair_name ~keypair = let open Keypair in - let public_key_file = + let privkey_password = "naughty blue worm" in + let public_key = Public_key.Compressed.to_base58_check (Public_key.compress keypair.public_key) - ^ "\n" in - let private_key_file = + let private_key = let plaintext = Bigstring.to_bytes (Private_key.to_bigstring keypair.private_key) in - let password = Bytes.of_string "naughty blue worm" in + let password = Bytes.of_string privkey_password in Secrets.Secret_box.encrypt ~plaintext ~password |> Secrets.Secret_box.to_yojson |> Yojson.Safe.to_string in - { keypair; secret_name; public_key_file; private_key_file } + { keypair; keypair_name; privkey_password; public_key; private_key } diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index b6842dbbb13..f7dc1dec79e 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -8,8 +8,21 @@ module Container_images = struct } end -module Wallet = struct - type t = { balance : string; timing : Mina_base.Account_timing.t } +module Test_Account = struct + type t = + { account_name : string + ; balance : string + ; timing : Mina_base.Account_timing.t + } +end + +module Block_producer_node = struct + type t = { node_name : string; account_name : string } +end + +module Snark_coordinator_node = struct + type t = { node_name : string; account_name : string; worker_nodes : int } + [@@deriving to_yojson] end type constants = @@ -19,21 +32,23 @@ type constants = [@@deriving to_yojson] type t = - { (* temporary flag to enable/disable graphql ingress deployments *) - requires_graphql : bool + { requires_graphql : bool + (* temporary flag to enable/disable graphql ingress deployments *) + (* testnet topography *) + ; genesis_ledger : Test_Account.t list + ; block_producers : Block_producer_node.t list + ; snark_coordinator : Snark_coordinator_node.t option + ; snark_worker_fee : string + ; num_archive_nodes : int + ; log_precomputed_blocks : bool + (* ; num_plain_nodes : int *) + (* blockchain constants *) + ; proof_config : Runtime_config.Proof_keys.t ; k : int ; delta : int ; slots_per_epoch : int ; slots_per_sub_window : int ; txpool_max_size : int - ; block_producers : Wallet.t list - ; extra_genesis_accounts : Wallet.t list - ; num_snark_workers : int - ; num_archive_nodes : int - ; log_precomputed_blocks : bool - ; snark_worker_fee : string - ; snark_worker_public_key : string - ; proof_config : Runtime_config.Proof_keys.t } let proof_config_default : Runtime_config.Proof_keys.t = @@ -51,19 +66,16 @@ let proof_config_default : Runtime_config.Proof_keys.t = let default = { requires_graphql = false + ; genesis_ledger = [] + ; block_producers = [] + ; snark_coordinator = None + ; snark_worker_fee = "0.025" + ; num_archive_nodes = 0 + ; log_precomputed_blocks = false (* ; num_plain_nodes = 0 *) + ; proof_config = proof_config_default ; k = 20 ; slots_per_epoch = 3 * 8 * 20 ; slots_per_sub_window = 2 ; delta = 0 ; txpool_max_size = 3000 - ; block_producers = [] - ; extra_genesis_accounts = [] - ; num_snark_workers = 0 - ; num_archive_nodes = 0 - ; log_precomputed_blocks = false - ; snark_worker_fee = "0.025" - ; snark_worker_public_key = - (let pk, _ = (Lazy.force Mina_base.Sample_keypairs.keypairs).(0) in - Signature_lib.Public_key.Compressed.to_string pk ) - ; proof_config = proof_config_default }