From 2d4b113ba5a88fdeae52a6b7d0f9bdc03b41c397 Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 28 Jun 2020 21:28:10 -0400 Subject: [PATCH 001/107] gem bumps --- modules/Gemfile.lock | 858 ++++++++++++++++++++++--------------------- 1 file changed, 431 insertions(+), 427 deletions(-) diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index b4af0b84d..32b3a106f 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -44,7 +44,7 @@ PATH GEM remote: https://rubygems.org/ specs: - activesupport (6.0.3) + activesupport (6.0.3.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -52,13 +52,13 @@ GEM zeitwerk (~> 2.2, >= 2.2.2) addressable (2.5.2) public_suffix (>= 2.0.2, < 4.0) - ast (2.4.0) + ast (2.4.1) aws-eventstream (1.1.0) - aws-sdk-core (2.11.504) + aws-sdk-core (2.11.537) aws-sigv4 (~> 1.0) jmespath (~> 1.0) - aws-sigv4 (1.1.3) - aws-eventstream (~> 1.0, >= 1.0.2) + aws-sigv4 (1.2.1) + aws-eventstream (~> 1, >= 1.0.2) azure-core (0.1.15) faraday (~> 0.9) faraday_middleware (~> 0.10) @@ -68,413 +68,413 @@ GEM faraday (~> 0.9) faraday_middleware (~> 0.10) nokogiri (~> 1.6, >= 1.6.8) - azure_cognitiveservices_anomalydetector (0.17.0) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_autosuggest (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_computervision (0.20.1) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_contentmoderator (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_customimagesearch (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_customsearch (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_customvisionprediction (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_customvisiontraining (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_entitysearch (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_face (0.19.0) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_formrecognizer (0.17.1) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_imagesearch (0.18.2) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_localsearch (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_luisauthoring (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_luisruntime (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_newssearch (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_personalizer (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_qnamaker (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_qnamakerruntime (0.17.1) - ms_rest_azure (~> 0.11.1) - azure_cognitiveservices_spellcheck (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_textanalytics (0.17.3) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_videosearch (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_visualsearch (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_cognitiveservices_websearch (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_event_grid (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_graph_rbac (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_key_vault (0.17.3) - ms_rest_azure (~> 0.11.0) - azure_mgmt_adhybridhealth_service (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_advisor (0.17.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_alerts_management (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_analysis_services (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_api_management (0.19.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_appconfiguration (0.17.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_attestation (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_authorization (0.18.4) - ms_rest_azure (~> 0.11.0) - azure_mgmt_automation (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_azurestack (0.17.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_batch (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_batchai (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_billing (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_bot_service (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_cdn (0.17.3) - ms_rest_azure (~> 0.11.0) - azure_mgmt_cognitive_services (0.19.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_commerce (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_compute (0.19.2) - ms_rest_azure (~> 0.11.1) - azure_mgmt_consumption (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_container_instance (0.17.4) - ms_rest_azure (~> 0.11.0) - azure_mgmt_container_registry (0.18.3) - ms_rest_azure (~> 0.11.1) - azure_mgmt_container_service (0.20.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_cosmosdb (0.21.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_cost_management (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_customer_insights (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_data_factory (0.18.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_data_migration (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_databox (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_datalake_analytics (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_datalake_store (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_datashare (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_deployment_manager (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_dev_spaces (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_devtestlabs (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_dns (0.17.4) - ms_rest_azure (~> 0.11.0) - azure_mgmt_edgegateway (0.18.0) - ms_rest_azure (~> 0.11.0) - azure_mgmt_event_grid (0.19.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_event_hub (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_features (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_hanaonazure (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_hdinsight (0.17.7) - ms_rest_azure (~> 0.11.1) - azure_mgmt_import_export (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_iot_central (0.19.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_iot_hub (0.17.3) - ms_rest_azure (~> 0.11.1) - azure_mgmt_key_vault (0.17.5) - ms_rest_azure (~> 0.11.1) - azure_mgmt_kubernetes_configuration (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_kusto (0.19.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_labservices (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_links (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_locks (0.17.3) - ms_rest_azure (~> 0.11.0) - azure_mgmt_logic (0.18.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_machine_learning (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_machine_learning_services (0.17.2) - ms_rest_azure (~> 0.11.1) - azure_mgmt_maintenance (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_managed_applications (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_mariadb (0.17.2) - ms_rest_azure (~> 0.11.1) - azure_mgmt_marketplace_ordering (0.17.4) - ms_rest_azure (~> 0.11.0) - azure_mgmt_media_services (0.20.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_migrate (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_mixedreality (0.17.2) - ms_rest_azure (~> 0.11.1) - azure_mgmt_monitor (0.17.5) - ms_rest_azure (~> 0.11.1) - azure_mgmt_msi (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_mysql (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_netapp (0.19.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_network (0.23.2) - ms_rest_azure (~> 0.11.1) - azure_mgmt_notification_hubs (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_operational_insights (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_operations_management (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_peering (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_policy (0.17.8) - ms_rest_azure (~> 0.11.1) - azure_mgmt_policy_insights (0.17.6) - ms_rest_azure (~> 0.11.1) - azure_mgmt_portal (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_postgresql (0.17.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_powerbi_dedicated (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_powerbi_embedded (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_privatedns (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_recovery_services (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_recovery_services_backup (0.18.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_recovery_services_site_recovery (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_redis (0.17.3) - ms_rest_azure (~> 0.11.0) - azure_mgmt_relay (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_reservations (0.19.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_resource_health (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_resourcegraph (0.17.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_resources (0.17.8) - ms_rest_azure (~> 0.11.1) - azure_mgmt_resources_management (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_scheduler (0.17.1) - ms_rest_azure (~> 0.11.0) - azure_mgmt_search (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_security (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_serialconsole (0.17.0) - ms_rest_azure (~> 0.11.0) - azure_mgmt_service_bus (0.17.3) - ms_rest_azure (~> 0.11.0) - azure_mgmt_service_fabric (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_signalr (0.17.4) - ms_rest_azure (~> 0.11.1) - azure_mgmt_sql (0.19.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_sqlvirtualmachine (0.18.1) - ms_rest_azure (~> 0.11.1) - azure_mgmt_stor_simple8000_series (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_storage (0.21.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_storagecache (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_storagesync (0.18.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_stream_analytics (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_subscriptions (0.18.2) - ms_rest_azure (~> 0.11.1) - azure_mgmt_support (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_synapse (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_time_series_insights (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_traffic_manager (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_vmware_cloudsimple (0.17.0) - ms_rest_azure (~> 0.11.1) - azure_mgmt_web (0.17.5) - ms_rest_azure (~> 0.11.1) - azure_sdk (0.56.0) + azure_cognitiveservices_anomalydetector (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_autosuggest (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_computervision (0.20.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_contentmoderator (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_customimagesearch (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_customsearch (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_customvisionprediction (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_customvisiontraining (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_entitysearch (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_face (0.19.1) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_formrecognizer (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_imagesearch (0.18.3) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_localsearch (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_luisauthoring (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_luisruntime (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_newssearch (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_personalizer (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_qnamaker (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_qnamakerruntime (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_spellcheck (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_textanalytics (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_videosearch (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_visualsearch (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_cognitiveservices_websearch (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_event_grid (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_graph_rbac (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_key_vault (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_adhybridhealth_service (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_advisor (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_alerts_management (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_analysis_services (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_api_management (0.19.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_appconfiguration (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_attestation (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_authorization (0.18.5) + ms_rest_azure (~> 0.12.0) + azure_mgmt_automation (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_azurestack (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_batch (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_batchai (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_billing (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_bot_service (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_cdn (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_cognitive_services (0.19.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_commerce (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_compute (0.19.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_consumption (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_container_instance (0.17.5) + ms_rest_azure (~> 0.12.0) + azure_mgmt_container_registry (0.18.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_container_service (0.20.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_cosmosdb (0.21.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_cost_management (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_customer_insights (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_data_factory (0.18.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_data_migration (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_databox (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_datalake_analytics (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_datalake_store (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_datashare (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_deployment_manager (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_dev_spaces (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_devtestlabs (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_dns (0.17.5) + ms_rest_azure (~> 0.12.0) + azure_mgmt_edgegateway (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_event_grid (0.20.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_event_hub (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_features (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_hanaonazure (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_hdinsight (0.18.0) + ms_rest_azure (~> 0.12.0) + azure_mgmt_import_export (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_iot_central (0.19.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_iot_hub (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_key_vault (0.17.6) + ms_rest_azure (~> 0.12.0) + azure_mgmt_kubernetes_configuration (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_kusto (0.19.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_labservices (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_links (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_locks (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_logic (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_machine_learning (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_machine_learning_services (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_maintenance (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_managed_applications (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_mariadb (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_marketplace_ordering (0.17.5) + ms_rest_azure (~> 0.12.0) + azure_mgmt_media_services (0.20.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_migrate (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_mixedreality (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_monitor (0.17.6) + ms_rest_azure (~> 0.12.0) + azure_mgmt_msi (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_mysql (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_netapp (0.20.0) + ms_rest_azure (~> 0.12.0) + azure_mgmt_network (0.23.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_notification_hubs (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_operational_insights (0.18.0) + ms_rest_azure (~> 0.12.0) + azure_mgmt_operations_management (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_peering (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_policy (0.17.9) + ms_rest_azure (~> 0.12.0) + azure_mgmt_policy_insights (0.17.7) + ms_rest_azure (~> 0.12.0) + azure_mgmt_portal (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_postgresql (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_powerbi_dedicated (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_powerbi_embedded (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_privatedns (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_recovery_services (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_recovery_services_backup (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_recovery_services_site_recovery (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_redis (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_relay (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_reservations (0.19.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_resource_health (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_resourcegraph (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_resources (0.17.9) + ms_rest_azure (~> 0.12.0) + azure_mgmt_resources_management (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_scheduler (0.17.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_search (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_security (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_serialconsole (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_service_bus (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_service_fabric (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_signalr (0.17.5) + ms_rest_azure (~> 0.12.0) + azure_mgmt_sql (0.19.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_sqlvirtualmachine (0.18.2) + ms_rest_azure (~> 0.12.0) + azure_mgmt_stor_simple8000_series (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_storage (0.21.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_storagecache (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_storagesync (0.18.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_stream_analytics (0.17.3) + ms_rest_azure (~> 0.12.0) + azure_mgmt_subscriptions (0.18.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_support (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_synapse (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_time_series_insights (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_traffic_manager (0.17.4) + ms_rest_azure (~> 0.12.0) + azure_mgmt_vmware_cloudsimple (0.17.1) + ms_rest_azure (~> 0.12.0) + azure_mgmt_web (0.17.6) + ms_rest_azure (~> 0.12.0) + azure_sdk (0.59.0) azure-storage (~> 0.14.0.preview) - azure_cognitiveservices_anomalydetector (~> 0.17.0) - azure_cognitiveservices_autosuggest (~> 0.17.1) - azure_cognitiveservices_computervision (~> 0.20.1) - azure_cognitiveservices_contentmoderator (~> 0.17.2) - azure_cognitiveservices_customimagesearch (~> 0.17.1) - azure_cognitiveservices_customsearch (~> 0.18.1) - azure_cognitiveservices_customvisionprediction (~> 0.17.2) - azure_cognitiveservices_customvisiontraining (~> 0.17.2) - azure_cognitiveservices_entitysearch (~> 0.18.1) - azure_cognitiveservices_face (~> 0.19.0) - azure_cognitiveservices_formrecognizer (~> 0.17.1) - azure_cognitiveservices_imagesearch (~> 0.18.2) - azure_cognitiveservices_localsearch (~> 0.17.1) - azure_cognitiveservices_luisauthoring (~> 0.18.0) - azure_cognitiveservices_luisruntime (~> 0.18.0) - azure_cognitiveservices_newssearch (~> 0.18.1) - azure_cognitiveservices_personalizer (~> 0.17.0) - azure_cognitiveservices_qnamaker (~> 0.18.0) - azure_cognitiveservices_qnamakerruntime (~> 0.17.1) - azure_cognitiveservices_spellcheck (~> 0.18.1) - azure_cognitiveservices_textanalytics (~> 0.17.3) - azure_cognitiveservices_videosearch (~> 0.18.1) - azure_cognitiveservices_visualsearch (~> 0.18.1) - azure_cognitiveservices_websearch (~> 0.18.1) - azure_event_grid (~> 0.18.0) - azure_graph_rbac (~> 0.17.1) - azure_key_vault (~> 0.17.3) - azure_mgmt_adhybridhealth_service (~> 0.17.0) - azure_mgmt_advisor (~> 0.17.1) - azure_mgmt_alerts_management (~> 0.17.0) - azure_mgmt_analysis_services (~> 0.17.2) - azure_mgmt_api_management (~> 0.19.0) - azure_mgmt_appconfiguration (~> 0.17.1) - azure_mgmt_attestation (~> 0.17.0) - azure_mgmt_authorization (~> 0.18.4) - azure_mgmt_automation (~> 0.17.2) - azure_mgmt_azurestack (~> 0.17.1) - azure_mgmt_batch (~> 0.18.0) - azure_mgmt_batchai (~> 0.17.0) - azure_mgmt_billing (~> 0.17.2) - azure_mgmt_bot_service (~> 0.17.0) - azure_mgmt_cdn (~> 0.17.3) - azure_mgmt_cognitive_services (~> 0.19.1) - azure_mgmt_commerce (~> 0.17.1) - azure_mgmt_compute (~> 0.19.2) - azure_mgmt_consumption (~> 0.18.0) - azure_mgmt_container_instance (~> 0.17.4) - azure_mgmt_container_registry (~> 0.18.3) - azure_mgmt_container_service (~> 0.20.1) - azure_mgmt_cosmosdb (~> 0.21.0) - azure_mgmt_cost_management (~> 0.17.0) - azure_mgmt_customer_insights (~> 0.17.2) - azure_mgmt_data_factory (~> 0.18.1) - azure_mgmt_data_migration (~> 0.18.0) - azure_mgmt_databox (~> 0.17.0) - azure_mgmt_datalake_analytics (~> 0.17.2) - azure_mgmt_datalake_store (~> 0.17.2) - azure_mgmt_datashare (~> 0.17.0) - azure_mgmt_deployment_manager (~> 0.17.0) - azure_mgmt_dev_spaces (~> 0.17.2) - azure_mgmt_devtestlabs (~> 0.18.0) - azure_mgmt_dns (~> 0.17.4) - azure_mgmt_edgegateway (~> 0.18.0) - azure_mgmt_event_grid (~> 0.19.0) - azure_mgmt_event_hub (~> 0.18.0) - azure_mgmt_features (~> 0.17.2) - azure_mgmt_hanaonazure (~> 0.18.0) - azure_mgmt_hdinsight (~> 0.17.7) - azure_mgmt_import_export (~> 0.17.0) - azure_mgmt_iot_central (~> 0.19.0) - azure_mgmt_iot_hub (~> 0.17.3) - azure_mgmt_key_vault (~> 0.17.5) - azure_mgmt_kubernetes_configuration (~> 0.17.0) - azure_mgmt_kusto (~> 0.19.1) - azure_mgmt_labservices (~> 0.17.1) - azure_mgmt_links (~> 0.17.2) - azure_mgmt_locks (~> 0.17.3) - azure_mgmt_logic (~> 0.18.1) - azure_mgmt_machine_learning (~> 0.17.2) - azure_mgmt_machine_learning_services (~> 0.17.2) - azure_mgmt_maintenance (~> 0.17.0) - azure_mgmt_managed_applications (~> 0.17.2) - azure_mgmt_mariadb (~> 0.17.1) - azure_mgmt_marketplace_ordering (~> 0.17.4) - azure_mgmt_media_services (~> 0.20.0) - azure_mgmt_migrate (~> 0.17.0) - azure_mgmt_mixedreality (~> 0.17.2) - azure_mgmt_monitor (~> 0.17.5) - azure_mgmt_msi (~> 0.17.1) - azure_mgmt_mysql (~> 0.17.0) - azure_mgmt_netapp (~> 0.19.0) - azure_mgmt_network (~> 0.23.2) - azure_mgmt_notification_hubs (~> 0.17.2) - azure_mgmt_operational_insights (~> 0.17.2) - azure_mgmt_operations_management (~> 0.17.0) - azure_mgmt_peering (~> 0.17.0) - azure_mgmt_policy (~> 0.17.8) - azure_mgmt_policy_insights (~> 0.17.6) - azure_mgmt_portal (~> 0.17.0) - azure_mgmt_postgresql (~> 0.17.1) - azure_mgmt_powerbi_dedicated (~> 0.17.0) - azure_mgmt_powerbi_embedded (~> 0.17.1) - azure_mgmt_privatedns (~> 0.17.1) - azure_mgmt_recovery_services (~> 0.18.0) - azure_mgmt_recovery_services_backup (~> 0.18.1) - azure_mgmt_recovery_services_site_recovery (~> 0.17.2) - azure_mgmt_redis (~> 0.17.3) - azure_mgmt_relay (~> 0.17.2) - azure_mgmt_reservations (~> 0.19.1) - azure_mgmt_resource_health (~> 0.17.0) - azure_mgmt_resourcegraph (~> 0.17.1) - azure_mgmt_resources (~> 0.17.8) - azure_mgmt_resources_management (~> 0.17.1) - azure_mgmt_scheduler (~> 0.17.1) - azure_mgmt_search (~> 0.17.2) - azure_mgmt_security (~> 0.18.0) - azure_mgmt_serialconsole (~> 0.17.0) - azure_mgmt_service_bus (~> 0.17.3) - azure_mgmt_service_fabric (~> 0.17.2) - azure_mgmt_signalr (~> 0.17.4) - azure_mgmt_sql (~> 0.19.0) - azure_mgmt_sqlvirtualmachine (~> 0.18.1) - azure_mgmt_stor_simple8000_series (~> 0.17.2) - azure_mgmt_storage (~> 0.21.0) - azure_mgmt_storagecache (~> 0.18.0) - azure_mgmt_storagesync (~> 0.18.0) - azure_mgmt_stream_analytics (~> 0.17.2) - azure_mgmt_subscriptions (~> 0.18.2) - azure_mgmt_support (~> 0.17.0) - azure_mgmt_synapse (~> 0.17.0) - azure_mgmt_time_series_insights (~> 0.17.0) - azure_mgmt_traffic_manager (~> 0.17.2) - azure_mgmt_vmware_cloudsimple (~> 0.17.0) - azure_mgmt_web (~> 0.17.5) - azure_service_fabric (~> 0.18.0) - azure_service_fabric (0.18.0) - ms_rest_azure (~> 0.11.1) + azure_cognitiveservices_anomalydetector (~> 0.17.1) + azure_cognitiveservices_autosuggest (~> 0.17.2) + azure_cognitiveservices_computervision (~> 0.20.2) + azure_cognitiveservices_contentmoderator (~> 0.17.3) + azure_cognitiveservices_customimagesearch (~> 0.17.2) + azure_cognitiveservices_customsearch (~> 0.18.2) + azure_cognitiveservices_customvisionprediction (~> 0.17.3) + azure_cognitiveservices_customvisiontraining (~> 0.17.3) + azure_cognitiveservices_entitysearch (~> 0.18.2) + azure_cognitiveservices_face (~> 0.19.1) + azure_cognitiveservices_formrecognizer (~> 0.17.2) + azure_cognitiveservices_imagesearch (~> 0.18.3) + azure_cognitiveservices_localsearch (~> 0.17.2) + azure_cognitiveservices_luisauthoring (~> 0.18.2) + azure_cognitiveservices_luisruntime (~> 0.18.1) + azure_cognitiveservices_newssearch (~> 0.18.2) + azure_cognitiveservices_personalizer (~> 0.17.1) + azure_cognitiveservices_qnamaker (~> 0.18.1) + azure_cognitiveservices_qnamakerruntime (~> 0.17.2) + azure_cognitiveservices_spellcheck (~> 0.18.2) + azure_cognitiveservices_textanalytics (~> 0.17.4) + azure_cognitiveservices_videosearch (~> 0.18.2) + azure_cognitiveservices_visualsearch (~> 0.18.2) + azure_cognitiveservices_websearch (~> 0.18.2) + azure_event_grid (~> 0.18.1) + azure_graph_rbac (~> 0.17.2) + azure_key_vault (~> 0.17.4) + azure_mgmt_adhybridhealth_service (~> 0.17.1) + azure_mgmt_advisor (~> 0.17.2) + azure_mgmt_alerts_management (~> 0.17.1) + azure_mgmt_analysis_services (~> 0.17.3) + azure_mgmt_api_management (~> 0.19.1) + azure_mgmt_appconfiguration (~> 0.17.2) + azure_mgmt_attestation (~> 0.17.1) + azure_mgmt_authorization (~> 0.18.5) + azure_mgmt_automation (~> 0.17.3) + azure_mgmt_azurestack (~> 0.17.2) + azure_mgmt_batch (~> 0.18.1) + azure_mgmt_batchai (~> 0.17.1) + azure_mgmt_billing (~> 0.17.3) + azure_mgmt_bot_service (~> 0.17.1) + azure_mgmt_cdn (~> 0.17.4) + azure_mgmt_cognitive_services (~> 0.19.2) + azure_mgmt_commerce (~> 0.17.2) + azure_mgmt_compute (~> 0.19.3) + azure_mgmt_consumption (~> 0.18.1) + azure_mgmt_container_instance (~> 0.17.5) + azure_mgmt_container_registry (~> 0.18.4) + azure_mgmt_container_service (~> 0.20.2) + azure_mgmt_cosmosdb (~> 0.21.2) + azure_mgmt_cost_management (~> 0.17.1) + azure_mgmt_customer_insights (~> 0.17.3) + azure_mgmt_data_factory (~> 0.18.3) + azure_mgmt_data_migration (~> 0.18.1) + azure_mgmt_databox (~> 0.17.1) + azure_mgmt_datalake_analytics (~> 0.17.3) + azure_mgmt_datalake_store (~> 0.17.3) + azure_mgmt_datashare (~> 0.17.2) + azure_mgmt_deployment_manager (~> 0.17.1) + azure_mgmt_dev_spaces (~> 0.17.3) + azure_mgmt_devtestlabs (~> 0.18.1) + azure_mgmt_dns (~> 0.17.5) + azure_mgmt_edgegateway (~> 0.18.1) + azure_mgmt_event_grid (~> 0.20.2) + azure_mgmt_event_hub (~> 0.18.1) + azure_mgmt_features (~> 0.17.4) + azure_mgmt_hanaonazure (~> 0.18.1) + azure_mgmt_hdinsight (~> 0.18.0) + azure_mgmt_import_export (~> 0.17.1) + azure_mgmt_iot_central (~> 0.19.1) + azure_mgmt_iot_hub (~> 0.17.4) + azure_mgmt_key_vault (~> 0.17.6) + azure_mgmt_kubernetes_configuration (~> 0.17.1) + azure_mgmt_kusto (~> 0.19.2) + azure_mgmt_labservices (~> 0.17.2) + azure_mgmt_links (~> 0.17.3) + azure_mgmt_locks (~> 0.17.4) + azure_mgmt_logic (~> 0.18.2) + azure_mgmt_machine_learning (~> 0.17.3) + azure_mgmt_machine_learning_services (~> 0.17.3) + azure_mgmt_maintenance (~> 0.17.1) + azure_mgmt_managed_applications (~> 0.17.3) + azure_mgmt_mariadb (~> 0.17.3) + azure_mgmt_marketplace_ordering (~> 0.17.5) + azure_mgmt_media_services (~> 0.20.1) + azure_mgmt_migrate (~> 0.17.1) + azure_mgmt_mixedreality (~> 0.17.3) + azure_mgmt_monitor (~> 0.17.6) + azure_mgmt_msi (~> 0.17.2) + azure_mgmt_mysql (~> 0.17.1) + azure_mgmt_netapp (~> 0.20.0) + azure_mgmt_network (~> 0.23.4) + azure_mgmt_notification_hubs (~> 0.17.3) + azure_mgmt_operational_insights (~> 0.18.0) + azure_mgmt_operations_management (~> 0.17.1) + azure_mgmt_peering (~> 0.17.1) + azure_mgmt_policy (~> 0.17.9) + azure_mgmt_policy_insights (~> 0.17.7) + azure_mgmt_portal (~> 0.17.1) + azure_mgmt_postgresql (~> 0.17.2) + azure_mgmt_powerbi_dedicated (~> 0.17.1) + azure_mgmt_powerbi_embedded (~> 0.17.2) + azure_mgmt_privatedns (~> 0.17.2) + azure_mgmt_recovery_services (~> 0.18.1) + azure_mgmt_recovery_services_backup (~> 0.18.2) + azure_mgmt_recovery_services_site_recovery (~> 0.17.3) + azure_mgmt_redis (~> 0.17.4) + azure_mgmt_relay (~> 0.17.3) + azure_mgmt_reservations (~> 0.19.2) + azure_mgmt_resource_health (~> 0.17.1) + azure_mgmt_resourcegraph (~> 0.17.2) + azure_mgmt_resources (~> 0.17.9) + azure_mgmt_resources_management (~> 0.17.2) + azure_mgmt_scheduler (~> 0.17.2) + azure_mgmt_search (~> 0.17.3) + azure_mgmt_security (~> 0.18.2) + azure_mgmt_serialconsole (~> 0.17.1) + azure_mgmt_service_bus (~> 0.17.4) + azure_mgmt_service_fabric (~> 0.17.3) + azure_mgmt_signalr (~> 0.17.5) + azure_mgmt_sql (~> 0.19.1) + azure_mgmt_sqlvirtualmachine (~> 0.18.2) + azure_mgmt_stor_simple8000_series (~> 0.17.3) + azure_mgmt_storage (~> 0.21.1) + azure_mgmt_storagecache (~> 0.18.1) + azure_mgmt_storagesync (~> 0.18.1) + azure_mgmt_stream_analytics (~> 0.17.3) + azure_mgmt_subscriptions (~> 0.18.4) + azure_mgmt_support (~> 0.17.1) + azure_mgmt_synapse (~> 0.17.1) + azure_mgmt_time_series_insights (~> 0.17.1) + azure_mgmt_traffic_manager (~> 0.17.4) + azure_mgmt_vmware_cloudsimple (~> 0.17.1) + azure_mgmt_web (~> 0.17.6) + azure_service_fabric (~> 0.18.1) + azure_service_fabric (0.18.1) + ms_rest_azure (~> 0.12.0) berkshelf (7.0.10) chef (>= 13.6.52) chef-config @@ -572,13 +572,13 @@ GEM cucumber-tag-expressions (~> 2.0, >= 2.0.4) cucumber-gherkin (13.0.0) cucumber-messages (~> 12.0, >= 12.0.0) - cucumber-messages (12.1.1) + cucumber-messages (12.2.0) protobuf-cucumber (~> 3.10, >= 3.10.8) cucumber-tag-expressions (2.0.4) daemons (1.3.1) declarative (0.0.10) declarative-option (0.1.0) - diff-lcs (1.3) + diff-lcs (1.4.2) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) erubi (1.9.0) @@ -591,8 +591,8 @@ GEM http-cookie (~> 1.0.0) faraday_middleware (0.14.0) faraday (>= 0.7.4, < 1.0) - ffi (1.12.2) - ffi-libarchive (1.0.0) + ffi (1.13.1) + ffi-libarchive (1.0.3) ffi (~> 1.0) ffi-yajl (2.3.3) libyajl2 (~> 1.2) @@ -613,7 +613,7 @@ GEM representable (~> 3.0) retriable (>= 2.0, < 4.0) signet (~> 0.12) - googleauth (0.12.0) + googleauth (0.13.0) faraday (>= 0.17.3, < 2.0) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -629,7 +629,7 @@ GEM http-cookie (1.0.3) domain_name (~> 0.5) httpclient (2.8.3) - i18n (1.8.2) + i18n (1.8.3) concurrent-ruby (~> 1.0) inifile (3.0.0) iniparse (1.5.0) @@ -648,7 +648,7 @@ GEM mini_mime (1.0.2) mini_portile2 (2.4.0) minitar (0.9) - minitest (5.14.0) + minitest (5.14.1) mixlib-archive (1.0.5) mixlib-log mixlib-authentication (2.1.1) @@ -667,12 +667,11 @@ GEM concurrent-ruby (~> 1.0) faraday (>= 0.9, < 2.0.0) timeliness (~> 0.3.10) - ms_rest_azure (0.11.2) + ms_rest_azure (0.12.0) concurrent-ruby (~> 1.0) faraday (>= 0.9, < 2.0.0) faraday-cookie_jar (~> 0.0.6) ms_rest (~> 0.7.6) - unf_ext (= 0.0.7.2) multi_json (1.14.1) multipart-post (2.1.1) mysql2 (0.5.2) @@ -712,9 +711,9 @@ GEM optimist (3.0.1) os (1.1.0) paint (1.0.1) - parallel (1.19.1) - parser (2.7.1.2) - ast (~> 2.4.0) + parallel (1.19.2) + parser (2.7.1.4) + ast (~> 2.4.1) plist (3.5.0) polyglot (0.3.5) protobuf-cucumber (3.10.8) @@ -724,9 +723,10 @@ GEM thread_safe proxifier (1.0.3) public_suffix (3.1.1) - rack (2.2.2) + rack (2.2.3) rainbow (3.0.0) rake (13.0.1) + regexp_parser (1.7.1) representable (3.0.4) declarative (< 0.1.0) declarative-option (< 0.2.0) @@ -753,13 +753,17 @@ GEM rspec_junit_formatter (0.2.3) builder (< 4) rspec-core (>= 2, < 4, != 2.12.0) - rubocop (0.83.0) + rubocop (0.86.0) parallel (~> 1.10) parser (>= 2.7.0.1) rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.7) rexml + rubocop-ast (>= 0.0.3, < 1.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) + rubocop-ast (0.1.0) + parser (>= 2.7.0.1) ruby-graphviz (1.2.5) rexml ruby-progressbar (1.10.1) @@ -787,7 +791,7 @@ GEM solve (4.0.3) molinillo (~> 0.6) semverse (>= 1.1, < 4.0) - specinfra (2.82.16) + specinfra (2.82.17) net-scp net-ssh (>= 2.7) net-telnet (= 0.1.1) @@ -809,7 +813,7 @@ GEM uber (0.1.0) unf (0.1.4) unf_ext - unf_ext (0.0.7.2) + unf_ext (0.0.7.7) unicode-display_width (1.7.0) uuidtools (2.1.5) winrm (2.3.4) From f9527e7e0184d8f29b77ddc85afb054e39823d15 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 29 Jun 2020 03:21:56 -0400 Subject: [PATCH 002/107] Bucket: add an upload stanza to schema; AWS::Bucket: implement that stanza --- modules/mu/config/bucket.rb | 17 +++++++++++++++++ modules/mu/providers/aws/bucket.rb | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/modules/mu/config/bucket.rb b/modules/mu/config/bucket.rb index 4ebed7d30..21eff4a93 100644 --- a/modules/mu/config/bucket.rb +++ b/modules/mu/config/bucket.rb @@ -50,6 +50,23 @@ def self.schema "default" => "index.html", "description" => "If +web_enabled+, return this object when \"diretory\" (a path not ending in a key/object) is invoked." }, + "upload" => { + "type" => "array", + "items" => { + "type" => "object", + "required" => ["source", "destination"], + "properties" => { + "source" => { + "type" => "string", + "description" => "A file or directory to upload. If a directory is specified, it will be recursively mirrored to the bucket +destination+." + }, + "destination" => { + "type" => "string", + "description" => "Path to which +source+ file(s) will be uploaded in the bucket, relative to +/+" + } + } + } + }, "policies" => { "type" => "array", "items" => MU::Config::Role.policy_primitive(subobjects: true, grant_to: true, permissions_optional: true, targets_optional: true) diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index d0affb698..b905d5806 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -146,6 +146,27 @@ def groom } ) end + + if @config['upload'] + @config['upload'].each { |batch| + urlbase = "s3://"+@cloud_id+batch['destination'] + urlbase += "/" if urlbase !~ /\/$/ + upload_me = if File.directory?(batch['source']) + Dir[batch['source']+'/**/*'].reject {|d| + File.directory?(d) + }.map { |f| + [ f, urlbase+f.sub(/^#{Regexp.quote(batch['source'])}\/?/, '') ] + } + else + batch['source'].match(/([^\/]+)$/) + [ [batch['source'], urlbase+Regexp.last_match[1]] ] + end + + Hash[upload_me].each_pair { |file, url| + self.class.upload(url, file: file, credentials: @credentials, region: @config['region']) + } + } + end end # Upload a file to a bucket. @@ -340,6 +361,7 @@ def self.validateConfig(bucket, _configurator) } end + ok end From 16b4dd2147b524ccbad4207c6fb7d51169cbec51 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 30 Jun 2020 01:00:37 -0400 Subject: [PATCH 003/107] AWS::Bucket.cleanup: handle non-empty buckets in a sensible fashion --- modules/mu/providers/aws/bucket.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index b905d5806..be05d6053 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -275,6 +275,21 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name) end end + rescue Aws::S3::Errors::BucketNotEmpty => e + if flags["skipsnapshots"] + del = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_objects(bucket: bucket.name).contents.map { |o| { key: o.key } } + del.concat(MU::Cloud::AWS.s3(credentials: credentials, region: region).list_object_versions(bucket: bucket.name).versions.map { |o| { key: o.key, version_id: o.version_id } }) + + MU.log "Purging #{del.size.to_s} objects and versions from #{bucket.name}" + begin + batch = del.slice!(0, (del.length >= 1000 ? 1000 : del.length)) + MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_objects(bucket: bucket.name, delete: { objects: batch } ) if !noop + end while del.size > 0 + + retry if !noop + else + MU.log "Bucket #{bucket.name} is non-empty, will preserve it and its contents. Use --skipsnapshots to forcibly remove.", MU::WARN + end rescue Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::PermanentRedirect next end From 2e3dd90968c886d840753de0ca742eb4fdd5e5d0 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 30 Jun 2020 02:44:25 -0400 Subject: [PATCH 004/107] evidently our base docker image stopped shipping ssh-keygen --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95118615c..c06c330fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN df -h RUN apt-get update -RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl +RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl openssh-client RUN apt-get upgrade -y From d4dff72f83fce70e919fce7f106c66615ac6425c Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 30 Jun 2020 13:56:43 -0400 Subject: [PATCH 005/107] AWS::NoSQLDB: actually apply tags --- modules/mu/providers/aws/nosqldb.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 48cce6e71..d7500f81f 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -99,6 +99,10 @@ def create } end + if @tags + params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } } + end + MU.log "Creating DynamoDB table #{@mu_name}", details: params resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params) From 8bc89dd81693f3fa22a2d7031083cbee46069482 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 30 Jun 2020 15:59:22 -0400 Subject: [PATCH 006/107] AWS::Bucket: resolve absolute paths to upload artifacts, and throw a config validation error if the thing doesn't exist --- modules/mu/providers/aws/bucket.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index be05d6053..2605abfb2 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -181,7 +181,7 @@ def self.upload(url, acl: "private", file: nil, data: nil, credentials: nil, reg if file and !file.empty? if !File.exist?(file) or !File.readable?(file) - raise MuError, "Unable to read #{file} for upload to #{url}" + raise MuError, "Unable to read #{file} for upload to #{url} (I'm at #{Dir.pwd}" else data = File.read(file) end @@ -376,6 +376,16 @@ def self.validateConfig(bucket, _configurator) } end + if bucket['upload'] + bucket['upload'].each { |batch| + if !File.exists?(batch['source']) + MU.log "Bucket '#{bucket['name']}' specifies upload for file/directory that does not exist", MU::ERR, details: batch + ok = false + next + end + batch['source'] = File.realpath(File.expand_path(batch['source'])) + } + end ok end From a22306cd77aa50e639c021fedc0fa99423963216 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 1 Jul 2020 00:56:59 -0400 Subject: [PATCH 007/107] AWS::Endpoint: halfway implemented APIG adoption, why not? --- modules/mu/providers/aws/endpoint.rb | 82 +++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 45f569ef3..064c38607 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -38,13 +38,14 @@ def generate_methods m["auth"] ||= m["iam_role"] ? "AWS_IAM" : "NONE" method_arn = "arn:#{MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws"}:execute-api:#{@config["region"]}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{@cloud_id}/*/#{m['type']}/#{m['path']}" + path_part = ["", "/"].include?(m['path']) ? nil : m['path'] resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( rest_api_id: @cloud_id ) ext_resource = nil resp.items.each { |resource| - if resource.path_part == m['path'] + if resource.path_part == path_part ext_resource = resource.id end } @@ -69,7 +70,7 @@ def generate_methods MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).create_resource( rest_api_id: @cloud_id, parent_id: root_resource, - path_part: m['path'] + path_part: path_part ) end parent_id = resp.id @@ -100,6 +101,7 @@ def generate_methods } if r['headers'] params[:response_parameters] = r['headers'].map { |h| + h['required'] ||= false ["method.response.header."+h['header'], h['required']] }.to_h end @@ -283,6 +285,82 @@ def self.find(**args) found end + # Reverse-map our cloud description into a runnable config hash. + # We assume that any values we have in +@config+ are placeholders, and + # calculate our own accordingly based on what's live in the cloud. + def toKitten(**_args) + bok = { + "cloud" => "AWS", + "credentials" => @config['credentials'], + "cloud_id" => @cloud_id, + "region" => @config['region'] + } + + if !cloud_desc + MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config + return nil + end + + bok['name'] = cloud_desc.name +return nil if @cloud_id != "odl63ekwda" + + MU.log "REST API", MU::NOTICE, details: cloud_desc + resources = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( + rest_api_id: @cloud_id, + ).items + MU.log "resources", MU::NOTICE, details: resources + resources.each { |r| + r.resource_methods.each_pair { |http_type, m| + bok['methods'] ||= [] + method = {} + m_desc = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_method( + rest_api_id: @cloud_id, + resource_id: r.id, + http_method: http_type + ) + method['type'] = http_type + method['path'] = r.path_part || r.path + if m_desc.method_responses + m_desc.method_responses.each_pair { |code, resp_desc| + method['responses'] ||= [] + resp = { "code" => code.to_i } + if resp_desc.response_parameters + resp_desc.response_parameters.each_pair { |hdr, reqd| + resp['headers'] ||= [] + if hdr.match(/^method\.response\.header\.(.*)/) + resp['headers'] << { + "header" => Regexp.last_match[1], + "required" => reqd + } + else + MU.log "I don't know what to do with APIG response parameter #{hdr}", MU::ERR, details: resp_desc + end + + } + end + if resp_desc.response_models + resp_desc.response_models.each_pair { |content_type, body| + resp['body'] ||= [] + resp['body'] << { + "content_type" => content_type, + "is_error" => (body == "Error") + } + } + + end + method['responses'] << resp + + } + end + bok['methods'] << method + MU.log "method #{http_type}", MU::NOTICE, details: m_desc + } + } + puts "" +pp bok + bok + end + # Cloud-specific configuration properties. # @param _config [MU::Config]: The calling MU::Config object # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource From 12349ef325b928e262074f584bea7057e66848a2 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 1 Jul 2020 02:06:16 -0400 Subject: [PATCH 008/107] AWS::Endpoint: partial adoption lambda integration --- modules/mu/providers/aws/endpoint.rb | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 064c38607..bfc3575d1 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -352,8 +352,31 @@ def toKitten(**_args) } end + + if m_desc.method_integration + if m_desc.method_integration.type == "AWS" + if m_desc.method_integration.uri.match(/:lambda:path:\d{4}-\d{2}-\d{2}\/functions\/(arn:.*?)\/invocations$/) + method['integrate_with'] = MU::Config::Ref.get( + id: Regexp.last_match[1], + type: "function", + integration_http_method: m_desc.method_integration.http_method + ) + else + m_desc.method_integration.uri.match(/#{@config['region']}:([^:]+):action\/(.*)/) + method['integrate_with'] = { + "type" => "aws_generic", + "integration_http_method" => m_desc.method_integration.http_method, + "aws_generic_action" => Regexp.last_match[1]+":"+Regexp.last_match[2] + } + end + elsif m_desc.method_integration.type == "MOCK" + method['integrate_with'] = { + "type" => "mock" + } + end + end + bok['methods'] << method - MU.log "method #{http_type}", MU::NOTICE, details: m_desc } } puts "" From 5d76501ecd44ba6bedff12cf1bef32be4a72451c Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 1 Jul 2020 19:19:45 -0400 Subject: [PATCH 009/107] Config::Ref: implement []= so we can act like a hash when we want to; AWS::Endpoint: integration stuff --- modules/mu/config/ref.rb | 11 +++++++ modules/mu/providers/aws/endpoint.rb | 26 ++++++++++++--- modules/mu/providers/aws/role.rb | 47 ++++++++++++++-------------- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/modules/mu/config/ref.rb b/modules/mu/config/ref.rb index 5a387975c..fd6b69a97 100644 --- a/modules/mu/config/ref.rb +++ b/modules/mu/config/ref.rb @@ -140,6 +140,13 @@ def [](attribute) end end + # Lets callers set attributes like a {Hash} + # @param attribute [String,Symbol] + def []=(attribute, value) + instance_variable_set("@#{attribute.to_s}".to_sym, value) + self.class.define_reader(attribute) + end + # Unset an attribute. Sort of. We can't actually do that, so nil it out # and we get the behavior we want. def delete(attribute) @@ -270,6 +277,10 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) return nil if !@cloud or !@type loglevel = debug ? MU::NOTICE : MU::DEBUG + if debug + MU.log "this mf spittin", MU::WARN, details: caller + end + if @obj @deploy_id ||= @obj.deploy_id @id ||= @obj.cloud_id diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index bfc3575d1..76d5baf3e 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -352,28 +352,44 @@ def toKitten(**_args) } end - +pp m_desc if m_desc.method_integration if m_desc.method_integration.type == "AWS" - if m_desc.method_integration.uri.match(/:lambda:path:\d{4}-\d{2}-\d{2}\/functions\/(arn:.*?)\/invocations$/) + if m_desc.method_integration.uri.match(/:lambda:path\/\d{4}-\d{2}-\d{2}\/functions\/arn:.*?:function:(.*?)\/invocations$/) method['integrate_with'] = MU::Config::Ref.get( id: Regexp.last_match[1], - type: "function", + type: "functions", + cloud: "AWS", integration_http_method: m_desc.method_integration.http_method ) - else - m_desc.method_integration.uri.match(/#{@config['region']}:([^:]+):action\/(.*)/) + elsif m_desc.method_integration.uri.match(/#{@config['region']}:([^:]+):action\/(.*)/) method['integrate_with'] = { "type" => "aws_generic", "integration_http_method" => m_desc.method_integration.http_method, "aws_generic_action" => Regexp.last_match[1]+":"+Regexp.last_match[2] } + else + MU.log "I don't know what to do with #{m_desc.method_integration.uri}", MU::ERR + end + if m_desc.method_integration.http_method + method['integrate_with']['backend_http_method'] = m_desc.method_integration.http_method end elsif m_desc.method_integration.type == "MOCK" method['integrate_with'] = { "type" => "mock" } end + + if m_desc.method_integration.passthrough_behavior + method['integrate_with']['passthrough_behavior'] = m_desc.method_integration.passthrough_behavior + end + + if m_desc.method_integration.request_templates and + !m_desc.method_integration.request_templates.empty? + method['integrate_with'] = m_desc.method_integration.request_templates.keys.map { |rt_content_type, template| + { "content_type" => rt_content_type, "template" => template } + } + end end bok['methods'] << method diff --git a/modules/mu/providers/aws/role.rb b/modules/mu/providers/aws/role.rb index 2d4ed959d..882a40945 100644 --- a/modules/mu/providers/aws/role.rb +++ b/modules/mu/providers/aws/role.rb @@ -216,7 +216,22 @@ def arn # populated with one or both depending on what this resource has # defined. def cloud_desc(use_cache: true) - return @cloud_desc_cache if @cloud_desc_cache and use_cache + + # we might inherit a naive cached description from the base cloud + # layer; rearrange it to our tastes + if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role) + new_desc = { + "role" => @cloud_desc_cache + } + @cloud_desc_cache = new_desc + elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy) + new_desc = { + "policies" => [@cloud_desc_cache] + } + @cloud_desc_cache = new_desc + end + + return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache @cloud_desc_cache = {} if @config['bare_policies'] @@ -535,29 +550,15 @@ def self.find(**args) end else - marker = nil - begin - resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles( - marker: marker - ) - break if !resp or !resp.roles - resp.roles.each { |role| - found[role.role_name] = role - } - marker = resp.marker - end while marker + resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles + resp.roles.each { |role| + found[role.role_name] = role + } - begin - resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies( - scope: "Local", - marker: marker - ) - break if !resp or !resp.policies - resp.policies.each { |pol| - found[pol.arn] = pol - } - marker = resp.marker - end while marker + resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local") + resp.policies.each { |pol| + found[pol.arn] = pol + } end found From 634c18e16a9700a521e0a73892921852fb6652dc Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 2 Jul 2020 00:51:52 -0400 Subject: [PATCH 010/107] Adoption: add a --pattern flag to the utility so you can filter what resources you inhale --- bin/mu-adopt | 13 ++++++++++++- modules/mu/adoption.rb | 7 ++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/bin/mu-adopt b/bin/mu-adopt index ea25983e2..f8827dbfd 100755 --- a/bin/mu-adopt +++ b/bin/mu-adopt @@ -45,6 +45,7 @@ $opt = Optimist::options do opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false + opt :pattern, "Only adopt resources whose resource name would match this pattern. Must be a valid regular expression. Alphabetical characters will be treated case-insensitively.", :required => false, :type => :string end ok = true @@ -60,6 +61,16 @@ if $opt[:diff] $opt[:savedeploys] = false end +pattern = nil +if $opt[:pattern] + begin + pattern = Regexp.new($opt[:pattern], true) + rescue RegexpError => e + MU.log "Invalid --pattern option: #{e.message}", MU::ERR + exit 1 + end +end + types = [] $opt[:types].each { |t| t_name = t.gsub(/-/, "_") @@ -98,7 +109,7 @@ if !ok exit 1 end -adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes]) +adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes], pattern: pattern) found = adoption.scrapeClouds if found.nil? or found.empty? MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types } diff --git a/modules/mu/adoption.rb b/modules/mu/adoption.rb index aa8dad6fc..bebbb16b7 100644 --- a/modules/mu/adoption.rb +++ b/modules/mu/adoption.rb @@ -30,8 +30,7 @@ class Incomplete < MU::MuNonFatal; end :omnibus => "Jam everything into one monolothic configuration" } - - def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false) + def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false, pattern: nil) @scraped = {} @clouds = clouds @types = types @@ -49,6 +48,7 @@ def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_type @habitats ||= [] @scrub_mu_isms = scrub_mu_isms @merge = merge + @pattern = pattern end # Walk cloud providers with available credentials to discover resources @@ -127,6 +127,7 @@ def scrapeClouds() if obj.habitat and !cloudclass.listHabitats(credset).include?(obj.habitat) next end + # XXX apply any filters (e.g. MU-ID tags) if obj.cloud_id.nil? MU.log "This damn thing gave me no cloud id, what do I even do with that", MU::ERR, details: obj @@ -292,7 +293,7 @@ def generateBaskets(prefix: "") start = Time.now kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats, types: @types) - if kitten_cfg + if kitten_cfg and (!@pattern or @pattern.match(kitten_cfg['name'])) print "." kitten_cfg.delete("credentials") if @target_creds class_semaphore.synchronize { From 100e41ffd695a6e65259c04c1e48685e32e4a386 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 2 Jul 2020 04:05:38 -0400 Subject: [PATCH 011/107] AWS::Bucket: move upload path validation to generic config class; AWS::Function: start to support pointing to a directory full of code in addition to zipfile or s3 sources --- modules/mu/config/bucket.rb | 15 +++++++++++++-- modules/mu/config/function.rb | 19 ++++++++++++------- modules/mu/providers/aws/bucket.rb | 11 ----------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/modules/mu/config/bucket.rb b/modules/mu/config/bucket.rb index 21eff4a93..48a3a46e4 100644 --- a/modules/mu/config/bucket.rb +++ b/modules/mu/config/bucket.rb @@ -76,12 +76,23 @@ def self.schema end # Generic pre-processing of {MU::Config::BasketofKittens::buckets}, bare and unvalidated. - # @param _bucket [Hash]: The resource to process and validate + # @param bucket [Hash]: The resource to process and validate # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member # @return [Boolean]: True if validation succeeded, False otherwise - def self.validate(_bucket, _configurator) + def self.validate(bucket, _configurator) ok = true + if bucket['upload'] + bucket['upload'].each { |batch| + if !File.exists?(batch['source']) + MU.log "Bucket '#{bucket['name']}' specifies upload for file/directory that does not exist", MU::ERR, details: batch + ok = false + next + end + batch['source'] = File.realpath(File.expand_path(batch['source'])) + } + end + ok end diff --git a/modules/mu/config/function.rb b/modules/mu/config/function.rb index 6f5f3731d..bd16aaa36 100644 --- a/modules/mu/config/function.rb +++ b/modules/mu/config/function.rb @@ -106,13 +106,18 @@ def self.validate(function, _configurator) if !function['code'] ok = false end - if function['code'] and function['code']['zip_file'] - if !File.readable?(function['code']['zip_file']) - MU.log "Can't read Function deployment package #{function['code']['zip_file']}", MU::ERR - ok = false - else - function['code']['zip_file'] = File.realpath(File.expand_path(function['code']['zip_file'])) - end + + if function['code'] + ['zip_file', 'path'].each { |src| + if function['code'][src] + if !File.readable?(function['code'][src]) and !Dir.exists?(function['code'][src]) + MU.log "Function '#{function['name']}' specifies a deployment package that I can't read at #{function['code'][src]}", MU::ERR + ok = false + else + function['code'][src] = File.realpath(File.expand_path(function['code'][src])) + end + end + } end ok diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index 2605abfb2..b3ca3b84d 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -376,17 +376,6 @@ def self.validateConfig(bucket, _configurator) } end - if bucket['upload'] - bucket['upload'].each { |batch| - if !File.exists?(batch['source']) - MU.log "Bucket '#{bucket['name']}' specifies upload for file/directory that does not exist", MU::ERR, details: batch - ok = false - next - end - batch['source'] = File.realpath(File.expand_path(batch['source'])) - } - end - ok end From 3b0af23b05d038f706055a74f72debe78f3455a7 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 2 Jul 2020 15:35:15 -0400 Subject: [PATCH 012/107] Function: finish capability for zipping a deployment package from a flat directory, at least in AWS --- modules/Gemfile.lock | 20 ++++++++++---------- modules/mu/config/function.rb | 4 ++++ modules/mu/master.rb | 18 ++++++++++++++++++ modules/mu/providers/aws/function.rb | 17 ++++++++++++++--- modules/mu/providers/aws/role.rb | 2 +- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index 32b3a106f..7bd023b58 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -33,7 +33,7 @@ PATH rack (~> 2.0) rubocop (~> 0.58) ruby-graphviz (~> 1.2) - rubyzip (~> 2.0) + rubyzip (~> 2.3) simple-password-gen (~> 0.1) slack-notifier (~> 2.3) solve (~> 4.0) @@ -54,7 +54,7 @@ GEM public_suffix (>= 2.0.2, < 4.0) ast (2.4.1) aws-eventstream (1.1.0) - aws-sdk-core (2.11.537) + aws-sdk-core (2.11.540) aws-sigv4 (~> 1.0) jmespath (~> 1.0) aws-sigv4 (1.2.1) @@ -566,19 +566,19 @@ GEM concurrent-ruby (1.1.6) cookbook-omnifetch (0.9.1) mixlib-archive (>= 0.4, < 2.0) - cucumber-core (7.0.0) - cucumber-gherkin (~> 13.0, >= 13.0.0) - cucumber-messages (~> 12.1, >= 12.1.1) + cucumber-core (7.1.0) + cucumber-gherkin (~> 14.0, >= 14.0.1) + cucumber-messages (~> 12.2, >= 12.2.0) cucumber-tag-expressions (~> 2.0, >= 2.0.4) - cucumber-gherkin (13.0.0) - cucumber-messages (~> 12.0, >= 12.0.0) + cucumber-gherkin (14.0.1) + cucumber-messages (~> 12.2, >= 12.2.0) cucumber-messages (12.2.0) protobuf-cucumber (~> 3.10, >= 3.10.8) cucumber-tag-expressions (2.0.4) daemons (1.3.1) - declarative (0.0.10) + declarative (0.0.20) declarative-option (0.1.0) - diff-lcs (1.4.2) + diff-lcs (1.4.4) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) erubi (1.9.0) @@ -836,7 +836,7 @@ GEM winrm (~> 2.0) wmi-lite (1.0.5) yard (0.9.25) - zeitwerk (2.3.0) + zeitwerk (2.3.1) PLATFORMS ruby diff --git a/modules/mu/config/function.rb b/modules/mu/config/function.rb index bd16aaa36..a88639dcf 100644 --- a/modules/mu/config/function.rb +++ b/modules/mu/config/function.rb @@ -71,6 +71,10 @@ def self.schema "zip_file" => { "type" => "string", "description" => "Path to a zipped deployment package to upload." + }, + "path" => { + "type" => "string", + "description" => "Path to a directory that can be zipped into deployment package to upload." } } }, diff --git a/modules/mu/master.rb b/modules/mu/master.rb index 607a61301..add6dd170 100644 --- a/modules/mu/master.rb +++ b/modules/mu/master.rb @@ -880,5 +880,23 @@ def self.syncMonitoringConfig(blocking = true) end end + def self.zipDir(srcdir, outfile) + require 'zip' + ::Zip::File.open(outfile, ::Zip::File::CREATE) { |zipfile| + addpath = Proc.new { |zip_path, parent_path| + Dir.entries(parent_path).reject{ |d| [".", ".."].include?(d) }.each { |entry| + src = File.join(parent_path, entry) + dst = File.join(zip_path, entry).sub(/^\//, '') + if File.directory?(src) + addpath.call(dst, src) + else + zipfile.add(dst, src) + end + } + } + addpath.call("", srcdir) + } + end + end end diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 2e43f3446..1a2ee0593 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -53,10 +53,21 @@ def create runtime: @config['runtime'], } - if @config['code']['zip_file'] + if @config['code']['zip_file'] or @config['code']['path'] + tempfile = nil + if @config['code']['path'] + tempfile = Tempfile.new + MU.log "Creating deployment package from #{@config['code']['path']}" + MU::Master.zipDir(@config['code']['path'], tempfile.path) + @config['code']['zip_file'] = tempfile.path + end zip = File.read(@config['code']['zip_file']) MU.log "Uploading deployment package from #{@config['code']['zip_file']}" lambda_properties[:code][:zip_file] = zip + if tempfile + tempfile.close + tempfile.unlink + end else lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket'] lambda_properties[:code][:s3_key] = @config['code']['s3_key'] @@ -321,7 +332,7 @@ def toKitten(**_args) "cloud_id" => @cloud_id, "region" => @config['region'] } - +return if @cloud_id =~ /^(serverlessrepo-)/ if !cloud_desc MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config return nil @@ -336,6 +347,7 @@ def toKitten(**_args) function = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function(function_name: bok['name']) if function.code.repository_type == "S3" + pp function.code if @cloud_id.match(/pff/) bok['code'] = {} function.code.location.match(/^https:\/\/([^\.]+)\..*?\/([^?]+).*?(?:versionId=([^&]+))?/) bok['code']['s3_bucket'] = Regexp.last_match[1] @@ -393,7 +405,6 @@ def toKitten(**_args) if function.configuration.role shortname = function.configuration.role.sub(/.*?role\/([^\/]+)$/, '\1') -MU.log shortname, MU::NOTICE, details: function.configuration.role bok['role'] = MU::Config::Ref.get( id: shortname, name: shortname, diff --git a/modules/mu/providers/aws/role.rb b/modules/mu/providers/aws/role.rb index 882a40945..2ccc60d95 100644 --- a/modules/mu/providers/aws/role.rb +++ b/modules/mu/providers/aws/role.rb @@ -30,7 +30,7 @@ def initialize(**args) end end - @mu_name ||= @deploy.getResourceName(@config["name"]) + @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64) end # Called automatically by {MU::Deploy#createResources} From 59d188f688193e1f761bb565623156941407ba11 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 6 Jul 2020 01:42:52 -0400 Subject: [PATCH 013/107] AWS::SearchDomain: mostly complete adoption, geez that was easy --- cloud-mu.gemspec | 2 +- modules/mu/providers/aws/endpoint.rb | 4 +- modules/mu/providers/aws/search_domain.rb | 79 ++++++++++++++++++++--- 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/cloud-mu.gemspec b/cloud-mu.gemspec index dab88f203..8c4fa0112 100644 --- a/cloud-mu.gemspec +++ b/cloud-mu.gemspec @@ -57,7 +57,7 @@ EOF s.add_runtime_dependency 'rack', "~> 2.0" s.add_runtime_dependency 'ruby-graphviz', "~> 1.2" s.add_runtime_dependency 'rubocop', '~> 0.58' - s.add_runtime_dependency 'rubyzip', "~> 2.0" + s.add_runtime_dependency 'rubyzip', "~> 2.3" s.add_runtime_dependency 'simple-password-gen', "~> 0.1" s.add_runtime_dependency 'slack-notifier', "~> 2.3" s.add_runtime_dependency 'solve', '~> 4.0' diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 76d5baf3e..89c7f9c42 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -302,7 +302,6 @@ def toKitten(**_args) end bok['name'] = cloud_desc.name -return nil if @cloud_id != "odl63ekwda" MU.log "REST API", MU::NOTICE, details: cloud_desc resources = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( @@ -395,8 +394,7 @@ def toKitten(**_args) bok['methods'] << method } } - puts "" -pp bok + bok end diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 1b77f0c2b..24e0b1ff6 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -35,7 +35,7 @@ def create params = genParams MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params - MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).create_elasticsearch_domain(params).domain_status + MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).create_elasticsearch_domain(params).domain_status tagDomain @@ -51,7 +51,7 @@ def groom waitWhileProcessing # wait until the create finishes, if still going MU.log "Updating ElasticSearch domain #{@config['domain_name']}", MU::NOTICE, details: params - MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).update_elasticsearch_domain_config(params) + MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).update_elasticsearch_domain_config(params) end waitWhileProcessing # don't return until creation/updating is complete @@ -64,11 +64,11 @@ def groom def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache @cloud_desc_cache = if @config['domain_name'] - MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).describe_elasticsearch_domain( + MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( domain_name: @config['domain_name'] ).domain_status elsif @deploydata and @deploydata['domain_name'] - MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).describe_elasticsearch_domain( + MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( domain_name: @deploydata['domain_name'] ).domain_status else @@ -87,7 +87,7 @@ def arn # @return [Hash] def notify deploy_struct = MU.structToHash(cloud_desc) - tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).list_tags(arn: deploy_struct[:arn]).tag_list + tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: deploy_struct[:arn]).tag_list deploy_struct['tags'] = tags.map { |t| { t.key => t.value } } if deploy_struct['endpoint'] deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/" @@ -191,6 +191,69 @@ def self.find(**args) found end + # Reverse-map our cloud description into a runnable config hash. + # We assume that any values we have in +@config+ are placeholders, and + # calculate our own accordingly based on what's live in the cloud. + def toKitten(**_args) + bok = { + "cloud" => "AWS", + "credentials" => @credentials, + "cloud_id" => @cloud_id, + "region" => @config['region'] + } + + if !cloud_desc + MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config + return nil + end +pp cloud_desc + bok['name'] = cloud_desc.domain_name + bok['elasticsearch_version'] = cloud_desc.elasticsearch_version + bok['instance_count'] = cloud_desc.elasticsearch_cluster_config.instance_count + bok['instance_type'] = cloud_desc.elasticsearch_cluster_config.instance_type + bok['zone_aware'] = cloud_desc.elasticsearch_cluster_config.zone_awareness_enabled + + if cloud_desc.elasticsearch_cluster_config.dedicated_master_enabled + bok['dedicated_masters'] = cloud_desc.elasticsearch_cluster_config.dedicated_master_count + bok['master_instance_type'] = cloud_desc.elasticsearch_cluster_config.dedicated_master_type + end + + if cloud_desc.access_policies + bok['access_policies'] = JSON.parse(cloud_desc.access_policies) + end + + if cloud_desc.advanced_options and !cloud_desc.advanced_options.empty? + bok['advanced_options'] = cloud_desc.advanced_options + end + + bok['ebs_size'] = cloud_desc.ebs_options.volume_size + bok['ebs_type'] = cloud_desc.ebs_options.volume_type + bok['ebs_iops'] = cloud_desc.ebs_options.iops if cloud_desc.ebs_options.iops + + if cloud_desc.snapshot_options and cloud_desc.snapshot_options.automated_snapshot_start_hour + bok['snapshot_hour'] = cloud_desc.snapshot_options.automated_snapshot_start_hour + end + + if cloud_desc.cognito_options.user_pool_id and + cloud_desc.cognito_options.identity_pool_id + bok['user_pool_id'] = cloud_desc.cognito_options.user_pool_id + bok['identity_pool_id'] = cloud_desc.cognito_options.identity_pool_id + end + + tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: cloud_desc.arn).tag_list + if tags and !tags.empty? + bok['tags'] = MU.structToHash(tags) + end + + # vpc (vpc_options) + # security groups + # slow_logs (log_publishing_options) + +MU.log "desc", MU::NOTICE, details: cloud_desc +MU.log "bok", MU::NOTICE, details: bok + bok + end + # Cloud-specific configuration properties. # @param _config [MU::Config]: The calling MU::Config object # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource @@ -200,7 +263,7 @@ def self.schema(_config) versions = begin MU::Cloud::AWS.elasticsearch.list_elasticsearch_versions.elasticsearch_versions rescue MuError - ["7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6"] + ["7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6"] end instance_types = begin MU::Cloud::AWS.elasticsearch.list_elasticsearch_instance_types( @@ -246,7 +309,7 @@ def self.schema(_config) }, "ebs_type" => { "type" => "string", - "default" => "standard", + "default" => "gp2", "description" => "Type of EBS storage to use for cluster nodes. If 'none' is specified, EBS storage will not be used, but this is only valid for certain instance types.", "enum" => ["standard", "gp2", "io1", "none"] }, @@ -677,7 +740,7 @@ def tagDomain raise MU::MuError, "Can't tag ElasticSearch domain, cloud descriptor came back without an ARN" end - MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).add_tags( + MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).add_tags( arn: domain.arn, tag_list: tags ) From 545371c009dc25498ba8420cd360db8518011a13 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 6 Jul 2020 13:35:55 -0400 Subject: [PATCH 014/107] AWS::NoSQLDB: adoption, dead simple apparentluy --- modules/mu/providers/aws/nosqldb.rb | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index d7500f81f..30906499c 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -248,6 +248,51 @@ def self.find(**args) found end + # Reverse-map our cloud description into a runnable config hash. + # We assume that any values we have in +@config+ are placeholders, and + # calculate our own accordingly based on what's live in the cloud. + def toKitten(**_args) + bok = { + "cloud" => "AWS", + "credentials" => @config['credentials'], + "cloud_id" => @cloud_id, + "region" => @config['region'] + } + + if !cloud_desc + MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config + return nil + end + bok['name'] = cloud_desc.table_name + bok['read_capacity'] = cloud_desc.provisioned_throughput.read_capacity_units + bok['write_capacity'] = cloud_desc.provisioned_throughput.write_capacity_units + + cloud_desc.attribute_definitions.each { |attr| + bok['attributes'] ||= [] + newattr = { + "name" => attr.attribute_name, + "type" => attr.attribute_type + } + if cloud_desc.key_schema + cloud_desc.key_schema.each { |key| + next if key.attribute_name == attr.attribute_name + if key.key_type == "RANGE" + newattr["primary_partition"] = true + elsif key.key_type == "HASH" + newattr["primary_sort"] = true + end + } + end + bok['attributes'] << newattr + } + + if cloud_desc.stream_specification and cloud_desc.stream_specification.stream_enabled + bok['stream'] = cloud_desc.stream_specification.stream_view_type + end + + bok + end + # Cloud-specific configuration properties. # @param _config [MU::Config]: The calling MU::Config object # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource From 9e0304ca66d05480e6166cc04d0351e3a52e429c Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 7 Jul 2020 02:15:49 -0400 Subject: [PATCH 015/107] AWS::SearchDomain: finish #toKitten (VPCs, Security Groups, slow logs) --- modules/mu/providers/aws/search_domain.rb | 37 ++++++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 24e0b1ff6..0dee9f057 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -245,12 +245,39 @@ def toKitten(**_args) bok['tags'] = MU.structToHash(tags) end - # vpc (vpc_options) - # security groups - # slow_logs (log_publishing_options) + if cloud_desc.vpc_options + bok['vpc'] = MU::Config::Ref.get( + id: cloud_desc.vpc_options.vpc_id, + cloud: "AWS", + credentials: @credentials, + type: "vpcs", + region: @config['region'], + subnets: cloud_desc.vpc_options.subnet_ids.map { |s| { "subnet_id" => s } } + ) + if cloud_desc.vpc_options.security_group_ids and + !cloud_desc.vpc_options.security_group_ids.empty? + bok['add_firewall_rules'] = cloud_desc.vpc_options.security_group_ids.map { |sg| + MU::Config::Ref.get( + id: sg, + cloud: "AWS", + credentials: @credentials, + region: @config['region'], + type: "firewall_rules", + ) + } + end + end + + if cloud_desc.log_publishing_options + # XXX this is primitive... there are multiple other log types now, + # and this should be a Ref blob, not a flat string + cloud_desc.log_publishing_options.each_pair { |type, whither| + if type == "SEARCH_SLOW_LOGS" + bok['slow_logs'] = whither.cloud_watch_logs_log_group_arn + end + } + end -MU.log "desc", MU::NOTICE, details: cloud_desc -MU.log "bok", MU::NOTICE, details: bok bok end From 20652027d13265d7c7f0609003baf41ebda65f18 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 7 Jul 2020 04:21:33 -0400 Subject: [PATCH 016/107] Notifier: rework schema to behave better with referenced sibling resources --- modules/mu/config/bucket.rb | 1 + modules/mu/config/notifier.rb | 11 +++-- modules/mu/master.rb | 3 ++ modules/mu/providers/aws/notifier.rb | 61 +++++++++++++++++++++------- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/modules/mu/config/bucket.rb b/modules/mu/config/bucket.rb index 48a3a46e4..11ce963f6 100644 --- a/modules/mu/config/bucket.rb +++ b/modules/mu/config/bucket.rb @@ -52,6 +52,7 @@ def self.schema }, "upload" => { "type" => "array", + "description" => "Upload objects to a bucket, where supported", "items" => { "type" => "object", "required" => ["source", "destination"], diff --git a/modules/mu/config/notifier.rb b/modules/mu/config/notifier.rb index 22163a997..6d1b62021 100644 --- a/modules/mu/config/notifier.rb +++ b/modules/mu/config/notifier.rb @@ -36,12 +36,12 @@ def self.schema "items" => { "type" => "object", "description" => "A list of people or resources which should receive notifications", - "required" => ["endpoint"], "properties" => { "endpoint" => { "type" => "string", - "description" => "The endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number." - } + "description" => "Shorthand for an endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number. For complex cases, such as referencing an AWS Lambda function defined elsewhere in your Mu stack, use +resource+ instead." + }, + "resource" => MU::Config::Ref.schema(desc: "A cloud resource that is a valid notification target for this notifier. For simple use cases, such as external email addresses or SMS, use +endpoint+ instead.") } } } @@ -56,6 +56,11 @@ def self.schema def self.validate(notifier, _configurator) ok = true + if !notifier['endpoint'] and !notifier['resource'] + MU.log "Notifier '#{notifier['name']}' must specify either resource or endpoint", MU::ERR + ok = false + end + if notifier['subscriptions'] notifier['subscriptions'].each { |sub| if !sub["type"] diff --git a/modules/mu/master.rb b/modules/mu/master.rb index add6dd170..d0972113b 100644 --- a/modules/mu/master.rb +++ b/modules/mu/master.rb @@ -880,6 +880,9 @@ def self.syncMonitoringConfig(blocking = true) end end + # Recursively zip a directory + # @param srcdir [String] + # @param outfile [String] def self.zipDir(srcdir, outfile) require 'zip' ::Zip::File.open(outfile, ::Zip::File::CREATE) { |zipfile| diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index 1edd90e66..b13a88902 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -120,6 +120,34 @@ def self.find(**args) found end + # Reverse-map our cloud description into a runnable config hash. + # We assume that any values we have in +@config+ are placeholders, and + # calculate our own accordingly based on what's live in the cloud. + def toKitten(**_args) + bok = { + "cloud" => "AWS", + "credentials" => @config['credentials'], + "cloud_id" => @cloud_id, + "region" => @config['region'] + } + + if !cloud_desc + MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config + return nil + end + + bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"] + MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub| + bok['subcriptions'] ||= [] + bok['subcriptions'] << { + "type" => sub.protocol, + "endpoint" => sub.endpoint + } + } + + bok + end + # Cloud-specific configuration properties. # @param _config [MU::Config]: The calling MU::Config object # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource @@ -130,11 +158,10 @@ def self.schema(_config) "type" => "array", "items" => { "type" => "object", - "required" => ["endpoint"], "properties" => { "type" => { "type" => "string", - "description" => "", + "description" => "Type of endpoint or resource which should receive notifications. If not specified, will attempt to auto-detect.", "enum" => ["http", "https", "email", "email-json", "sms", "sqs", "application", "lambda"] } } @@ -156,18 +183,24 @@ def self.validateConfig(notifier, _configurator) if notifier['subscriptions'] notifier['subscriptions'].each { |sub| if !sub["type"] - if sub["endpoint"].match(/^http:/i) - sub["type"] = "http" - elsif sub["endpoint"].match(/^https:/i) - sub["type"] = "https" - elsif sub["endpoint"].match(/^sqs:/i) - sub["type"] = "sqs" - elsif sub["endpoint"].match(/^\+?[\d\-]+$/) - sub["type"] = "sms" - elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i) - sub["type"] = "email" - else - MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR + if sub['resource'] + MU::Config::Ref.get(sub['resource']) + elsif sub['endpoint'] + if sub["endpoint"].match(/^http:/i) + sub["type"] = "http" + elsif sub["endpoint"].match(/^https:/i) + sub["type"] = "https" + elsif sub["endpoint"].match(/^sqs:/i) + sub["type"] = "sqs" + elsif sub["endpoint"].match(/^\+?[\d\-]+$/) + sub["type"] = "sms" + elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i) + sub["type"] = "email" + end + end + + if !sub['type'] + MU.log "Notifier #{notifier['name']} subscription did not specify a type, and I'm unable to guess one", MU::ERR, details: sub ok = false end end From aaab2293264c39723017a140b81701b0f1dfc2ec Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 7 Jul 2020 19:23:17 -0400 Subject: [PATCH 017/107] AWS::Notified: finish the 'resource' flag for subscriptions: Adoption: enhance some ref lookup messaging --- modules/mu.rb | 4 +- modules/mu/adoption.rb | 3 +- modules/mu/config/ref.rb | 15 +++++- modules/mu/providers/aws/endpoint.rb | 2 +- modules/mu/providers/aws/function.rb | 4 +- modules/mu/providers/aws/notifier.rb | 56 ++++++++++++++++++----- modules/mu/providers/aws/role.rb | 7 +-- modules/mu/providers/aws/search_domain.rb | 2 +- 8 files changed, 68 insertions(+), 25 deletions(-) diff --git a/modules/mu.rb b/modules/mu.rb index 37ebf391f..805acdbd9 100644 --- a/modules/mu.rb +++ b/modules/mu.rb @@ -299,8 +299,8 @@ def initialize(message = nil, silent: false, details: nil) # Wrapper class for temporary Exceptions. Gives our internals something to # inherit that will log a notice message appropriately before bubbling up. class MuNonFatal < StandardError - def initialize(message = nil, silent: false) - MU.log message, MU::NOTICE if !message.nil? and !silent + def initialize(message = nil, silent: false, details: nil) + MU.log message, MU::NOTICE, details: details if !message.nil? and !silent if MU.verbosity == MU::Logger::SILENT super "" else diff --git a/modules/mu/adoption.rb b/modules/mu/adoption.rb index bebbb16b7..11e784574 100644 --- a/modules/mu/adoption.rb +++ b/modules/mu/adoption.rb @@ -792,8 +792,7 @@ def resolveReferences(cfg, deploy, parent) elsif hashcfg["id"] and !hashcfg["name"] hashcfg.delete("deploy_id") else - pp parent.cloud_desc - raise Incomplete, "Failed to resolve reference on behalf of #{parent}" + raise Incomplete.new "Failed to resolve reference on behalf of #{parent}", details: hashcfg end hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id diff --git a/modules/mu/config/ref.rb b/modules/mu/config/ref.rb index fd6b69a97..efae39fcf 100644 --- a/modules/mu/config/ref.rb +++ b/modules/mu/config/ref.rb @@ -278,7 +278,7 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) loglevel = debug ? MU::NOTICE : MU::DEBUG if debug - MU.log "this mf spittin", MU::WARN, details: caller + MU.log "this mf kitten", MU::WARN, details: caller end if @obj @@ -334,6 +334,18 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) [@habitat.to_s] end + MU.log "Ref#kitten calling findStray", loglevel, details: { + cloud: @cloud, + type: @type, + name: @name, + cloud_id: @id, + deploy_id: try_deploy_id, + region: @region, + habitats: hab_arg, + credentials: @credentials, + dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type)) + } + found = MU::MommaCat.findStray( @cloud, @type, @@ -345,6 +357,7 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) credentials: @credentials, dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type)) ) + MU.log "Ref#kitten results from findStray", loglevel, details: found @obj ||= found.first if found rescue MU::MommaCat::MultipleMatches => e if try_deploy_id.nil? and MU.deploy_id diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 89c7f9c42..b929eebe9 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -351,7 +351,7 @@ def toKitten(**_args) } end -pp m_desc + if m_desc.method_integration if m_desc.method_integration.type == "AWS" if m_desc.method_integration.uri.match(/:lambda:path\/\d{4}-\d{2}-\d{2}\/functions\/arn:.*?:function:(.*?)\/invocations$/) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 1a2ee0593..675552dda 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -332,7 +332,7 @@ def toKitten(**_args) "cloud_id" => @cloud_id, "region" => @config['region'] } -return if @cloud_id =~ /^(serverlessrepo-)/ + if !cloud_desc MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config return nil @@ -347,7 +347,6 @@ def toKitten(**_args) function = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function(function_name: bok['name']) if function.code.repository_type == "S3" - pp function.code if @cloud_id.match(/pff/) bok['code'] = {} function.code.location.match(/^https:\/\/([^\.]+)\..*?\/([^?]+).*?(?:versionId=([^&]+))?/) bok['code']['s3_bucket'] = Regexp.last_match[1] @@ -407,7 +406,6 @@ def toKitten(**_args) shortname = function.configuration.role.sub(/.*?role\/([^\/]+)$/, '\1') bok['role'] = MU::Config::Ref.get( id: shortname, - name: shortname, cloud: "AWS", type: "roles" ) diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index b13a88902..05105e56d 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -36,6 +36,9 @@ def create def groom if @config['subscriptions'] @config['subscriptions'].each { |sub| + if sub['resource'] and !sub['endpoint'] + sub['endpoint'] = MU::Config::Ref.get(sub['resource']).kitten.arn + end MU::Cloud::AWS::Notifier.subscribe( arn: arn, endpoint: sub['endpoint'], @@ -137,12 +140,35 @@ def toKitten(**_args) end bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"] + svcmap = { + "lambda" => "functions", + "sqs" => "msg_queues" + } MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub| bok['subcriptions'] ||= [] - bok['subcriptions'] << { - "type" => sub.protocol, - "endpoint" => sub.endpoint - } + bok['subcriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/) + _wholestring, service, region, account, id = Regexp.last_match.to_a + { + "type" => sub.protocol, + "resource" => MU::Config::Ref.get( + type: svcmap[service], + region: region, + credentials: @credentials, + id: id, + cloud: "AWS", + habitat: MU::Config::Ref.get( + id: account, + cloud: "AWS", + credentials: @credentials + ) + ) + } + else + { + "type" => sub.protocol, + "endpoint" => sub.endpoint + } + end } bok @@ -183,19 +209,25 @@ def self.validateConfig(notifier, _configurator) if notifier['subscriptions'] notifier['subscriptions'].each { |sub| if !sub["type"] - if sub['resource'] - MU::Config::Ref.get(sub['resource']) + sub['type'] = if sub['resource'] + if sub['resource']['type'] == "functions" + "lambda" + elsif sub['resource']['type'] == "msg_queues" + "sqs" + end elsif sub['endpoint'] if sub["endpoint"].match(/^http:/i) - sub["type"] = "http" + "http" elsif sub["endpoint"].match(/^https:/i) - sub["type"] = "https" - elsif sub["endpoint"].match(/^sqs:/i) - sub["type"] = "sqs" + "https" + elsif sub["endpoint"].match(/:sqs:/i) + "sqs" + elsif sub["endpoint"].match(/:lambda:/i) + "lambda" elsif sub["endpoint"].match(/^\+?[\d\-]+$/) - sub["type"] = "sms" + "sms" elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i) - sub["type"] = "email" + "email" end end diff --git a/modules/mu/providers/aws/role.rb b/modules/mu/providers/aws/role.rb index 2ccc60d95..3bd216d2d 100644 --- a/modules/mu/providers/aws/role.rb +++ b/modules/mu/providers/aws/role.rb @@ -531,7 +531,7 @@ def self.find(**args) begin # managed policies get fetched by ARN, roles by plain name. Ok! - if args[:cloud_id].match(/^arn:/) + if args[:cloud_id].match(/^arn:.*?:policy\//) resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy( policy_arn: args[:cloud_id] ) @@ -540,10 +540,11 @@ def self.find(**args) end else resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role( - role_name: args[:cloud_id] + role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable ) + if resp and resp.role - found[args[:cloud_id]] = resp.role + found[resp.role.role_name] = resp.role end end rescue ::Aws::IAM::Errors::NoSuchEntity diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 0dee9f057..a07407e48 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -206,7 +206,7 @@ def toKitten(**_args) MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config return nil end -pp cloud_desc + bok['name'] = cloud_desc.domain_name bok['elasticsearch_version'] = cloud_desc.elasticsearch_version bok['instance_count'] = cloud_desc.elasticsearch_cluster_config.instance_count From c001fd7b05557b355057a3b4024d16d3ed061fab Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 8 Jul 2020 01:33:23 -0400 Subject: [PATCH 018/107] AWS: shaking out bugs in new functionality from espier integration --- modules/mu/config/notifier.rb | 22 +++------------------- modules/mu/providers/aws/endpoint.rb | 10 +++++++--- modules/mu/providers/aws/nosqldb.rb | 2 +- modules/mu/providers/aws/notifier.rb | 12 ++++++++---- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/modules/mu/config/notifier.rb b/modules/mu/config/notifier.rb index 6d1b62021..ca1ee6b67 100644 --- a/modules/mu/config/notifier.rb +++ b/modules/mu/config/notifier.rb @@ -56,28 +56,12 @@ def self.schema def self.validate(notifier, _configurator) ok = true - if !notifier['endpoint'] and !notifier['resource'] - MU.log "Notifier '#{notifier['name']}' must specify either resource or endpoint", MU::ERR - ok = false - end if notifier['subscriptions'] notifier['subscriptions'].each { |sub| - if !sub["type"] - if sub["endpoint"].match(/^http:/i) - sub["type"] = "http" - elsif sub["endpoint"].match(/^https:/i) - sub["type"] = "https" - elsif sub["endpoint"].match(/^sqs:/i) - sub["type"] = "sqs" - elsif sub["endpoint"].match(/^\+?[\d\-]+$/) - sub["type"] = "sms" - elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i) - sub["type"] = "email" - else - MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR - ok = false - end + if !sub['endpoint'] and !sub['resource'] + MU.log "Notifier '#{notifier['name']}' must specify either resource or endpoint in subscription", MU::ERR, details: sub + ok = false end } end diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index b929eebe9..58bb5aff3 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -307,8 +307,9 @@ def toKitten(**_args) resources = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( rest_api_id: @cloud_id, ).items - MU.log "resources", MU::NOTICE, details: resources + resources.each { |r| + next if !r.respond_to?(:resource_methods) or r.resource_methods.nil? r.resource_methods.each_pair { |http_type, m| bok['methods'] ||= [] method = {} @@ -353,7 +354,7 @@ def toKitten(**_args) end if m_desc.method_integration - if m_desc.method_integration.type == "AWS" + if ["AWS", "AWS_PROXY"].include?(m_desc.method_integration.type) if m_desc.method_integration.uri.match(/:lambda:path\/\d{4}-\d{2}-\d{2}\/functions\/arn:.*?:function:(.*?)\/invocations$/) method['integrate_with'] = MU::Config::Ref.get( id: Regexp.last_match[1], @@ -377,6 +378,9 @@ def toKitten(**_args) method['integrate_with'] = { "type" => "mock" } + else + MU.log "I don't know what to do with this integration", MU::ERR, details: m_desc.method_integration + next end if m_desc.method_integration.passthrough_behavior @@ -502,7 +506,7 @@ def self.schema(_config) "type" => { "type" => "string", "description" => "A Mu resource type, for integrations with a sibling resource (e.g. a function), or the string +aws_generic+, which we can use in combination with +aws_generic_action+ to integrate with arbitrary AWS services.", - "enum" => ["aws_generic"].concat(MU::Cloud.resource_types.values.map { |t| t[:cfg_name] }.sort) + "enum" => ["aws_generic"].concat(MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] }.sort) }, "aws_generic_action" => { "type" => "string", diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 30906499c..d65bd6afa 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -103,7 +103,7 @@ def create params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } } end - MU.log "Creating DynamoDB table #{@mu_name}", details: params + MU.log "Creating DynamoDB table #{@mu_name}", MU::NOTICE, details: params resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params) @cloud_id = @mu_name diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index 05105e56d..e5202b30f 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -145,8 +145,8 @@ def toKitten(**_args) "sqs" => "msg_queues" } MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub| - bok['subcriptions'] ||= [] - bok['subcriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/) + bok['subscriptions'] ||= [] + bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/) _wholestring, service, region, account, id = Regexp.last_match.to_a { "type" => sub.protocol, @@ -201,13 +201,16 @@ def self.schema(_config) # Cloud-specific pre-processing of {MU::Config::BasketofKittens::notifier}, bare and unvalidated. # @param notifier [Hash]: The resource to process and validate - # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member + # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member # @return [Boolean]: True if validation succeeded, False otherwise - def self.validateConfig(notifier, _configurator) + def self.validateConfig(notifier, configurator) ok = true if notifier['subscriptions'] notifier['subscriptions'].each { |sub| + if sub['resource'] and configurator.haveLitterMate?(sub['resource']['name'], sub['resource']['type']) + MU::Config.addDependency(notifier, sub['resource']['name'], sub['resource']['type']) + end if !sub["type"] sub['type'] = if sub['resource'] if sub['resource']['type'] == "functions" @@ -215,6 +218,7 @@ def self.validateConfig(notifier, _configurator) elsif sub['resource']['type'] == "msg_queues" "sqs" end + exit elsif sub['endpoint'] if sub["endpoint"].match(/^http:/i) "http" From 6c38b1a536930102b6c195843d21a3a6e88592b1 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 9 Jul 2020 02:36:23 -0400 Subject: [PATCH 019/107] AWS::Function: inject key attributes of sibling resources as environment variables, so code can find databases dynamically --- modules/mu/config/vpc.rb | 1 + modules/mu/providers/aws/database.rb | 8 +- modules/mu/providers/aws/endpoint.rb | 6 +- modules/mu/providers/aws/function.rb | 221 +++++++++++++--------- modules/mu/providers/aws/nosqldb.rb | 7 +- modules/mu/providers/aws/notifier.rb | 3 +- modules/mu/providers/aws/search_domain.rb | 35 ++-- 7 files changed, 166 insertions(+), 115 deletions(-) diff --git a/modules/mu/config/vpc.rb b/modules/mu/config/vpc.rb index a2fc2640a..af44d66ee 100644 --- a/modules/mu/config/vpc.rb +++ b/modules/mu/config/vpc.rb @@ -417,6 +417,7 @@ def self.validate(vpc, configurator) using_default_cidr = false if !vpc['ip_block'] if configurator.updating and configurator.existing_deploy and + configurator.existing_deploy.original_config and configurator.existing_deploy.original_config['vpcs'] configurator.existing_deploy.original_config['vpcs'].each { |v| if v['name'].to_s == vpc['name'].to_s diff --git a/modules/mu/providers/aws/database.rb b/modules/mu/providers/aws/database.rb index b440e216c..5c16c343b 100644 --- a/modules/mu/providers/aws/database.rb +++ b/modules/mu/providers/aws/database.rb @@ -680,7 +680,7 @@ def allowHost(cidr) # Return the metadata for this ContainerCluster # @return [Hash] def notify - deploy_struct = MU.structToHash(cloud_desc) + deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) deploy_struct['cloud_id'] = @cloud_id deploy_struct["region"] ||= @config['region'] deploy_struct["db_name"] ||= @config['db_name'] @@ -1274,7 +1274,7 @@ def add_basic def add_cluster_node - cluster = MU::Config::Ref.get(@config["member_of_cluster"]).kitten(@deploy, debug: true) + cluster = MU::Config::Ref.get(@config["member_of_cluster"]).kitten(@deploy) if cluster.nil? or cluster.cloud_id.nil? raise MuError.new "Failed to resolve parent cluster of #{@mu_name}", details: @config["member_of_cluster"].to_h end @@ -1355,7 +1355,7 @@ def create_basic # creation_style = point_in_time def create_point_in_time - @config["source"].kitten(@deploy, debug: true) + @config["source"].kitten(@deploy) if !@config["source"].id raise MuError.new "Database '#{@config['name']}' couldn't resolve cloud id for source database", details: @config["source"].to_h end @@ -1381,7 +1381,7 @@ def create_point_in_time # creation_style = new, existing and read_replica_of is not nil def create_read_replica - @config["source"].kitten(@deploy, debug: true) + @config["source"].kitten(@deploy) if !@config["source"].id raise MuError.new "Database '#{@config['name']}' couldn't resolve cloud id for source database", details: @config["source"].to_h end diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 58bb5aff3..ff1fcda0c 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -133,7 +133,7 @@ def generate_methods uri, type = if m['integrate_with']['type'] == "aws_generic" svc, action = m['integrate_with']['aws_generic_action'].split(/:/) ["arn:aws:apigateway:"+@config['region']+":#{svc}:action/#{action}", "AWS"] - elsif m['integrate_with']['type'] == "function" + elsif m['integrate_with']['type'] == "functions" function_obj = @deploy.findLitterMate(name: m['integrate_with']['name'], type: "functions").cloudobj ["arn:aws:apigateway:"+@config['region']+":lambda:path/2015-03-31/functions/"+function_obj.arn+"/invocations", "AWS"] elsif m['integrate_with']['type'] == "mock" @@ -166,6 +166,7 @@ def generate_methods } end +MU.log "integration", MU::NOTICE, details: params resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).put_integration(params) if m['integrate_with']['type'] == "function" @@ -234,7 +235,8 @@ def cloud_desc(use_cache: true) # Return the metadata for this API # @return [Hash] def notify - deploy_struct = MU.structToHash(cloud_desc) + deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) + deploy_struct['url'] = "https://"+@cloud_id+".execute-api."+@config['region']+".amazonaws.com/"+@config['deploy_to'] # XXX stages and whatnot return deploy_struct end diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 675552dda..21b7b642b 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -18,6 +18,17 @@ class AWS # A function as configured in {MU::Config::BasketofKittens::functions} class Function < MU::Cloud::Function + # If we have sibling resources in our deployment, automatically inject + # interesting things about them into our function's environment + # variables. + SIBLING_VARS = { + "servers" => ["private_ip_address", "public_ip_address"], + "search_domains" => ["endpoint"], + "databases" => ["endpoint"], + "endpoints" => ["url"], + "nosqldbs" => ["table_arn"] + } + # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us. # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat def initialize(**args) @@ -42,103 +53,31 @@ def assign_tag(resource_arn, tag_list, region=@config['region']) # Called automatically by {MU::Deploy#createResources} def create - role_arn = get_role_arn(@config['iam_role']) - - lambda_properties = { - code: {}, - function_name: @mu_name, - handler: @config['handler'], - publish: true, - role: role_arn, - runtime: @config['runtime'], - } - - if @config['code']['zip_file'] or @config['code']['path'] - tempfile = nil - if @config['code']['path'] - tempfile = Tempfile.new - MU.log "Creating deployment package from #{@config['code']['path']}" - MU::Master.zipDir(@config['code']['path'], tempfile.path) - @config['code']['zip_file'] = tempfile.path - end - zip = File.read(@config['code']['zip_file']) - MU.log "Uploading deployment package from #{@config['code']['zip_file']}" - lambda_properties[:code][:zip_file] = zip - if tempfile - tempfile.close - tempfile.unlink - end - else - lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket'] - lambda_properties[:code][:s3_key] = @config['code']['s3_key'] - if @config['code']['s3_object_version'] - lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version'] - end - end - - if @config.has_key?('timeout') - lambda_properties[:timeout] = @config['timeout'].to_i ## secs - end - - if @config.has_key?('memory') - lambda_properties[:memory_size] = @config['memory'].to_i - end - if @config.has_key?('environment_variables') - lambda_properties[:environment] = { - variables: {@config['environment_variables'][0]['key'] => @config['environment_variables'][0]['value']} - } - end + lambda_properties = get_properties - lambda_properties[:tags] = {} - MU::MommaCat.listStandardTags.each_pair { |k, v| - lambda_properties[:tags][k] = v + MU.retrier([Aws::Lambda::Errors::InvalidParameterValueException], max: 5, wait: 10) { + resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties) + @cloud_id = resp.function_name } - if @config['tags'] - @config['tags'].each { |tag| - lambda_properties[:tags][tag.key.first] = tag.values.first - } - end - if @config.has_key?('vpc') - sgs = [] - if @config['add_firewall_rules'] - @config['add_firewall_rules'].each { |sg| - sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['name']) - sgs << sg.cloud_id if sg and sg.cloud_id - } - end - if !@vpc - raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded" - end - lambda_properties[:vpc_config] = { - :subnet_ids => @vpc.subnets.map { |s| s.cloud_id }, - :security_group_ids => sgs - } - end - - retries = 0 - resp = begin - MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties) - rescue Aws::Lambda::Errors::InvalidParameterValueException => e - # Freshly-made IAM roles sometimes aren't really ready - if retries < 5 - sleep 10 - retries += 1 - retry - end - raise e - end - - @cloud_id = resp.function_name end # Called automatically by {MU::Deploy#createResources} def groom - desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function( - function_name: @mu_name - ) - func_arn = desc.configuration.function_arn if !desc.empty? + + MU.log @cloud_id, MU::NOTICE, details: cloud_desc + old_props = MU.structToHash(cloud_desc) + new_props = get_properties + new_props.reject! { |k, _v| [:code, :publish, :tags].include?(k) } + changes = {} + new_props.each_pair { |k, v| + changes[k] = v if v != old_props[k] + } + if !changes.empty? + MU.log "Updating Lambda #{@mu_name}", MU::NOTICE, details: changes + MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).update_function_configuration(new_props) + end # tag_function = assign_tag(lambda_func.function_arn, @config['tags']) @@ -167,7 +106,7 @@ def groom MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties) rescue Aws::Lambda::Errors::ResourceConflictException end - adjust_trigger(tr['service'], trigger_arn, func_arn, @mu_name) + adjust_trigger(tr['service'], trigger_arn, arn, @mu_name) } end @@ -258,7 +197,8 @@ def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda' # Return the metadata for this Function rule # @return [Hash] def notify - deploy_struct = MU.structToHash(MU::Cloud::AWS::Function.find(cloud_id: @cloud_id, credentials: @config['credentials'], region: @config['region']).values.first) + deploy_struct = MU.structToHash(MU::Cloud::AWS::Function.find(cloud_id: @cloud_id, credentials: @credentials, region: @config['region']).values.first, stringify_keys: true) + deploy_struct['mu_name'] = @mu_name return deploy_struct end @@ -556,6 +496,105 @@ def self.validateConfig(function, configurator) private + def get_properties + role_arn = get_role_arn(@config['iam_role']) + + lambda_properties = { + code: {}, + function_name: @mu_name, + handler: @config['handler'], + publish: true, + role: role_arn, + runtime: @config['runtime'], + } + + if @config['code']['zip_file'] or @config['code']['path'] + tempfile = nil + if @config['code']['path'] + tempfile = Tempfile.new + MU.log "#{@mu_name} using code at #{@config['code']['path']}" + MU::Master.zipDir(@config['code']['path'], tempfile.path) + @config['code']['zip_file'] = tempfile.path + else + MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}" + end + zip = File.read(@config['code']['zip_file']) + lambda_properties[:code][:zip_file] = zip + if tempfile + tempfile.close + tempfile.unlink + end + else + lambda_properties[:code][:s3_bucket] = @config['code']['s3_bucket'] + lambda_properties[:code][:s3_key] = @config['code']['s3_key'] + if @config['code']['s3_object_version'] + lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version'] + end + end + + if @config.has_key?('timeout') + lambda_properties[:timeout] = @config['timeout'].to_i ## secs + end + + if @config.has_key?('memory') + lambda_properties[:memory_size] = @config['memory'].to_i + end + + SIBLING_VARS.each_key { |sib_type| + siblings = @deploy.findLitterMate(return_all: true, type: sib_type, cloud: "AWS") + if siblings + siblings.each_value { |sibling| + metadata = sibling.notify + next if !metadata + SIBLING_VARS[sib_type].each { |var| + if metadata[var] + @config['environment_variables'] ||= [] + @config['environment_variables'] << { + "key" => (sibling.config['name']+"_"+var).gsub(/[^a-z0-9_]/i, '_'), + "value" => metadata[var] + } + end + } + } + end + } + + if @config.has_key?('environment_variables') + lambda_properties[:environment] = { + variables: Hash[@config['environment_variables'].map { |v| [v['key'], v['value']] }] + } + end + + lambda_properties[:tags] = {} + MU::MommaCat.listStandardTags.each_pair { |k, v| + lambda_properties[:tags][k] = v + } + if @config['tags'] + @config['tags'].each { |tag| + lambda_properties[:tags][tag.key.first] = tag.values.first + } + end + + if @config.has_key?('vpc') + sgs = [] + if @config['add_firewall_rules'] + @config['add_firewall_rules'].each { |sg| + sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['name']) + sgs << sg.cloud_id if sg and sg.cloud_id + } + end + if !@vpc + raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded" + end + lambda_properties[:vpc_config] = { + :subnet_ids => @vpc.subnets.map { |s| s.cloud_id }, + :security_group_ids => sgs + } + end + + lambda_properties + end + # Given an IAM role name, resolve to ARN. Will attempt to identify any # sibling Mu role resources by this name first, and failing that, will # do a plain get_role() to the IAM API for the provided name. diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index d65bd6afa..b3b19ffee 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -67,6 +67,11 @@ def create } end } + # apparently the HASH key always has to be before RANGE, so sort it + # lexically by that field and call it a day + params[:key_schema].sort! { |a, b| + a[:key_type] <=> b[:key_type] + } if @config['secondary_indexes'] @config['secondary_indexes'].each { |idx| @@ -218,7 +223,7 @@ def arn # Return the metadata for this user cofiguration # @return [Hash] def notify - MU.structToHash(cloud_desc) + MU.structToHash(cloud_desc, stringify_keys: true) end # Locate an existing DynamoDB table diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index e5202b30f..ba7793036 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -37,7 +37,7 @@ def groom if @config['subscriptions'] @config['subscriptions'].each { |sub| if sub['resource'] and !sub['endpoint'] - sub['endpoint'] = MU::Config::Ref.get(sub['resource']).kitten.arn + sub['endpoint'] = MU::Config::Ref.get(sub['resource']).kitten(@deploy).arn end MU::Cloud::AWS::Notifier.subscribe( arn: arn, @@ -209,6 +209,7 @@ def self.validateConfig(notifier, configurator) if notifier['subscriptions'] notifier['subscriptions'].each { |sub| if sub['resource'] and configurator.haveLitterMate?(sub['resource']['name'], sub['resource']['type']) + sub['resource']['cloud'] = "AWS" MU::Config.addDependency(notifier, sub['resource']['name'], sub['resource']['type']) end if !sub["type"] diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index a07407e48..cd8dba0c6 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -22,9 +22,9 @@ class SearchDomain < MU::Cloud::SearchDomain # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat def initialize(**args) super - if @cloud_id and !@config['domain_name'] - @config['domain_name'] = @cloud_id - end + describe if @mu_name and !@deploydata + @cloud_id ||= @deploydata['domain_name'] if @deploydata + @mu_name ||= @deploy.getResourceName(@config["name"]) end @@ -44,7 +44,7 @@ def create # Called automatically by {MU::Deploy#createResources} def groom tagDomain - @config['domain_name'] ||= @deploydata['domain_name'] + @config['domain_name'] ||= @cloud_id params = genParams(cloud_desc) # get parameters that would change only if params.size > 1 @@ -63,17 +63,20 @@ def groom # our druthers. def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache - @cloud_desc_cache = if @config['domain_name'] - MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( - domain_name: @config['domain_name'] - ).domain_status - elsif @deploydata and @deploydata['domain_name'] - MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( - domain_name: @deploydata['domain_name'] + @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( + domain_name: @cloud_id ).domain_status - else - raise MuError, "#{@mu_name} can't find its official Elasticsearch domain name!" - end +# if @config['domain_name'] +# MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( +# domain_name: @config['domain_name'] +# ).domain_status +# elsif @deploydata and @deploydata['domain_name'] +# MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( +# domain_name: @deploydata['domain_name'] +# ).domain_status +# else +# raise MuError, "#{@mu_name} can't find its official Elasticsearch domain name!" +# end @cloud_desc_cache end @@ -86,8 +89,8 @@ def arn # Return the metadata for this SearchDomain rule # @return [Hash] def notify - deploy_struct = MU.structToHash(cloud_desc) - tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: deploy_struct[:arn]).tag_list + deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) + tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: arn).tag_list deploy_struct['tags'] = tags.map { |t| { t.key => t.value } } if deploy_struct['endpoint'] deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/" From f28383eb7ad0e796522e633bfeb01dd63ee15a64 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 9 Jul 2020 04:32:49 -0400 Subject: [PATCH 020/107] AWS::Function: regroom also updating code now, sort of --- modules/mu/providers/aws/function.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 21b7b642b..f6146db0e 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -26,6 +26,7 @@ class Function < MU::Cloud::Function "search_domains" => ["endpoint"], "databases" => ["endpoint"], "endpoints" => ["url"], + "notifiers" => ["TopicArn"], "nosqldbs" => ["table_arn"] } @@ -68,7 +69,9 @@ def groom MU.log @cloud_id, MU::NOTICE, details: cloud_desc old_props = MU.structToHash(cloud_desc) + new_props = get_properties + code_block = new_props[:code] new_props.reject! { |k, _v| [:code, :publish, :tags].include?(k) } changes = {} new_props.each_pair { |k, v| @@ -79,6 +82,13 @@ def groom MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).update_function_configuration(new_props) end + if @code_sha256 and @code_sha256 != cloud_desc.code_sha_256.chomp + MU.log "Updating code in Lambda #{@mu_name}", MU::NOTICE, details: { "old" => @code_sha256, "new" => cloud_desc.code_sha_256 } + code_block[:publish] = true + code_block[:function_name] = @cloud_id + MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).update_function_code(code_block) + end + # tag_function = assign_tag(lambda_func.function_arn, @config['tags']) ### The most common triggers can be ==> SNS, S3, Cron, API-Gateway @@ -519,6 +529,7 @@ def get_properties MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}" end zip = File.read(@config['code']['zip_file']) + @code_sha256 = Base64.encode64(Digest::SHA256.digest(zip)).chomp lambda_properties[:code][:zip_file] = zip if tempfile tempfile.close @@ -530,6 +541,7 @@ def get_properties if @config['code']['s3_object_version'] lambda_properties[:code][:s3_object_version] = @config['code']['s3_object_version'] end +# XXX need to download to a temporarily file, read it in, and calculate the digest in order to trigger updates in groom end if @config.has_key?('timeout') From 549ed035da47e4579a215318b55c43070570fc46 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 9 Jul 2020 23:11:24 -0400 Subject: [PATCH 021/107] Cleanup: dammit, just pass deploy_id around explicitly --- modules/mu/cleanup.rb | 4 +++- modules/mu/cloud/wrappers.rb | 4 ++++ modules/mu/providers/aws/alarm.rb | 4 ++-- modules/mu/providers/aws/bucket.rb | 4 ++-- modules/mu/providers/aws/cache_cluster.rb | 8 ++++---- modules/mu/providers/aws/collection.rb | 4 ++-- modules/mu/providers/aws/container_cluster.rb | 16 ++++++++-------- modules/mu/providers/aws/database.rb | 14 +++++++------- modules/mu/providers/aws/dnszone.rb | 10 +++++----- modules/mu/providers/aws/endpoint.rb | 4 ++-- modules/mu/providers/aws/firewall_rule.rb | 4 ++-- modules/mu/providers/aws/folder.rb | 2 +- modules/mu/providers/aws/function.rb | 4 ++-- modules/mu/providers/aws/group.rb | 4 ++-- modules/mu/providers/aws/habitat.rb | 4 ++-- modules/mu/providers/aws/loadbalancer.rb | 14 +++++++------- modules/mu/providers/aws/log.rb | 6 +++--- modules/mu/providers/aws/msg_queue.rb | 4 ++-- modules/mu/providers/aws/nosqldb.rb | 4 ++-- modules/mu/providers/aws/notifier.rb | 4 ++-- modules/mu/providers/aws/role.rb | 10 +++++----- modules/mu/providers/aws/search_domain.rb | 6 +++--- modules/mu/providers/aws/server.rb | 16 ++++++++-------- modules/mu/providers/aws/server_pool.rb | 4 ++-- modules/mu/providers/aws/storage_pool.rb | 4 ++-- modules/mu/providers/aws/user.rb | 8 ++++---- modules/mu/providers/aws/vpc.rb | 6 +++--- modules/mu/providers/azure/server.rb | 2 +- modules/mu/providers/google/bucket.rb | 2 +- modules/mu/providers/google/container_cluster.rb | 2 +- modules/mu/providers/google/database.rb | 2 +- modules/mu/providers/google/firewall_rule.rb | 2 +- modules/mu/providers/google/folder.rb | 2 +- modules/mu/providers/google/function.rb | 2 +- modules/mu/providers/google/group.rb | 2 +- modules/mu/providers/google/habitat.rb | 2 +- modules/mu/providers/google/loadbalancer.rb | 2 +- modules/mu/providers/google/role.rb | 2 +- modules/mu/providers/google/server.rb | 2 +- modules/mu/providers/google/server_pool.rb | 2 +- modules/mu/providers/google/user.rb | 2 +- modules/mu/providers/google/vpc.rb | 2 +- 42 files changed, 106 insertions(+), 100 deletions(-) diff --git a/modules/mu/cleanup.rb b/modules/mu/cleanup.rb index 8854c3723..4d458b340 100644 --- a/modules/mu/cleanup.rb +++ b/modules/mu/cleanup.rb @@ -52,6 +52,7 @@ def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, ver @onlycloud = onlycloud @skipcloud = skipcloud @ignoremaster = ignoremaster + @deploy_id = deploy_id if @skipcloud and @onlycloud # you actually mean noop @onlycloud = @skipcloud = false @@ -344,7 +345,8 @@ def self.call_cleanup(type, credset, provider, flags, region) region: region, cloud: provider, flags: flags, - credentials: credset + credentials: credset, + deploy_id: @deploy_id ) else true diff --git a/modules/mu/cloud/wrappers.rb b/modules/mu/cloud/wrappers.rb index 28922c563..c71020edc 100644 --- a/modules/mu/cloud/wrappers.rb +++ b/modules/mu/cloud/wrappers.rb @@ -126,6 +126,10 @@ def self.cleanup(*flags) clouds = [params[:cloud]] params.delete(:cloud) end + params[:deploy_id] ||= MU.deploy_id + if !params[:deploy_id] or params[:deploy_id].empty? + raise MuError, "Can't call cleanup methods without a deploy id" + end clouds.each { |cloud| begin diff --git a/modules/mu/providers/aws/alarm.rb b/modules/mu/providers/aws/alarm.rb index d7a0d4ded..dad5b4960 100644 --- a/modules/mu/providers/aws/alarm.rb +++ b/modules/mu/providers/aws/alarm.rb @@ -124,7 +124,7 @@ def self.isGlobal? # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::Alarm.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Alarm artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster alarms = [] @@ -132,7 +132,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent # This can miss alarms in some cases (eg. cache_cluster) so we might want to delete alarms from each API as well. MU::Cloud::AWS.cloudwatch(credentials: credentials, region: region).describe_alarms.each { |page| page.metric_alarms.map(&:alarm_name).each { |alarm_name| - alarms << alarm_name if alarm_name.match(MU.deploy_id) + alarms << alarm_name if alarm_name.match(deploy_id) } } diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index b3ca3b84d..d014d636e 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -228,7 +228,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::Bucket.cleanup: need to support flags['known']", MU::DEBUG, details: flags resp = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_buckets @@ -263,7 +263,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent deploy_match = false master_match = false tags.each { |tag| - if tag.key == "MU-ID" and tag.value == MU.deploy_id + if tag.key == "MU-ID" and tag.value == deploy_id deploy_match = true elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip master_match = true diff --git a/modules/mu/providers/aws/cache_cluster.rb b/modules/mu/providers/aws/cache_cluster.rb index 5c35f5118..96bdcaa00 100644 --- a/modules/mu/providers/aws/cache_cluster.rb +++ b/modules/mu/providers/aws/cache_cluster.rb @@ -569,7 +569,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server. # @param region [String]: The cloud provider's region in which to operate. # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {}) skipsnapshots = flags["skipsnapshots"] all_clusters = MU::Cloud::AWS.elasticache(credentials: credentials, region: region).describe_cache_clusters our_clusters = [] @@ -577,7 +577,7 @@ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, region: MU. # Because we can't run list_tags_for_resource on a cache cluster that isn't in "available" state we're loading the deploy to make sure we have a cache cluster to cleanup. # To ensure we don't miss cache clusters that have been terminated mid creation we'll load the 'original_config'. We might want to find a better approach for this. - deploy = MU::MommaCat.getLitter(MU.deploy_id) + deploy = MU::MommaCat.getLitter(deploy_id) if deploy.original_config && deploy.original_config.has_key?("cache_clusters") && !deploy.original_config["cache_clusters"].empty? # The ElastiCache API and documentation are a mess, the replication group ARN resource_type is not documented, and is not easily guessable. @@ -615,14 +615,14 @@ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, region: MU. sleep 30 retry else - raise MuError, "Failed to get tags for cache cluster #{cluster_id}, MU-ID #{MU.deploy_id}: #{e.inspect}" + raise MuError, "Failed to get tags for cache cluster #{cluster_id}, MU-ID #{deploy_id}: #{e.inspect}" end end found_muid = false found_master = false tags.each { |tag| - found_muid = true if tag.key == "MU-ID" && tag.value == MU.deploy_id + found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip } next if !found_muid diff --git a/modules/mu/providers/aws/collection.rb b/modules/mu/providers/aws/collection.rb index 96b6014c8..5c30a7576 100644 --- a/modules/mu/providers/aws/collection.rb +++ b/modules/mu/providers/aws/collection.rb @@ -242,7 +242,7 @@ def self.isGlobal? # @param region [String]: The cloud provider region # @param wait [Boolean]: Block on the removal of this stack; AWS deletion will continue in the background otherwise if false. # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, wait: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, wait: false, credentials: nil, flags: {}) MU.log "AWS::Collection.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Collection artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster @@ -251,7 +251,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, wait: f resp.stacks.each { |stack| ok = false stack.tags.each { |tag| - ok = true if (tag.key == "MU-ID") and tag.value == MU.deploy_id + ok = true if (tag.key == "MU-ID") and tag.value == deploy_id } if ok MU.log "Deleting CloudFormation stack #{stack.stack_name})" diff --git a/modules/mu/providers/aws/container_cluster.rb b/modules/mu/providers/aws/container_cluster.rb index 4bbceaca2..c670f89c8 100644 --- a/modules/mu/providers/aws/container_cluster.rb +++ b/modules/mu/providers/aws/container_cluster.rb @@ -406,17 +406,17 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::ContainerCluster.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS ContainerCluster artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster - purge_ecs_clusters(noop: noop, region: region, credentials: credentials) + purge_ecs_clusters(noop: noop, region: region, credentials: credentials, deploy_id: deploy_id) - purge_eks_clusters(noop: noop, region: region, credentials: credentials) + purge_eks_clusters(noop: noop, region: region, credentials: credentials, deploy_id: deploy_id) end - def self.purge_eks_clusters(noop: false, region: MU.curRegion, credentials: nil) + def self.purge_eks_clusters(noop: false, region: MU.curRegion, credentials: nil, deploy_id: MU.deploy_id) return if !MU::Cloud::AWS::ContainerCluster.EKSRegions(credentials, region: region).include?(region) resp = begin MU::Cloud::AWS.eks(credentials: credentials, region: region).list_clusters @@ -429,7 +429,7 @@ def self.purge_eks_clusters(noop: false, region: MU.curRegion, credentials: nil) return if !resp or !resp.clusters resp.clusters.each { |cluster| - if cluster.match(/^#{MU.deploy_id}-/) + if cluster.match(/^#{deploy_id}-/) desc = MU::Cloud::AWS.eks(credentials: credentials, region: region).describe_cluster( name: cluster @@ -473,13 +473,13 @@ def self.purge_eks_clusters(noop: false, region: MU.curRegion, credentials: nil) end private_class_method :purge_eks_clusters - def self.purge_ecs_clusters(noop: false, region: MU.curRegion, credentials: nil) + def self.purge_ecs_clusters(noop: false, region: MU.curRegion, credentials: nil, deploy_id: MU.deploy_id) resp = MU::Cloud::AWS.ecs(credentials: credentials, region: region).list_clusters return if !resp or !resp.cluster_arns or resp.cluster_arns.empty? resp.cluster_arns.each { |arn| - if arn.match(/:cluster\/(#{MU.deploy_id}[^:]+)$/) + if arn.match(/:cluster\/(#{deploy_id}[^:]+)$/) cluster = Regexp.last_match[1] svc_resp = MU::Cloud::AWS.ecs(region: region, credentials: credentials).list_services( @@ -525,7 +525,7 @@ def self.purge_ecs_clusters(noop: false, region: MU.curRegion, credentials: nil) } tasks = MU::Cloud::AWS.ecs(region: region, credentials: credentials).list_task_definitions( - family_prefix: MU.deploy_id + family_prefix: deploy_id ) if tasks and tasks.task_definition_arns diff --git a/modules/mu/providers/aws/database.rb b/modules/mu/providers/aws/database.rb index 5c16c343b..f89f9474a 100644 --- a/modules/mu/providers/aws/database.rb +++ b/modules/mu/providers/aws/database.rb @@ -761,7 +761,7 @@ def self.quality end # @return [Array] - def self.threaded_resource_purge(describe_method, list_method, id_method, arn_type, region, credentials, ignoremaster, known: []) + def self.threaded_resource_purge(describe_method, list_method, id_method, arn_type, region, credentials, ignoremaster, known: [], deploy_id: MU.deploy_id) deletia = [] resp = MU::Cloud::AWS.rds(credentials: credentials, region: region).send(describe_method) @@ -774,7 +774,7 @@ def self.threaded_resource_purge(describe_method, list_method, id_method, arn_ty next end - if should_delete?(tags, resource.send(id_method), ignoremaster, MU.deploy_id, MU.mu_public_ip, known) + if should_delete?(tags, resource.send(id_method), ignoremaster, deploy_id, MU.mu_public_ip, known) deletia << resource.send(id_method) end } @@ -795,18 +795,18 @@ def self.threaded_resource_purge(describe_method, list_method, id_method, arn_ty # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region in which to operate # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {}) ["instance", "cluster"].each { |type| - threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known']) { |id| - terminate_rds_instance(nil, noop: noop, skipsnapshots: flags["skipsnapshots"], region: region, deploy_id: MU.deploy_id, cloud_id: id, mu_name: id.upcase, credentials: credentials, cluster: (type == "cluster"), known: flags['known']) + threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id| + terminate_rds_instance(nil, noop: noop, skipsnapshots: flags["skipsnapshots"], region: region, deploy_id: deploy_id, cloud_id: id, mu_name: id.upcase, credentials: credentials, cluster: (type == "cluster"), known: flags['known']) }.each { |t| t.join } } - threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known']) { |id| + threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id| MU.log "Deleting RDS subnet group #{id}" MU.retrier([Aws::RDS::Errors::InvalidDBSubnetGroupStateFault], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBSubnetGroupNotFoundFault]) { MU::Cloud::AWS.rds(region: region).delete_db_subnet_group(db_subnet_group_name: id) if !noop @@ -814,7 +814,7 @@ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, region: MU. } ["db", "db_cluster"].each { |type| - threads.concat threaded_resource_purge("describe_#{type}_parameter_groups".to_sym, "#{type}_parameter_groups".to_sym, "#{type}_parameter_group_name".to_sym, (type == "db" ? "pg" : "cluster-pg"), region, credentials, ignoremaster, known: flags['known']) { |id| + threads.concat threaded_resource_purge("describe_#{type}_parameter_groups".to_sym, "#{type}_parameter_groups".to_sym, "#{type}_parameter_group_name".to_sym, (type == "db" ? "pg" : "cluster-pg"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id| MU.log "Deleting RDS #{type} parameter group #{id}" MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBParameterGroupNotFound]) { MU::Cloud::AWS.rds(region: region).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop diff --git a/modules/mu/providers/aws/dnszone.rb b/modules/mu/providers/aws/dnszone.rb index f813352e2..4269d3807 100644 --- a/modules/mu/providers/aws/dnszone.rb +++ b/modules/mu/providers/aws/dnszone.rb @@ -42,7 +42,7 @@ def create params = { :name => @config['name'], :hosted_zone_config => { - :comment => MU.deploy_id + :comment => @deploy.deploy_id }, :caller_reference => @deploy.getResourceName(@config['name']) } @@ -666,7 +666,7 @@ def self.quality # Called by {MU::Cleanup}. Locates resources that were created by the # currently-loaded deployment, and purges them. - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::DNSZone.cleanup: need to support flags['known']", MU::DEBUG, details: flags threads = [] @@ -679,7 +679,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent muid_match = false mumaster_match = false tags.each { |tag| - muid_match = true if tag.key == "MU-ID" and tag.value == MU.deploy_id + muid_match = true if tag.key == "MU-ID" and tag.value == deploy_id mumaster_match = true if tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip } @@ -723,7 +723,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent t.join } - zones = MU::Cloud::DNSZone.find(deploy_id: MU.deploy_id, region: region) + zones = MU::Cloud::DNSZone.find(deploy_id: deploy_id, region: region) zones.values.each { |zone| MU.log "Purging DNS Zone '#{zone.name}' (#{zone.id})" if !noop @@ -779,7 +779,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent # TO DO: if we have more than one record it will retry the deletion multiple times and will throw Aws::Route53::Errors::InvalidChangeBatch / record not found even though the record was deleted zone_rrsets.each { |record| - if record.name.match(MU.deploy_id.downcase) + if record.name.match(deploy_id.downcase) resource_records = [] record.resource_records.each { |rrecord| resource_records << rrecord.value diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index ff1fcda0c..bed50749a 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -246,7 +246,7 @@ def notify # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::Endpoint.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Endpoint artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster @@ -254,7 +254,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent if resp and resp.items resp.items.each { |api| # The stupid things don't have tags - if api.description == MU.deploy_id + if api.description == deploy_id MU.log "Deleting API Gateway #{api.name} (#{api.id})" if !noop MU::Cloud::AWS.apig(region: region, credentials: credentials).delete_rest_api( diff --git a/modules/mu/providers/aws/firewall_rule.rb b/modules/mu/providers/aws/firewall_rule.rb index 000719778..0edc28017 100644 --- a/modules/mu/providers/aws/firewall_rule.rb +++ b/modules/mu/providers/aws/firewall_rule.rb @@ -381,14 +381,14 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) filters = if flags and flags["vpc_id"] [ {name: "vpc-id", values: [flags["vpc_id"]]} ] else filters = [ - {name: "tag:MU-ID", values: [MU.deploy_id]} + {name: "tag:MU-ID", values: [deploy_id]} ] if !ignoremaster filters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]} diff --git a/modules/mu/providers/aws/folder.rb b/modules/mu/providers/aws/folder.rb index bd762b016..451eab869 100644 --- a/modules/mu/providers/aws/folder.rb +++ b/modules/mu/providers/aws/folder.rb @@ -59,7 +59,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) end # Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials. diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index f6146db0e..9fcb1c5ed 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -231,14 +231,14 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::Function.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f| desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function( function_name: f.function_name ) - if desc.tags and desc.tags["MU-ID"] == MU.deploy_id and (desc.tags["MU-MASTER-IP"] == MU.mu_public_ip or ignoremaster) + if desc.tags and desc.tags["MU-ID"] == deploy_id and (desc.tags["MU-MASTER-IP"] == MU.mu_public_ip or ignoremaster) MU.log "Deleting Lambda function #{f.function_name}" if !noop MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function( diff --git a/modules/mu/providers/aws/group.rb b/modules/mu/providers/aws/group.rb index a823a6b21..7442e18fc 100644 --- a/modules/mu/providers/aws/group.rb +++ b/modules/mu/providers/aws/group.rb @@ -186,12 +186,12 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) MU.log "AWS::Group.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Group artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster resp = MU::Cloud::AWS.iam(credentials: credentials).list_groups( - path_prefix: "/"+MU.deploy_id+"/" + path_prefix: "/"+deploy_id+"/" ) if resp and resp.groups resp.groups.each { |g| diff --git a/modules/mu/providers/aws/habitat.rb b/modules/mu/providers/aws/habitat.rb index 9fc4cf867..1d5b6e351 100644 --- a/modules/mu/providers/aws/habitat.rb +++ b/modules/mu/providers/aws/habitat.rb @@ -90,7 +90,7 @@ def self.isGlobal? # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) return if !orgMasterCreds?(credentials) MU.log "AWS::Habitat.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Habitat artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster @@ -99,7 +99,7 @@ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) if resp and resp.accounts resp.accounts.each { |acct| - if acct.name.match(/^#{Regexp.quote(MU.deploy_id)}/) or acct.name.match(/BUNS/) + if acct.name.match(/^#{Regexp.quote(deploy_id)}/) or acct.name.match(/BUNS/) if !noop pp acct end diff --git a/modules/mu/providers/aws/loadbalancer.rb b/modules/mu/providers/aws/loadbalancer.rb index 584e69ac9..750dd39fc 100644 --- a/modules/mu/providers/aws/loadbalancer.rb +++ b/modules/mu/providers/aws/loadbalancer.rb @@ -671,8 +671,8 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) - if (MU.deploy_id.nil? or MU.deploy_id.empty?) and (!flags or !flags["vpc_id"]) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + if (deploy_id.nil? or deploy_id.empty?) and (!flags or !flags["vpc_id"]) raise MuError, "Can't touch ELBs without MU-ID or vpc_id flag" end @@ -682,7 +682,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent # @param region [String]: The cloud provider region # @param ignoremaster [Boolean]: Whether to ignore the MU-MASTER-IP tag # @param classic [Boolean]: Whether to look for a classic ELB instead of an ALB (ELB2) - def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = false) + def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = false, deploy_id: MU.deploy_id) tags = [] if classic tags = MU::Cloud::AWS.elb(credentials: credentials, region: region).describe_tags( @@ -699,7 +699,7 @@ def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = fals if !tags.nil? tags.each { |tag| saw_tags << tag.key - muid_match = true if tag.key == "MU-ID" and tag.value == MU.deploy_id + muid_match = true if tag.key == "MU-ID" and tag.value == deploy_id mumaster_match = true if tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip } end @@ -725,9 +725,9 @@ def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = fals matched = true if lb.vpc_id == flags['vpc_id'] else if classic - matched = self.checkForTagMatch(lb.load_balancer_name, region, ignoremaster, credentials, classic) + matched = self.checkForTagMatch(lb.load_balancer_name, region, ignoremaster, credentials, classic, deploy_id: deploy_id) else - matched = self.checkForTagMatch(lb.load_balancer_arn, region, ignoremaster, credentials, classic) + matched = self.checkForTagMatch(lb.load_balancer_arn, region, ignoremaster, credentials, classic, deploy_id: deploy_id) end end if matched @@ -773,7 +773,7 @@ def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = fals tgs.each { |tg| - if self.checkForTagMatch(tg.target_group_arn, region, ignoremaster, credentials) + if self.checkForTagMatch(tg.target_group_arn, region, ignoremaster, credentials, deploy_id: deploy_id) MU.log "Removing Load Balancer Target Group #{tg.target_group_name}" retries = 0 begin diff --git a/modules/mu/providers/aws/log.rb b/modules/mu/providers/aws/log.rb index 7556f1292..43eb772ac 100644 --- a/modules/mu/providers/aws/log.rb +++ b/modules/mu/providers/aws/log.rb @@ -202,14 +202,14 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::Log.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Log artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster log_groups = self.find(credentials: credentials, region: region).values if !log_groups.empty? log_groups.each{ |lg| - if lg.log_group_name.match(MU.deploy_id) + if lg.log_group_name.match(deploy_id) log_streams = MU::Cloud::AWS.cloudwatchlogs(credentials: credentials, region: region).describe_log_streams(log_group_name: lg.log_group_name).log_streams if !log_streams.empty? log_streams.each{ |ls| @@ -232,7 +232,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent # unless noop # MU::Cloud::AWS.iam(credentials: credentials).list_roles.roles.each{ |role| -# match_string = "#{MU.deploy_id}.*CloudTrail" +# match_string = "#{deploy_id}.*CloudTrail" # Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud.resourceClass("AWS", "Server"). # MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(match_string) # } diff --git a/modules/mu/providers/aws/msg_queue.rb b/modules/mu/providers/aws/msg_queue.rb index 2f8ae9f2b..bf95c0159 100644 --- a/modules/mu/providers/aws/msg_queue.rb +++ b/modules/mu/providers/aws/msg_queue.rb @@ -133,12 +133,12 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::MsgQueue.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS MsgQueue artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster resp = MU::Cloud::AWS.sqs(credentials: credentials, region: region).list_queues( - queue_name_prefix: MU.deploy_id + queue_name_prefix: deploy_id ) if resp and resp.queue_urls threads = [] diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index b3b19ffee..5448af509 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -172,7 +172,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::NoSQLDb.cleanup: need to support flags['known']", MU::DEBUG, details: flags resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tables @@ -192,7 +192,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent deploy_match = false master_match = false tags.tags.each { |tag| - if tag.key == "MU-ID" and tag.value == MU.deploy_id + if tag.key == "MU-ID" and tag.value == deploy_id deploy_match = true elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip master_match = true diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index ba7793036..fe766fa4d 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -68,12 +68,12 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::Notifier.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Notifier artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster MU::Cloud::AWS.sns(region: region, credentials: credentials).list_topics.topics.each { |topic| - if topic.topic_arn.match(MU.deploy_id) + if topic.topic_arn.match(deploy_id) # We don't have a way to tag our SNS topics, so we will delete any topic that has the MU-ID in its ARN. # This may fail to find notifier groups in some cases (eg. cache_cluster) so we might want to delete from each API as well. MU.log "Deleting SNS topic: #{topic.topic_arn}" diff --git a/modules/mu/providers/aws/role.rb b/modules/mu/providers/aws/role.rb index 3bd216d2d..9e86c6faf 100644 --- a/modules/mu/providers/aws/role.rb +++ b/modules/mu/providers/aws/role.rb @@ -434,14 +434,14 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies( - path_prefix: "/"+MU.deploy_id+"/" + path_prefix: "/"+deploy_id+"/" ) if resp and resp.policies resp.policies.each { |policy| - MU.log "Deleting IAM policy /#{MU.deploy_id}/#{policy.policy_name}" + MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}" if !noop purgePolicy(policy.arn, credentials) end @@ -452,7 +452,7 @@ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) roles = MU::Cloud::AWS::Role.find(credentials: credentials).values roles.each { |r| next if !r.respond_to?(:role_name) - if r.path.match(/^\/#{Regexp.quote(MU.deploy_id)}/) + if r.path.match(/^\/#{Regexp.quote(deploy_id)}/) deleteme << r next end @@ -464,7 +464,7 @@ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) master_match = false deploy_match = false desc.role.tags.each { |t| - if t.key == "MU-ID" and t.value == MU.deploy_id + if t.key == "MU-ID" and t.value == deploy_id deploy_match = true elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip master_match = true diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index cd8dba0c6..d90883f22 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -122,7 +122,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::SearchDomain.cleanup: need to support flags['known']", MU::DEBUG, details: flags list = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_domain_names @@ -138,7 +138,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent deploy_match = false master_match = false tags.tag_list.each { |tag| - if tag.key == "MU-ID" and tag.value == MU.deploy_id + if tag.key == "MU-ID" and tag.value == deploy_id deploy_match = true elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip master_match = true @@ -160,7 +160,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent resp = MU::Cloud::AWS.iam(credentials: credentials).list_roles(marker: marker) resp.roles.each{ |role| # XXX Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud.resourceClass("AWS", "Server"). -# MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(MU.deploy_id)}/) +# MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(deploy_id)}/) } marker = resp.marker end while resp.is_truncated diff --git a/modules/mu/providers/aws/server.rb b/modules/mu/providers/aws/server.rb index 71809f6f8..f155d9408 100644 --- a/modules/mu/providers/aws/server.rb +++ b/modules/mu/providers/aws/server.rb @@ -89,7 +89,7 @@ def initialize(**args) template_variables: { "deployKey" => Base64.urlsafe_encode64(@deploy.public_key), "deploySSHKey" => @deploy.ssh_public_key, - "muID" => MU.deploy_id, + "muID" => @deploy.deploy_id, "muUser" => MU.mu_user, "publicIP" => MU.mu_public_ip, "mommaCatPort" => MU.mommaCatPort, @@ -1454,11 +1454,11 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) onlycloud = flags["onlycloud"] skipsnapshots = flags["skipsnapshots"] tagfilters = [ - {name: "tag:MU-ID", values: [MU.deploy_id]} + {name: "tag:MU-ID", values: [deploy_id]} ] if !ignoremaster tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]} @@ -1492,7 +1492,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent threads << Thread.new(instance) { |myinstance| MU.dupGlobals(parent_thread_id) Thread.abort_on_exception = true - MU::Cloud::AWS::Server.terminateInstance(id: myinstance.instance_id, noop: noop, onlycloud: onlycloud, region: region, deploy_id: MU.deploy_id, credentials: credentials) + MU::Cloud::AWS::Server.terminateInstance(id: myinstance.instance_id, noop: noop, onlycloud: onlycloud, region: region, deploy_id: deploy_id, credentials: credentials) } } @@ -1503,7 +1503,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent threads << Thread.new(volume) { |myvolume| MU.dupGlobals(parent_thread_id) Thread.abort_on_exception = true - delete_volume(myvolume, noop, skipsnapshots, credentials: credentials) + delete_volume(myvolume, noop, skipsnapshots, credentials: credentials, deploy_id: deploy_id) } } @@ -1889,7 +1889,7 @@ def self.imageTimeStamp(ami_id, credentials: nil, region: nil) # @param volume [OpenStruct]: The cloud provider's description of the volume. # @param region [String]: The cloud provider region # @return [void] - def self.delete_volume(volume, noop, skipsnapshots, region: MU.curRegion, credentials: nil) + def self.delete_volume(volume, noop, skipsnapshots, region: MU.curRegion, credentials: nil, deploy_id: MU.deploy_id) if !volume.nil? resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_volumes(volume_ids: [volume.volume_id]) volume = resp.data.volumes.first @@ -1904,9 +1904,9 @@ def self.delete_volume(volume, noop, skipsnapshots, region: MU.curRegion, creden if !noop if !skipsnapshots if !name.nil? and !name.empty? - desc = "#{MU.deploy_id}-MUfinal (#{name})" + desc = "#{deploy_id}-MUfinal (#{name})" else - desc = "#{MU.deploy_id}-MUfinal" + desc = "#{deploy_id}-MUfinal" end begin diff --git a/modules/mu/providers/aws/server_pool.rb b/modules/mu/providers/aws/server_pool.rb index 05135b5d5..38ba19bba 100644 --- a/modules/mu/providers/aws/server_pool.rb +++ b/modules/mu/providers/aws/server_pool.rb @@ -1050,7 +1050,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::ServerPool.cleanup: need to support flags['known']", MU::DEBUG, details: flags filters = [{name: "key", values: ["MU-ID"]}] @@ -1073,7 +1073,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent if asg.key == "MU-MASTER-IP" and asg.value != MU.mu_public_ip and !ignoremaster no_purge << asg.resource_id end - if asg.key == "MU-ID" and asg.value == MU.deploy_id + if asg.key == "MU-ID" and asg.value == deploy_id maybe_purge << asg.resource_id end } diff --git a/modules/mu/providers/aws/storage_pool.rb b/modules/mu/providers/aws/storage_pool.rb index 5e33029a3..8f8121ad1 100644 --- a/modules/mu/providers/aws/storage_pool.rb +++ b/modules/mu/providers/aws/storage_pool.rb @@ -333,7 +333,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region in which to operate # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::StoragePool.cleanup: need to support flags['known']", MU::DEBUG, details: flags supported_regions = %w{us-west-2 us-east-1 eu-west-1} @@ -358,7 +358,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent found_muid = false found_master = false tags.each { |tag| - found_muid = true if tag.key == "MU-ID" && tag.value == MU.deploy_id + found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip } next if !found_muid diff --git a/modules/mu/providers/aws/user.rb b/modules/mu/providers/aws/user.rb index b8725e3a5..c75a8a585 100644 --- a/modules/mu/providers/aws/user.rb +++ b/modules/mu/providers/aws/user.rb @@ -190,16 +190,16 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) MU.log "AWS::User.cleanup: need to support flags['known']", MU::DEBUG, details: flags # XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies( - path_prefix: "/"+MU.deploy_id+"/" + path_prefix: "/"+deploy_id+"/" ) if resp and resp.policies resp.policies.each { |policy| - MU.log "Deleting policy /#{MU.deploy_id}/#{policy.policy_name}" + MU.log "Deleting policy /#{deploy_id}/#{policy.policy_name}" if !noop attachments = begin MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy( @@ -277,7 +277,7 @@ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) has_ourdeploy = false has_ourmaster = false tags.each { |tag| - if tag.key == "MU-ID" and tag.value == MU.deploy_id + if tag.key == "MU-ID" and tag.value == deploy_id has_ourdeploy = true elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip has_ourmaster = true diff --git a/modules/mu/providers/aws/vpc.rb b/modules/mu/providers/aws/vpc.rb index 02b791351..304e0d744 100644 --- a/modules/mu/providers/aws/vpc.rb +++ b/modules/mu/providers/aws/vpc.rb @@ -822,11 +822,11 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) MU.log "AWS::VPC.cleanup: need to support flags['known']", MU::DEBUG, details: flags tagfilters = [ - {name: "tag:MU-ID", values: [MU.deploy_id]} + {name: "tag:MU-ID", values: [deploy_id]} ] if !ignoremaster tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]} @@ -876,7 +876,7 @@ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credent # unless noop # MU::Cloud::AWS.iam.list_roles.roles.each{ |role| -# match_string = "#{MU.deploy_id}.*TRAFFIC-LOG" +# match_string = "#{deploy_id}.*TRAFFIC-LOG" # } # end end diff --git a/modules/mu/providers/azure/server.rb b/modules/mu/providers/azure/server.rb index 638b5e7e8..9e577b67a 100644 --- a/modules/mu/providers/azure/server.rb +++ b/modules/mu/providers/azure/server.rb @@ -452,7 +452,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) end # Cloud-specific configuration properties. diff --git a/modules/mu/providers/google/bucket.rb b/modules/mu/providers/google/bucket.rb index e1d8cc963..fcd89002d 100644 --- a/modules/mu/providers/google/bucket.rb +++ b/modules/mu/providers/google/bucket.rb @@ -144,7 +144,7 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['habitat']) diff --git a/modules/mu/providers/google/container_cluster.rb b/modules/mu/providers/google/container_cluster.rb index 7a3946961..70e5c27cf 100644 --- a/modules/mu/providers/google/container_cluster.rb +++ b/modules/mu/providers/google/container_cluster.rb @@ -744,7 +744,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region in which to operate # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials) diff --git a/modules/mu/providers/google/database.rb b/modules/mu/providers/google/database.rb index cb3e9f5ec..e824ee2b7 100644 --- a/modules/mu/providers/google/database.rb +++ b/modules/mu/providers/google/database.rb @@ -100,7 +100,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region in which to operate # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) # instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['habitat'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"}) diff --git a/modules/mu/providers/google/firewall_rule.rb b/modules/mu/providers/google/firewall_rule.rb index 59bf5600e..cc5643727 100644 --- a/modules/mu/providers/google/firewall_rule.rb +++ b/modules/mu/providers/google/firewall_rule.rb @@ -207,7 +207,7 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials) filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")} diff --git a/modules/mu/providers/google/folder.rb b/modules/mu/providers/google/folder.rb index c0b3b0ae1..f27a4cd4b 100644 --- a/modules/mu/providers/google/folder.rb +++ b/modules/mu/providers/google/folder.rb @@ -162,7 +162,7 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")} if !ignoremaster and MU.mu_public_ip filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")} diff --git a/modules/mu/providers/google/function.rb b/modules/mu/providers/google/function.rb index 1ee04b2c9..8c87ab4a0 100644 --- a/modules/mu/providers/google/function.rb +++ b/modules/mu/providers/google/function.rb @@ -233,7 +233,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials) # Make sure we catch regional *and* zone functions diff --git a/modules/mu/providers/google/group.rb b/modules/mu/providers/google/group.rb index b7bd180d6..a0c3af1b9 100644 --- a/modules/mu/providers/google/group.rb +++ b/modules/mu/providers/google/group.rb @@ -140,7 +140,7 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) MU::Cloud::Google.getDomains(credentials) my_org = MU::Cloud::Google.getOrg(credentials) diff --git a/modules/mu/providers/google/habitat.rb b/modules/mu/providers/google/habitat.rb index ec4b86538..62c4d19f5 100644 --- a/modules/mu/providers/google/habitat.rb +++ b/modules/mu/providers/google/habitat.rb @@ -222,7 +222,7 @@ def self.isLive?(project_id, credentials = nil) # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects if resp and resp.projects diff --git a/modules/mu/providers/google/loadbalancer.rb b/modules/mu/providers/google/loadbalancer.rb index 3a4a91e72..a8339a4db 100644 --- a/modules/mu/providers/google/loadbalancer.rb +++ b/modules/mu/providers/google/loadbalancer.rb @@ -146,7 +146,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: nil, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: nil, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials) filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")} diff --git a/modules/mu/providers/google/role.rb b/modules/mu/providers/google/role.rb index 3d0ff10bc..e199843c3 100644 --- a/modules/mu/providers/google/role.rb +++ b/modules/mu/providers/google/role.rb @@ -465,7 +465,7 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) customer = MU::Cloud::Google.customerID(credentials) my_org = MU::Cloud::Google.getOrg(credentials) diff --git a/modules/mu/providers/google/server.rb b/modules/mu/providers/google/server.rb index 30fff8fbf..6b071256e 100644 --- a/modules/mu/providers/google/server.rb +++ b/modules/mu/providers/google/server.rb @@ -1290,7 +1290,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials) diff --git a/modules/mu/providers/google/server_pool.rb b/modules/mu/providers/google/server_pool.rb index 3ae7fb757..eb8643a7f 100644 --- a/modules/mu/providers/google/server_pool.rb +++ b/modules/mu/providers/google/server_pool.rb @@ -431,7 +431,7 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials) filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")} diff --git a/modules/mu/providers/google/user.rb b/modules/mu/providers/google/user.rb index 6e07c36c6..599e5c6dd 100644 --- a/modules/mu/providers/google/user.rb +++ b/modules/mu/providers/google/user.rb @@ -254,7 +254,7 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) MU::Cloud::Google.getDomains(credentials) my_org = MU::Cloud::Google.getOrg(credentials) diff --git a/modules/mu/providers/google/vpc.rb b/modules/mu/providers/google/vpc.rb index bb11b7272..c3ebd78aa 100644 --- a/modules/mu/providers/google/vpc.rb +++ b/modules/mu/providers/google/vpc.rb @@ -537,7 +537,7 @@ def self.quality # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] - def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials) return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials) filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")} From cf3af21ddde7716dddab27fa260b5194739affdb Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 10 Jul 2020 13:10:54 -0400 Subject: [PATCH 022/107] AWS: handle it better if someone calls notify() before the resource is created --- modules/mu/providers/aws/container_cluster.rb | 1 + modules/mu/providers/aws/endpoint.rb | 1 + modules/mu/providers/aws/group.rb | 1 + modules/mu/providers/aws/loadbalancer.rb | 1 + modules/mu/providers/aws/msg_queue.rb | 1 + modules/mu/providers/aws/search_domain.rb | 1 + modules/mu/providers/aws/server.rb | 1 + modules/mu/providers/aws/server_pool.rb | 1 + 8 files changed, 8 insertions(+) diff --git a/modules/mu/providers/aws/container_cluster.rb b/modules/mu/providers/aws/container_cluster.rb index c670f89c8..7d997297e 100644 --- a/modules/mu/providers/aws/container_cluster.rb +++ b/modules/mu/providers/aws/container_cluster.rb @@ -287,6 +287,7 @@ def self.tasksRunning?(cluster, log: true, region: MU.myRegion, credentials: nil # @return [OpenStruct] def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@cloud_id @cloud_desc_cache = if @config['flavor'] == "EKS" or (@config['flavor'] == "Fargate" and !@config['containers']) resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).describe_cluster( diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index bed50749a..56630fd3b 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -226,6 +226,7 @@ def groom # @return [Struct] def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@cloud_id @cloud_desc_cache = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_rest_api( rest_api_id: @cloud_id ) diff --git a/modules/mu/providers/aws/group.rb b/modules/mu/providers/aws/group.rb index 7442e18fc..52e3131f7 100644 --- a/modules/mu/providers/aws/group.rb +++ b/modules/mu/providers/aws/group.rb @@ -155,6 +155,7 @@ def arn # return [Struct] def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@mu_name @cloud_desc_cache = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group( group_name: @mu_name ) diff --git a/modules/mu/providers/aws/loadbalancer.rb b/modules/mu/providers/aws/loadbalancer.rb index 750dd39fc..485c138e2 100644 --- a/modules/mu/providers/aws/loadbalancer.rb +++ b/modules/mu/providers/aws/loadbalancer.rb @@ -583,6 +583,7 @@ def arn # Wrapper for cloud_desc method that deals with elb vs. elb2 resources. def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@cloud_id if @config['classic'] @cloud_desc_cache = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers( load_balancer_names: [@cloud_id] diff --git a/modules/mu/providers/aws/msg_queue.rb b/modules/mu/providers/aws/msg_queue.rb index bf95c0159..ed1aebe01 100644 --- a/modules/mu/providers/aws/msg_queue.rb +++ b/modules/mu/providers/aws/msg_queue.rb @@ -80,6 +80,7 @@ def arn # @return [Hash]: AWS doesn't return anything but the SQS URL, so supplement with attributes def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@cloud_id if !@cloud_id resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).list_queues( diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index d90883f22..b690b065f 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -63,6 +63,7 @@ def groom # our druthers. def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@cloud_id @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( domain_name: @cloud_id ).domain_status diff --git a/modules/mu/providers/aws/server.rb b/modules/mu/providers/aws/server.rb index f155d9408..88beaf8f3 100644 --- a/modules/mu/providers/aws/server.rb +++ b/modules/mu/providers/aws/server.rb @@ -891,6 +891,7 @@ def arn # @return [Openstruct] def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@cloud_id max_retries = 5 retries = 0 if !@cloud_id.nil? diff --git a/modules/mu/providers/aws/server_pool.rb b/modules/mu/providers/aws/server_pool.rb index 38ba19bba..f3d59c84e 100644 --- a/modules/mu/providers/aws/server_pool.rb +++ b/modules/mu/providers/aws/server_pool.rb @@ -425,6 +425,7 @@ def groom # @return [OpenStruct] def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + return nil if !@cloud_id @cloud_desc_cache = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups( auto_scaling_group_names: [@mu_name] ).auto_scaling_groups.first From b045d8402218175397905a4ca6c620cbb827a482 Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 10 Jul 2020 19:37:59 -0400 Subject: [PATCH 023/107] AWS::SearchDomain: don't blow up if someone calls notify early --- modules/mu/providers/aws/search_domain.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index b690b065f..83f8715cd 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -90,6 +90,7 @@ def arn # Return the metadata for this SearchDomain rule # @return [Hash] def notify + return nil if !cloud_desc deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: arn).tag_list deploy_struct['tags'] = tags.map { |t| { t.key => t.value } } From 4a6d105ce20b88e1a0a02d693a3974abb1802400 Mon Sep 17 00:00:00 2001 From: John Stange Date: Sat, 11 Jul 2020 00:39:32 -0400 Subject: [PATCH 024/107] AWS::SearchDomain: still fidgeting with the order on the create process --- modules/mu/providers/aws/search_domain.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 83f8715cd..93fac5469 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -35,6 +35,7 @@ def create params = genParams MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params + @cloud_id = @config['domain_name'] MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).create_elasticsearch_domain(params).domain_status tagDomain From 6fc9069d8ae90d3fb40e4b503433d46a0b960ad1 Mon Sep 17 00:00:00 2001 From: John Stange Date: Sat, 11 Jul 2020 13:25:17 -0400 Subject: [PATCH 025/107] AWS::Function: notify fixups for early calls --- modules/mu/providers/aws/function.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 9fcb1c5ed..5976f4703 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -207,10 +207,8 @@ def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda' # Return the metadata for this Function rule # @return [Hash] def notify - deploy_struct = MU.structToHash(MU::Cloud::AWS::Function.find(cloud_id: @cloud_id, credentials: @credentials, region: @config['region']).values.first, stringify_keys: true) - - deploy_struct['mu_name'] = @mu_name - return deploy_struct + return nil if !cloud_desc + MU.structToHash(cloud_desc, stringify_keys: true) end # Does this resource type exist as a global (cloud-wide) artifact, or From ece0831b14e0a814adf038e8617656bb4c29cc1b Mon Sep 17 00:00:00 2001 From: John Stange Date: Sat, 11 Jul 2020 19:37:24 -0400 Subject: [PATCH 026/107] AWS: teach magic API pagination about Lambda's weird marker attributes --- modules/mu/providers/aws.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index 951e632cf..194063272 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -1481,13 +1481,17 @@ def method_missing(method_sym, *arguments) if !retval.nil? begin - page_markers = [:marker, :next_token] + page_markers = { + :marker => :marker, + :next_token => :next_token, + :next_marker => :marker + } paginator = nil new_page = nil - [:next_token, :marker].each { |m| + page_markers.each_key { |m| if !retval.nil? and retval.respond_to?(m) paginator = m - new_page = retval.send(paginator) + new_page = retval.send(m) break end } @@ -1506,12 +1510,12 @@ def method_missing(method_sym, *arguments) if new_args.is_a?(Array) new_args << {} if new_args.empty? if new_args.size == 1 and new_args.first.is_a?(Hash) - new_args[0][paginator] = new_page + new_args[0][page_markers[paginator]] = new_page else MU.log "I don't know how to insert a #{paginator} into these arguments for #{method_sym}", MU::WARN, details: new_args end elsif new_args.is_a?(Hash) - new_args[paginator] = new_page + new_args[page_markers[paginator]] = new_page end MU.log "Attempting magic pagination for #{method_sym}", MU::DEBUG, details: new_args From 62b727695df9bf9eb34637ae299c0982d81ed3ea Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 12 Jul 2020 02:22:32 -0400 Subject: [PATCH 027/107] AWS::FirewallRule: do some brain-dead defaulting because somehow the config parser isn't, but only in the container in gitlab because of course it would break in a sealed box --- modules/mu/providers/aws/firewall_rule.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/mu/providers/aws/firewall_rule.rb b/modules/mu/providers/aws/firewall_rule.rb index 0edc28017..a23c4c697 100644 --- a/modules/mu/providers/aws/firewall_rule.rb +++ b/modules/mu/providers/aws/firewall_rule.rb @@ -860,8 +860,11 @@ def convertToEc2(rules) p_start = rule['port'].to_i p_end = rule['port'].to_i elsif rule['proto'] != "icmp" - raise MuError, "Can't create a TCP or UDP security group rule without specifying ports: #{rule}" + MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule + p_start = "0" + p_end = "65535" end + if rule['proto'] != "icmp" if p_start.nil? or p_end.nil? raise MuError, "Got nil ports out of rule #{rule}" From b9d2dfa69726d39bb45c1ccaaba9467cd70d06eb Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 12 Jul 2020 15:25:47 -0400 Subject: [PATCH 028/107] AWS::SearchDomain: yet more grace for early notify calls --- modules/mu/providers/aws/search_domain.rb | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 93fac5469..fcce95c1a 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -56,6 +56,7 @@ def groom end waitWhileProcessing # don't return until creation/updating is complete + MU.log "Search Domain #{@config['name']}: #{cloud_desc.endpoint}", MU::SUMMARY end @cloud_desc_cache = nil @@ -64,21 +65,14 @@ def groom # our druthers. def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache + @cloud_id ||= @config['domain_name'] return nil if !@cloud_id - @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( + MU.retrier([::Aws::ElasticsearchService::Errors::ResourceNotFoundException], wait: 10, max: 12) { + @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( domain_name: @cloud_id ).domain_status -# if @config['domain_name'] -# MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( -# domain_name: @config['domain_name'] -# ).domain_status -# elsif @deploydata and @deploydata['domain_name'] -# MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain( -# domain_name: @deploydata['domain_name'] -# ).domain_status -# else -# raise MuError, "#{@mu_name} can't find its official Elasticsearch domain name!" -# end + } + @cloud_desc_cache end From 51951e0b1a634479dc3ff779a2324976ea91f6c3 Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 12 Jul 2020 16:50:43 -0400 Subject: [PATCH 029/107] AWS::SearchDomain: yet more grace for early notify calls --- modules/mu/providers/aws/endpoint.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 56630fd3b..45bed2f6b 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -236,6 +236,7 @@ def cloud_desc(use_cache: true) # Return the metadata for this API # @return [Hash] def notify + return nil if !@cloud_id or !cloud_desc deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) deploy_struct['url'] = "https://"+@cloud_id+".execute-api."+@config['region']+".amazonaws.com/"+@config['deploy_to'] # XXX stages and whatnot From 0ce7714031f6d49a3837da4d7c02f9019bd2a203 Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 12 Jul 2020 19:12:04 -0400 Subject: [PATCH 030/107] AWS::Bucket: add a MU::SUMMARY line at end of groom --- modules/mu/providers/aws/bucket.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index d014d636e..ef3f68f59 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -167,6 +167,8 @@ def groom } } end + + MU.log "Bucket #{@config['name']}: s3://#{@cloud_id}", MU::SUMMARY end # Upload a file to a bucket. From 409986ab55e3321d03d88054cb7aa1207794615b Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 12 Jul 2020 20:28:26 -0400 Subject: [PATCH 031/107] don't have a nutty if a notify returns a nil --- modules/mu/cloud/resource_base.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/mu/cloud/resource_base.rb b/modules/mu/cloud/resource_base.rb index 309734669..88f09d8a8 100644 --- a/modules/mu/cloud/resource_base.rb +++ b/modules/mu/cloud/resource_base.rb @@ -893,10 +893,11 @@ def resourceInitHook elsif method == :notify if retval.nil? MU.log self.to_s+" didn't return any metadata from notify", MU::WARN, details: @cloudobj.cloud_desc + else + retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil? + retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil? + @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil? end - retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil? - retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil? - @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil? end @method_semaphore.synchronize { @method_locks.delete(method) From 80509565789139f06d3052cd2f3db553a686a9d9 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 13 Jul 2020 13:22:58 -0400 Subject: [PATCH 032/107] AWS::Notifier: notify resilience --- modules/mu/providers/aws/notifier.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index fe766fa4d..cca55b7d1 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -94,8 +94,13 @@ def arn # Return the metadata for this user cofiguration # @return [Hash] def notify - desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes - MU.structToHash(desc) + return nil if !@cloud_id + begin + desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes + MU.structToHash(desc) + rescue ::Aws::SNS::Errors::NotFound + nil + end end # Locate an existing notifier. From 07d8ffa3a1b582720fb30f52119690b20aebd807 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 13 Jul 2020 13:42:01 -0400 Subject: [PATCH 033/107] AWS::Bucket: helpfully include URL in summary output for web-enabled buckets --- modules/mu/providers/aws/bucket.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index ef3f68f59..fd2246ce2 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -169,6 +169,9 @@ def groom end MU.log "Bucket #{@config['name']}: s3://#{@cloud_id}", MU::SUMMARY + if @config['web'] + MU.log "Bucket #{@config['name']} web access: http://#{@cloud_id}.s3-website-#{@config['region']}.amazonaws.com/", MU::SUMMARY + end end # Upload a file to a bucket. From a9a88cb0d2f921f98a69347a4dc7f45c12a008f7 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 13 Jul 2020 16:35:13 -0400 Subject: [PATCH 034/107] AWS::Notifier: actually let's have a little less resilience, because notify() should work if @cloud_id is set --- modules/mu/providers/aws/notifier.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index cca55b7d1..94a998b2c 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -95,12 +95,8 @@ def arn # @return [Hash] def notify return nil if !@cloud_id - begin - desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes - MU.structToHash(desc) - rescue ::Aws::SNS::Errors::NotFound - nil - end + desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes + MU.structToHash(desc) end # Locate an existing notifier. From af480c0ad7dde91f48824c728bb80370d4e2b1d5 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 13 Jul 2020 20:01:05 -0400 Subject: [PATCH 035/107] AWS::Function: make an effort to actually assign the role we ask for --- modules/mu/providers/aws/function.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 5976f4703..af712fa63 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -505,7 +505,7 @@ def self.validateConfig(function, configurator) private def get_properties - role_arn = get_role_arn(@config['iam_role']) + role_arn = MU::Config::Ref.get(@config['iam_role']).arn lambda_properties = { code: {}, From d99ce53e67a4996796e00476f96825484387bbac Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 13 Jul 2020 21:33:14 -0400 Subject: [PATCH 036/107] AWS::Function: apparently role lookup was a hot mess --- modules/mu/config/ref.rb | 13 ++++---- modules/mu/providers/aws/function.rb | 45 +++++++++++++--------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/modules/mu/config/ref.rb b/modules/mu/config/ref.rb index efae39fcf..c93d8f9e0 100644 --- a/modules/mu/config/ref.rb +++ b/modules/mu/config/ref.rb @@ -273,8 +273,9 @@ def cloud_id # called in a live deploy, which is to say that if called during initial # configuration parsing, results may be incorrect. # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches. - def kitten(mommacat = @mommacat, shallow: false, debug: false) - return nil if !@cloud or !@type + def kitten(mommacat = @mommacat, shallow: false, debug: false, cloud: nil) + cloud ||= @cloud + return nil if !cloud or !@type loglevel = debug ? MU::NOTICE : MU::DEBUG if debug @@ -320,7 +321,7 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) end end - if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(@id)) and !shallow + if !@obj and !(cloud == "Google" and @id and @type == "users" and MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(@id)) and !shallow try_deploy_id = @deploy_id begin @@ -335,7 +336,7 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) end MU.log "Ref#kitten calling findStray", loglevel, details: { - cloud: @cloud, + cloud: cloud, type: @type, name: @name, cloud_id: @id, @@ -347,7 +348,7 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) } found = MU::MommaCat.findStray( - @cloud, + cloud, @type, name: @name, cloud_id: @id, @@ -361,7 +362,7 @@ def kitten(mommacat = @mommacat, shallow: false, debug: false) @obj ||= found.first if found rescue MU::MommaCat::MultipleMatches => e if try_deploy_id.nil? and MU.deploy_id - MU.log "Attempting to narrow down #{@cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE + MU.log "Attempting to narrow down #{cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE try_deploy_id = MU.deploy_id retry else diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index af712fa63..99beec77e 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -465,7 +465,10 @@ def self.validateConfig(function, configurator) MU::Config.addDependency(function, fwname, "firewall_rule") end - if !function['iam_role'] + function['role'] ||= function['iam_role'] + function.delete("iam_role") + + if !function['role'] policy_map = { "basic" => "AWSLambdaBasicExecutionRole", "kinesis" => "AWSLambdaKinesisExecutionRole", @@ -494,9 +497,21 @@ def self.validateConfig(function, configurator) } configurator.insertKitten(roledesc, "roles") - function['iam_role'] = function['name']+"execrole" + function['role'] = function['name']+"execrole" + + end + + if function['role'].is_a?(String) + function['role'] = MU::Config::Ref.get( + name: function['role'], + type: "roles", + cloud: "AWS", + credentials: function['credentials'] + ) + end - MU::Config.addDependency(function, function['name']+"execrole", "role") + if function['role']['name'] + MU::Config.addDependency(function, function['role']['name'], "role") end ok @@ -505,14 +520,15 @@ def self.validateConfig(function, configurator) private def get_properties - role_arn = MU::Config::Ref.get(@config['iam_role']).arn + role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS") + raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj lambda_properties = { code: {}, function_name: @mu_name, handler: @config['handler'], publish: true, - role: role_arn, + role: role_obj.arn, runtime: @config['runtime'], } @@ -605,25 +621,6 @@ def get_properties lambda_properties end - # Given an IAM role name, resolve to ARN. Will attempt to identify any - # sibling Mu role resources by this name first, and failing that, will - # do a plain get_role() to the IAM API for the provided name. - # @param name [String] - def get_role_arn(name) - sib_role = @deploy.findLitterMate(name: name, type: "roles") - return sib_role.cloudobj.arn if sib_role - - begin - role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({ - role_name: name.to_s - }) - return role['role']['arn'] - rescue StandardError => e - MU.log "#{e}", MU::ERR - end - nil - end - end end end From 71681e4681167efdcb293c70fee9b58bbbedc322 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 14 Jul 2020 01:44:30 -0400 Subject: [PATCH 037/107] AWS::Job: a new resource type! so's I have somewhere to put CloudWatch Events --- modules/mu/cloud.rb | 14 +++ modules/mu/cloud/dnszone.rb | 2 - modules/mu/config/job.rb | 89 ++++++++++++++++++ modules/mu/providers/aws.rb | 10 ++ modules/mu/providers/aws/job.rb | 161 ++++++++++++++++++++++++++++++++ 5 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 modules/mu/config/job.rb create mode 100644 modules/mu/providers/aws/job.rb diff --git a/modules/mu/cloud.rb b/modules/mu/cloud.rb index cfa30870f..bea6769ec 100644 --- a/modules/mu/cloud.rb +++ b/modules/mu/cloud.rb @@ -148,6 +148,9 @@ class Bucket; # Stub base class; real implementations generated at runtime class NoSQLDB; end + # Stub base class; real implementations generated at runtime + class Job; + end # Denotes a resource implementation which is missing significant # functionality or is largely untested. @@ -439,6 +442,17 @@ class NoSQLDB; :waits_on_parent_completion => true, :class => @@generic_class_methods, :instance => @@generic_instance_methods + [:groom] + }, + :Job => { + :has_multiples => false, + :can_live_in_vpc => false, + :cfg_name => "job", + :cfg_plural => "jobs", + :interface => self.const_get("Job"), + :deps_wait_on_my_creation => true, + :waits_on_parent_completion => true, + :class => @@generic_class_methods, + :instance => @@generic_instance_methods + [:groom] } }.freeze diff --git a/modules/mu/cloud/dnszone.rb b/modules/mu/cloud/dnszone.rb index 59f4fcdb1..3d6bcb192 100644 --- a/modules/mu/cloud/dnszone.rb +++ b/modules/mu/cloud/dnszone.rb @@ -29,8 +29,6 @@ def self.genericMuDNSEntry(*flags) # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all # requested records in background and returns immediately. - # @param cfg [Array]: An array of parsed {MU::Config::BasketofKittens::dnszones::records} objects. - # @param target [String]: Optional target for the records to be created. Overrides targets embedded in cfg records. def self.createRecordsFromConfig(*flags) cloudclass = MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone") if !flags.nil? and flags.size == 1 diff --git a/modules/mu/config/job.rb b/modules/mu/config/job.rb new file mode 100644 index 000000000..32fd94732 --- /dev/null +++ b/modules/mu/config/job.rb @@ -0,0 +1,89 @@ +# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved +# +# Licensed under the BSD-3 license (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the root of the project or at +# +# http://egt-labs.com/mu/LICENSE.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module MU + class Config + # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/job.rb + class Job + + # Base configuration schema for a scheduled job + # @return [Hash] + def self.schema + { + "type" => "object", + "additionalProperties" => false, + "description" => "A cloud provider-specific facility for triggered or scheduled tasks, such as AWS CloudWatch Events or Google Cloud Scheduler.", + "properties" => { + "name" => { + "type" => "string" + }, + "region" => MU::Config.region_primitive, + "credentials" => MU::Config.credentials_primitive, + "description" => { + "type" => "string", + "description" => "Human-readable description field for this job (this will field be overriden with the Mu deploy id on most providers unless +scrub_mu_isms+ is set)" + }, + "schedule" => { + "type" => "object", + "description" => "A schedule on which to invoke this task, typically unix crontab style.", + "properties" => { + "minute" => { + "type" => "string", + "description" => "The minute of the hour at which to invoke this job, typically an integer between 0 and 59. This will be validated by the cloud provider, where other more human-readable values may be supported.", + "default" => "0" + }, + "hour" => { + "type" => "string", + "description" => "The hour at which to invoke this job, typically an integer between 0 and 23. This will be validated by the cloud provider, where other more human-readable values may be supported.", + "default" => "*" + }, + "day_of_month" => { + "type" => "string", + "description" => "The day of the month which to invoke this job, typically an integer between 1 and 31. This will be validated by the cloud provider, where other more human-readable values may be supported.", + "default" => "*" + }, + "month" => { + "type" => "string", + "description" => "The month in which to invoke this job, typically an integer between 1 and 12. This will be validated by the cloud provider, where other more human-readable values may be supported.", + "default" => "*" + }, + "day_of_week" => { + "type" => "string", + "description" => "The day of the week on which to invoke this job, typically an integer between 0 and 6. This will be validated by the cloud provider, where other more human-readable values may be supported.", + "default" => "*" + }, + "year" => { + "type" => "string", + "description" => "The year in which to invoke this job. Not honored by all cloud providers.", + "default" => "*" + } + } + } + } + } + end + + # Generic pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated. + # @param _job [Hash]: The resource to process and validate + # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member + # @return [Boolean]: True if validation succeeded, False otherwise + def self.validate(_job, _configurator) + ok = true + + ok + end + + end + end +end diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index 194063272..b45bd348d 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -1029,6 +1029,14 @@ def self.cloudwatchlogs(region: MU.curRegion, credentials: nil) @@cloudwatchlogs_api[credentials][region] end + # Amazon's CloudWatchEvents API + def self.cloudwatchevents(region: MU.curRegion, credentials: nil) + region ||= myRegion + @@cloudwatchevents_api[credentials] ||= {} + @@cloudwatchevents_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials) + @@cloudwatchevents_api[credentials][region] + end + # Amazon's CloudFront API def self.cloudfront(region: MU.curRegion, credentials: nil) region ||= myRegion @@ -1461,6 +1469,7 @@ def method_missing(method_sym, *arguments) require "aws-sdk-core/ecs" require "aws-sdk-core/eks" require "aws-sdk-core/cloudwatchlogs" + require "aws-sdk-core/cloudwatchevents" require "aws-sdk-core/elasticloadbalancing" require "aws-sdk-core/elasticloadbalancingv2" require "aws-sdk-core/autoscaling" @@ -1581,6 +1590,7 @@ def method_missing(method_sym, *arguments) @@wafglobal = {} @@waf = {} @@cloudwatchlogs_api = {} + @@cloudwatchevents_api = {} @@cloudfront_api = {} @@elasticache_api = {} @@sns_api = {} diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb new file mode 100644 index 000000000..bc5035c7d --- /dev/null +++ b/modules/mu/providers/aws/job.rb @@ -0,0 +1,161 @@ +# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved +# +# Licensed under the BSD-3 license (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the root of the project or at +# +# http://egt-labs.com/mu/LICENSE.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module MU + class Cloud + class AWS + # A scheduled task facility as configured in {MU::Config::BasketofKittens::jobs} + class Job < MU::Cloud::Job + + # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us. + # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat + def initialize(**args) + super + @mu_name ||= @deploy.getResourceName(@config["name"]) + end + + # Called automatically by {MU::Deploy#createResources} + def create + end + + # Canonical Amazon Resource Number for this resource + # @return [String] + def arn + cloud_desc ? cloud_desc.arn : nil + end + + # Return the metadata for this job + # @return [Hash] + def notify + { + } + end + + # Does this resource type exist as a global (cloud-wide) artifact, or + # is it localized to a region/zone? + # @return [Boolean] + def self.isGlobal? + false + end + + # Denote whether this resource implementation is experiment, ready for + # testing, or ready for production use. + def self.quality + MU::Cloud::BETA + end + + # Remove all jobs associated with the currently loaded deployment. + # @param noop [Boolean]: If true, will only print what would be done + # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server + # @param region [String]: The cloud provider region + # @return [void] + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + end + + # Locate an existing event. + # @return [Hash]: The cloud provider's complete descriptions of matching CloudWatch Event + def self.find(**args) + found = {} + + MU::Cloud::AWS.cloudwatchevents(region: args[:region], credentials: args[:credentials]).list_rules.rules.each { |r| + next if args[:cloud_id] and ![r.name, r.arn].include?(args[:cloud_id]) + found[r.name] = r + } + + found + end + + # Reverse-map our cloud description into a runnable config hash. + # We assume that any values we have in +@config+ are placeholders, and + # calculate our own accordingly based on what's live in the cloud. + def toKitten(**_args) + bok = { + "cloud" => "AWS", + "credentials" => @config['credentials'], + "cloud_id" => @cloud_id, + "region" => @config['region'] + } + + if !cloud_desc + MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config + return nil + end + bok['name'] = cloud_desc.name + if cloud_desc.description and !cloud_desc.description.empty? + bok['description'] = cloud_desc.description + end + + bok['disabled'] = true if cloud_desc.state == "DISABLED" + +# schedule_expression="cron(15 6 * * ? *)" + if cloud_desc.schedule_expression + if cloud_desc.schedule_expression.match(/cron\((\S+) (\S+) (\S+) (\S+) (\S+) (\S+)\)/) + bok['schedule'] = { + "minute" => Regexp.last_match[1], + "hour" => Regexp.last_match[2], + "day_of_month" => Regexp.last_match[3], + "month" => Regexp.last_match[4], + "day_of_week" => Regexp.last_match[5], + "year" => Regexp.last_match[6] + } + else + MU.log "HALP", MU::ERR, details: cloud_desc.schedule_expression + end + end + + if cloud_desc.role_arn + shortname = cloud_desc.role_arn.sub(/.*?role\/([^\/]+)$/, '\1') + bok['role'] = MU::Config::Ref.get( + id: shortname, + cloud: "AWS", + type: "roles" + ) + end + +# XXX cloud_desc.event_pattern - what do we want to do with this? + + bok + end + + + # Cloud-specific configuration properties. + # @param _config [MU::Config]: The calling MU::Config object + # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource + def self.schema(_config) + toplevel_required = [] + schema = { + "disabled" => { + "type" => "boolean", + "description" => "Leave this job in place but disabled", + "default" => false + }, + "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to this CloudWatch Event.", omit_fields: ["region", "tag"]), + } + [toplevel_required, schema] + end + + # Cloud-specific pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated. + # @param job [Hash]: The resource to process and validate + # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member + # @return [Boolean]: True if validation succeeded, False otherwise + def self.validateConfig(job, _configurator) + ok = true + + ok + end + + end + end + end +end From fbf2fbb493f273038cc5545a4ca8615fbe765275 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 14 Jul 2020 20:52:43 -0400 Subject: [PATCH 038/107] AWS::Job: create/groom/cleanup lifecycle for CloudWatch Events now, minus targets --- modules/mu/cleanup.rb | 2 +- modules/mu/providers/aws/job.rb | 75 ++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/modules/mu/cleanup.rb b/modules/mu/cleanup.rb index 4d458b340..b1eb86175 100644 --- a/modules/mu/cleanup.rb +++ b/modules/mu/cleanup.rb @@ -32,7 +32,7 @@ class Cleanup # Resource types, in the order in which we generally have to clean them up # to disentangle them from one another. - TYPES_IN_ORDER = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"] + TYPES_IN_ORDER = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "Job", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"] # Purge all resources associated with a deployment. # @param deploy_id [String]: The identifier of the deployment to remove (typically seen in the MU-ID tag on a resource). diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index bc5035c7d..72ed17746 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -27,6 +27,31 @@ def initialize(**args) # Called automatically by {MU::Deploy#createResources} def create + @cloud_id = @mu_name + + params = get_properties + + MU.log "Creating CloudWatch Event #{@mu_name}", MU::NOTICE, details: params + + MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params) + end + + # Called automatically by {MU::Deploy#createResources} + def groom + new_props = get_properties + current = MU.structToHash(cloud_desc(use_cache: false)) + params = {} + new_props.each_pair { |k, v| + next if k == :tags # doesn't seem to do anything + if v != current[k] + params[k] = v + end + } + + if params.size > 0 + MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params + MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params) + end end # Canonical Amazon Resource Number for this resource @@ -38,8 +63,7 @@ def arn # Return the metadata for this job # @return [Hash] def notify - { - } + MU.structToHash(cloud_desc, stringify_keys: true) end # Does this resource type exist as a global (cloud-wide) artifact, or @@ -61,6 +85,21 @@ def self.quality # @param region [String]: The cloud provider region # @return [void] def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + found = find(region: region, credentials: credentials) + + found.each_pair { |id, desc| + if (desc.description and desc.description == deploy_id) or + (flags and flags['known'] and flags['known'].include?(id)) + MU.log "Deleting CloudWatch Event #{id}" + if !noop + # XXX purge all targets first + MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).delete_rule( + name: id, + event_bus_name: desc.event_bus_name + ) + end + end + } end # Locate an existing event. @@ -155,6 +194,38 @@ def self.validateConfig(job, _configurator) ok end + private + + def get_properties + params = { + name: @cloud_id, + state: @config['disabled'] ? "DISABLED" : "ENABLED", + event_bus_name: "default" # XXX expose, or create a deploy-specific one? + } + + params[:description] = if @config['description'] and @config['scrub_mu_isms'] + @config['description'] + else + @deploy.deploy_id + end + + if @tags + params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } } + end + + if @config['role'] + role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS") + raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj + params[:role_arn] = role_obj.arn + end + + if @config['schedule'] + params[:schedule_expression] = "cron(" + ["minute", "hour", "day_of_month", "month", "day_of_week", "year"].map { |i| @config['schedule'][i] }.join(" ") +")" + end + + params + end + end end end From f7320a3ac603c22c492665ba36e63ac55cac2c8b Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 15 Jul 2020 21:31:10 -0400 Subject: [PATCH 039/107] AWS::Job: targets now mostly working, enough to use for sitemonitor's lambdas --- modules/mu/config/ref.rb | 25 +++- modules/mu/providers/aws/endpoint.rb | 1 - modules/mu/providers/aws/function.rb | 1 - modules/mu/providers/aws/job.rb | 197 +++++++++++++++++++++++++++ 4 files changed, 220 insertions(+), 4 deletions(-) diff --git a/modules/mu/config/ref.rb b/modules/mu/config/ref.rb index c93d8f9e0..1486351e4 100644 --- a/modules/mu/config/ref.rb +++ b/modules/mu/config/ref.rb @@ -121,6 +121,10 @@ def initialize(cfg) @deploy_id = @mommacat.deploy_id end + # canonicalize the 'type' argument + _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false) + @type = cfg_plural if cfg_plural + kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this end @@ -157,7 +161,7 @@ def delete(attribute) # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}. # @param aliases [Array]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+). # @return [Hash] - def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: []) + def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [], any_type: false) parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1') desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource" schema = { @@ -212,7 +216,9 @@ def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields } end - if !type.nil? + if any_type + schema["properties"]["type"].delete("enum") + elsif !type.nil? schema["required"] = ["type"] schema["properties"]["type"]["default"] = type schema["properties"]["type"]["enum"] = [type] @@ -232,6 +238,13 @@ def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields schema end + # Is our +@type+ attribute a Mu-supported type, or some rando string? + # @return [Boolean] + def is_mu_type? + _shortclass, _cfg_name, type, _classname, _attrs = MU::Cloud.getResourceNames(@type, false) + !type.nil? + end + # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment, # of the sort that would have been used to declare this reference in the # first place. @@ -276,6 +289,14 @@ def cloud_id def kitten(mommacat = @mommacat, shallow: false, debug: false, cloud: nil) cloud ||= @cloud return nil if !cloud or !@type + + _shortclass, _cfg_name, cfg_plural, _classname, _attrs = MU::Cloud.getResourceNames(@type, false) + if cfg_plural + @type = cfg_plural # make sure this is the thing we expect + else + return nil # we don't do non-muish resources + end + loglevel = debug ? MU::NOTICE : MU::DEBUG if debug diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 45bed2f6b..a91f86cf3 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -166,7 +166,6 @@ def generate_methods } end -MU.log "integration", MU::NOTICE, details: params resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).put_integration(params) if m['integrate_with']['type'] == "function" diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 99beec77e..3cb66a66e 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -67,7 +67,6 @@ def create # Called automatically by {MU::Deploy#createResources} def groom - MU.log @cloud_id, MU::NOTICE, details: cloud_desc old_props = MU.structToHash(cloud_desc) new_props = get_properties diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index 72ed17746..0649f6a76 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -52,6 +52,43 @@ def groom MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params) end + + if @config['targets'] + target_params = [] + @config['targets'].each { |t| + target_ref = MU::Config::Ref.get(t) + target_obj = target_ref.kitten(cloud: "AWS") + this_target = if target_ref.is_mu_type? and target_obj and + !target_obj.arn.nil? + { + id: target_obj.cloud_id, + arn: target_obj.arn + } + else + { + id: target_ref.id || target_ref.name, + arn: target_ref.id + } + end + if t['role'] + role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS") + raise MuError.new "Failed to fetch object from role reference", details: t['role'].to_h if !role_obj + params[:role_arn] = role_obj.arn + end + [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters, :ecs_parameters].each { |attr| + if t[attr.to_s] + this_target[attr] = MU.structToHash(t[attr.to_s]) + end + } + target_params << this_target + } + MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_targets( + rule: @cloud_id, + event_bus_name: cloud_desc.event_bus_name, + targets: target_params + ) + end + end # Canonical Amazon Resource Number for this resource @@ -162,6 +199,47 @@ def toKitten(**_args) ) end + targets = MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).list_targets_by_rule( + rule: @cloud_id, + event_bus_name: cloud_desc.event_bus_name + ).targets + targets.each { |t| + bok['targets'] ||= [] + _arn, _plat, service, region, account, resource = t.arn.split(/:/, 6) + MU.log service+" in "+region+" under "+account, MU::WARN, details: resource + target_type = if service == "lambda" + resource.sub!(/^function:/, '') + "functions" + elsif service == "sns" + "notifiers" + elsif service == "sqs" + "msg_queues" + else + service + end + ref_params = { + id: resource, + region: region, + type: target_type, + cloud: "AWS", + credentials: @credentials, + habitat: MU::Config::Ref.get( + id: account, + cloud: "AWS", + credentials: @credentials + ) + } + [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters].each { |attr| + if t.respond_to?(attr) and !t.send(attr).nil? + ref_params[attr] = MU.structToHash(t.send(attr), stringify_keys: true) + end + } + + bok['targets'] << MU::Config::Ref.get(ref_params) + } + + MU.log @cloud_id, MU::NOTICE, details: bok + # XXX cloud_desc.event_pattern - what do we want to do with this? bok @@ -173,6 +251,120 @@ def toKitten(**_args) # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource def self.schema(_config) toplevel_required = [] + + target_schema = MU::Config::Ref.schema(any_type: true, desc: "A resource which will be invoked by this event. Can be a reference to a sibling Mu resource, typically a +Function+ or +MsgQueue+, or to an unadorned external cloud resource.") + target_params = { + "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to use when interacting with this target.", omit_fields: ["region", "tag"]), + "input" => { + "type" => "string" + }, + "input_path" => { + "type" => "string" + }, + "run_command_parameters" => { + "type" => "object", + "required" => ["run_command_targets"], + "properties" => { + "run_command_targets" => { + "type" => "array", + "items" => { + "type" => "object", + "required" => ["key", "values"], + "properties" => { + "key" => { + "type" => "string" + }, + "values" => { + "type" => "array", + "items" => { + "type" => "string" + } + } + } + } + } + } + }, + "input_transformer" => { + "type" => "object", + "required" => ["input_template"], + "properties" => { + "input_template" => { + "type" => "string" + }, + "input_paths_map" => { + "type" => "object", + } + } + }, + "batch_parameters" => { + "type" => "object", + "required" => ["job_definition", "job_name"], + "properties" => { + "job_definition" => { + "type" => "string" + }, + "job_name" => { + "type" => "string" + }, + "array_properties" => { + "type" => "object", + "properties" => { + "size" => { + "type" => "integer" + } + } + }, + "retry_strategy" => { + "type" => "object", + "properties" => { + "attempts" => { + "type" => "integer" + } + } + } + } + }, + "sqs_parameters" => { + "type" => "object", + "required" => ["message_group_id"], + "properties" => { + "message_group_id" => { + "type" => "string" + } + } + }, + "kinesis_parameters" => { + "type" => "object", + "required" => ["partition_key_path"], + "properties" => { + "partition_key_path" => { + "type" => "string" + } + } + }, + "http_parameters" => { + "type" => "object", + "properties" => { + "path_parameter_values" => { + "type" => "array", + "items" => { + "type" => "string" + } + }, + "header_parameters" => { + "description" => "Key => value pairs to pass as headers", + "type" => "object" + }, + "query_string_parameters" => { + "description" => "Key => value pairs to pass as query strings", + "type" => "object" + } + } + } + } + target_schema["properties"].merge!(target_params) + schema = { "disabled" => { "type" => "boolean", @@ -180,6 +372,10 @@ def self.schema(_config) "default" => false }, "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to this CloudWatch Event.", omit_fields: ["region", "tag"]), + "targets" => { + "type" => "array", + "items" => target_schema + } } [toplevel_required, schema] end @@ -223,6 +419,7 @@ def get_properties params[:schedule_expression] = "cron(" + ["minute", "hour", "day_of_month", "month", "day_of_week", "year"].map { |i| @config['schedule'][i] }.join(" ") +")" end + params end From 24fe0df340ab7c6367e0fc2466d54538dea92a73 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 16 Jul 2020 17:31:18 -0400 Subject: [PATCH 040/107] MommaCat: ugh remove the stupid daemon pid file before starting if it already exists --- modules/mu/mommacat/daemon.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/mu/mommacat/daemon.rb b/modules/mu/mommacat/daemon.rb index 4887f1593..71222c761 100644 --- a/modules/mu/mommacat/daemon.rb +++ b/modules/mu/mommacat/daemon.rb @@ -308,6 +308,7 @@ def self.start } return 0 if status + File.unlink(daemonPidFile) if File.exists?(daemonPidFile) MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}" origdir = Dir.getwd Dir.chdir(MU.myRoot+"/modules") From ef8f51a728c7b66c0938220b1b56ee2cf1b3a59d Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Sun, 19 Jul 2020 23:55:55 -0400 Subject: [PATCH 041/107] moar guards --- modules/mu/providers/aws/function.rb | 2 +- modules/mu/providers/aws/job.rb | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 3cb66a66e..1b5619f87 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -250,7 +250,7 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, regi # Canonical Amazon Resource Number for this resource # @return [String] def arn - cloud_desc.function_arn + cloud_desc ? cloud_desc.function_arn : nil end # Locate an existing function. diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index 0649f6a76..e8cdf172b 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -56,6 +56,7 @@ def groom if @config['targets'] target_params = [] @config['targets'].each { |t| + MU.retrier([MuNonFatal], max:5, wait: 9) { target_ref = MU::Config::Ref.get(t) target_obj = target_ref.kitten(cloud: "AWS") this_target = if target_ref.is_mu_type? and target_obj and @@ -64,11 +65,13 @@ def groom id: target_obj.cloud_id, arn: target_obj.arn } - else + elsif target_ref.id and target_ref.id.match(/^arn:/) { id: target_ref.id || target_ref.name, arn: target_ref.id } + else + raise MuNonFatal.new "Failed to retrieve ARN from CLoudWatch Event target descriptor", details: target_ref.to_h end if t['role'] role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS") @@ -81,6 +84,7 @@ def groom end } target_params << this_target + } } MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_targets( rule: @cloud_id, From 6a3d48fe410b7a13660060068f5fc5c06a9859f4 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 20 Jul 2020 14:36:20 -0400 Subject: [PATCH 042/107] MommaCat: guard daemon startup better for non-root users, maybe; AWS::Bucket: try to set mime type sensibly for SVG files --- bin/mu-load-config.rb | 3 ++- modules/mu/mommacat/daemon.rb | 16 +++++++++------- modules/mu/providers/aws/bucket.rb | 15 +++++++++++++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/bin/mu-load-config.rb b/bin/mu-load-config.rb index f7db85974..065a30f11 100755 --- a/bin/mu-load-config.rb +++ b/bin/mu-load-config.rb @@ -134,7 +134,7 @@ def loadMuConfig(default_cfg_overrides = nil) } end - global_cfg = { "config_files" => [] } + global_cfg = { "config_files" => [], "overridden_keys" => [] } if File.exist?(cfgPath) global_cfg = YAML.load(File.read(cfgPath)) global_cfg["config_files"] = [cfgPath] @@ -147,6 +147,7 @@ def loadMuConfig(default_cfg_overrides = nil) if localfile global_cfg.merge!(localfile) global_cfg["config_files"] << "#{home}/.mu.yaml" + global_cfg["overridden_keys"] = localfile.keys end end if !global_cfg.has_key?("installdir") diff --git a/modules/mu/mommacat/daemon.rb b/modules/mu/mommacat/daemon.rb index 71222c761..cd5598eeb 100644 --- a/modules/mu/mommacat/daemon.rb +++ b/modules/mu/mommacat/daemon.rb @@ -288,8 +288,8 @@ def self.daemonLogFile # Path to the PID file used by the Momma Cat daemon # @return [String] - def self.daemonPidFile - base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir + def self.daemonPidFile(root = false) + base = ((Process.uid == 0 or root) and !MU.localOnly) ? "/var" : MU.dataDir "#{base}/run/mommacat.pid" end @@ -306,7 +306,9 @@ def self.start Dir.mkdir(dir) end } - return 0 if status + if status or (Process.uid != 0 and $MU_CFG['overridden_keys'].include?("mommacat_port") and status(true)) + return 0 + end File.unlink(daemonPidFile) if File.exists?(daemonPidFile) MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}" @@ -347,12 +349,12 @@ def self.start # Return true if the Momma Cat daemon appears to be running # @return [Boolean] - def self.status + def self.status(root = false) if MU.inGem? and MU.muCfg['disable_mommacat'] return true end - if File.exist?(daemonPidFile) - pid = File.read(daemonPidFile).chomp.to_i + if File.exist?(daemonPidFile(root)) + pid = File.read(daemonPidFile(root)).chomp.to_i begin Process.getpgid(pid) MU.log "Momma Cat running with pid #{pid.to_s}", (@@notified_on_pid[pid] ? MU::DEBUG : MU::INFO) # shush @@ -361,7 +363,7 @@ def self.status rescue Errno::ESRCH end end - MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile + MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile(root) false end diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index fd2246ce2..58153be0a 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -21,6 +21,10 @@ class Bucket < MU::Cloud::Bucket @@region_cache = {} @@region_cache_semaphore = Mutex.new + MIME_MAP = { + ".svg" => "image/svg+xml" + } + # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us. # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat def initialize(**args) @@ -203,12 +207,19 @@ def self.upload(url, acl: "private", file: nil, data: nil, credentials: nil, reg begin MU.log "Writing #{path} to S3 bucket #{bucket}" - MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object( + params = { acl: acl, bucket: bucket, key: path, body: data - ) + } + + MIME_MAP.each_pair { |extension, content_type| + if path =~ /#{Regexp.quote(extension)}$/i + params[:content_type] = content_type + end + } + MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object(params) rescue Aws::S3::Errors => e raise MuError, "Got #{e.inspect} trying to write #{path} to #{bucket} (region: #{region}, credentials: #{credentials})" end From f2fdc57ddc5c70d49f59e7f2d7153616226aeb16 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 20 Jul 2020 15:56:03 -0400 Subject: [PATCH 043/107] AWS::NoSQLDB: crude pre-population for DynamoDB --- modules/mu/providers/aws.rb | 9 ++++++++ modules/mu/providers/aws/nosqldb.rb | 33 ++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index b45bd348d..98dec58a7 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -1125,6 +1125,14 @@ def self.dynamo(region: MU.curRegion, credentials: nil) @@dynamo_api[credentials][region] end + # Amazon's DynamoStream API + def self.dynamostream(region: MU.curRegion, credentials: nil) + region ||= myRegion + @@dynamostream_api[credentials] ||= {} + @@dynamostream_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDBStreams", region: region, credentials: credentials) + @@dynamostream_api[credentials][region] + end + # Amazon's Pricing API def self.pricing(region: MU.curRegion, credentials: nil) region ||= myRegion @@ -1609,6 +1617,7 @@ def method_missing(method_sym, *arguments) @@kms_api ={} @@organization_api ={} @@dynamo_api ={} + @@dynamostream_api ={} end end end diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 5448af509..52cc0f5fb 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -118,8 +118,24 @@ def create sleep 5 if resp.table.table_status == "CREATING" end while resp.table.table_status == "CREATING" - tagTable if !@config['scrub_mu_isms'] + + if @config['populate'] and !@config['populate'].empty? + MU.log "Preloading #{@mu_name} with #{@config['populate'].size.to_s} items" + @config['populate'].each { |item| + item_param = {} + item.each_pair { |k, v| + if v.is_a?(Integer) + item_param[k] = { n: v } + elsif v.is_a?(Boolean) + item_param[k] = { b: v } + else + item_param[k] = { s: v.to_s } + end + } + MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).put_item(table_name: @cloud_id, item: item_param) + } + end end # Apply tags to this DynamoDB table @@ -272,6 +288,7 @@ def toKitten(**_args) bok['read_capacity'] = cloud_desc.provisioned_throughput.read_capacity_units bok['write_capacity'] = cloud_desc.provisioned_throughput.write_capacity_units + cloud_desc.attribute_definitions.each { |attr| bok['attributes'] ||= [] newattr = { @@ -292,9 +309,16 @@ def toKitten(**_args) } if cloud_desc.stream_specification and cloud_desc.stream_specification.stream_enabled +MU.log @cloud_id, MU::NOTICE, details: cloud_desc bok['stream'] = cloud_desc.stream_specification.stream_view_type + cloud_desc.latest_stream_arn + pp MU::Cloud::AWS.dynamostream(credentials: @credentials, region: @config['region']).list_streams end + bok["populate"] = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @config['region']).scan( + table_name: @cloud_id + ).items + bok end @@ -306,6 +330,13 @@ def self.schema(_config) schema = { + "populate" => { + "type" => "array", + "items" => { + "type" => "object", + "description" => "Key-value pairs, compatible with the +attributes+ schema, with which to populate this +table+ during its initial creation." + } + }, "attributes" => { "type" => "array", "minItems" => 1, From 759f26b531a01151424a2a27dc89c720821a598c Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 20 Jul 2020 19:59:30 -0400 Subject: [PATCH 044/107] AWS::Job and AWS::NoSQLDB interplay --- modules/mu/providers/aws/job.rb | 7 +++++++ modules/mu/providers/aws/nosqldb.rb | 20 +++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index e8cdf172b..bddd2c227 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -391,6 +391,13 @@ def self.schema(_config) def self.validateConfig(job, _configurator) ok = true + job['targets'].each { |t| + target_ref = MU::Config::Ref.get(t) + if target_ref.is_mu_type? and target_ref.name + MU::Config.addDependency(job, target_ref.name, target_ref.type) + end + } + ok end diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 52cc0f5fb..4d07edcb4 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -47,7 +47,11 @@ def create } end + type_map = {} + @config['attributes'].each { |attr| + type_map[attr['name']] = attr['type'] + params[:attribute_definitions] << { :attribute_name => attr['name'], :attribute_type => attr['type'] @@ -123,17 +127,11 @@ def create if @config['populate'] and !@config['populate'].empty? MU.log "Preloading #{@mu_name} with #{@config['populate'].size.to_s} items" @config['populate'].each { |item| - item_param = {} - item.each_pair { |k, v| - if v.is_a?(Integer) - item_param[k] = { n: v } - elsif v.is_a?(Boolean) - item_param[k] = { b: v } - else - item_param[k] = { s: v.to_s } - end - } - MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).put_item(table_name: @cloud_id, item: item_param) + begin + MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).put_item(table_name: @cloud_id, item: item) + rescue ::Aws::DynamoDB::Errors::ValidationException => e + MU.log e.message, MU::ERR, details: item + end } end end From a7cc1627cef8e805e3fc0cb1486d756cbea7924f Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 20 Jul 2020 21:39:18 -0400 Subject: [PATCH 045/107] AWS::NoSQLDB: uuuuuuh ok maybe init dynamo with batch writes and also don't try to do 10,000 of them --- modules/mu/providers/aws/nosqldb.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 4d07edcb4..071668cd8 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -126,13 +126,19 @@ def create if @config['populate'] and !@config['populate'].empty? MU.log "Preloading #{@mu_name} with #{@config['populate'].size.to_s} items" - @config['populate'].each { |item| + items_to_write = @config['populate'].dup + begin + batch = items_to_write.slice!(0, (items_to_write.length >= 25 ? 25 : items_to_write.length)) begin - MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).put_item(table_name: @cloud_id, item: item) + MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).batch_write_item( + request_items: { + @cloud_id => batch.map { |i| { put_request: { item: i } } } + } + ) rescue ::Aws::DynamoDB::Errors::ValidationException => e MU.log e.message, MU::ERR, details: item end - } + end while !items_to_write.empty? end end From d807a515c16f4ea7bb81b219e5c9322f4cc9002f Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 22 Jul 2020 20:54:42 -0400 Subject: [PATCH 046/107] a whole bunch of dependency resolution stuff, and apparently some leftover bugs from Amrit's original Lambda implementation --- modules/mu/cloud.rb | 4 +- modules/mu/config.rb | 2 +- modules/mu/providers/aws.rb | 2 +- modules/mu/providers/aws/endpoint.rb | 17 ++++--- modules/mu/providers/aws/function.rb | 73 ++++++++++++++++++++++++---- modules/mu/providers/aws/job.rb | 14 +++++- modules/mu/providers/aws/notifier.rb | 16 ++++-- 7 files changed, 100 insertions(+), 28 deletions(-) diff --git a/modules/mu/cloud.rb b/modules/mu/cloud.rb index bea6769ec..515848c32 100644 --- a/modules/mu/cloud.rb +++ b/modules/mu/cloud.rb @@ -439,7 +439,7 @@ class Job; :cfg_plural => "nosqldbs", :interface => self.const_get("NoSQLDB"), :deps_wait_on_my_creation => true, - :waits_on_parent_completion => true, + :waits_on_parent_completion => false, :class => @@generic_class_methods, :instance => @@generic_instance_methods + [:groom] }, @@ -450,7 +450,7 @@ class Job; :cfg_plural => "jobs", :interface => self.const_get("Job"), :deps_wait_on_my_creation => true, - :waits_on_parent_completion => true, + :waits_on_parent_completion => false, :class => @@generic_class_methods, :instance => @@generic_instance_methods + [:groom] } diff --git a/modules/mu/config.rb b/modules/mu/config.rb index d8b99743b..326d791bf 100644 --- a/modules/mu/config.rb +++ b/modules/mu/config.rb @@ -437,7 +437,7 @@ def resolveTails(tree, indent= "") # @param type [String] # @param phase [String] # @param no_create_wait [Boolean] - def self.addDependency(resource, name, type, phase: nil, no_create_wait: false) + def self.addDependency(resource, name, type, phase: "create", no_create_wait: false) if ![nil, "create", "groom"].include?(phase) raise MuError, "Invalid phase '#{phase}' while adding dependency #{type} #{name} to #{resource['name']}" end diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index 98dec58a7..be25661a1 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -1556,7 +1556,7 @@ def method_missing(method_sym, *arguments) end return retval - rescue Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e + rescue Aws::Lambda::Errors::TooManyRequestsException, Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/) MU.log e.inspect, MU::ERR raise e diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index a91f86cf3..1fb0f4e3d 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -21,13 +21,11 @@ def create } ) @cloud_id = resp.id - generate_methods - - + generate_methods(false) end # Create/update all of the methods declared for this endpoint - def generate_methods + def generate_methods(integrations = true) resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( rest_api_id: @cloud_id, ) @@ -117,7 +115,7 @@ def generate_methods # fine to ignore end - if m['integrate_with'] + if integrations and m['integrate_with'] # role_arn = if m['iam_role'] # if m['iam_role'].match(/^arn:/) # m['iam_role'] @@ -134,8 +132,11 @@ def generate_methods svc, action = m['integrate_with']['aws_generic_action'].split(/:/) ["arn:aws:apigateway:"+@config['region']+":#{svc}:action/#{action}", "AWS"] elsif m['integrate_with']['type'] == "functions" - function_obj = @deploy.findLitterMate(name: m['integrate_with']['name'], type: "functions").cloudobj - ["arn:aws:apigateway:"+@config['region']+":lambda:path/2015-03-31/functions/"+function_obj.arn+"/invocations", "AWS"] + function_obj = nil + MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { function_obj.nil? }) { + function_obj = @deploy.findLitterMate(name: m['integrate_with']['name'], type: "functions") + } + ["arn:aws:apigateway:"+@config['region']+":lambda:path/2015-03-31/functions/"+function_obj.cloudobj.arn+"/invocations", "AWS"] elsif m['integrate_with']['type'] == "mock" [nil, "MOCK"] end @@ -306,12 +307,12 @@ def toKitten(**_args) bok['name'] = cloud_desc.name - MU.log "REST API", MU::NOTICE, details: cloud_desc resources = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( rest_api_id: @cloud_id, ).items resources.each { |r| + bok['deploy_to'] ||= r.path if r.path != "/" # XXX this is wrong/inadequate next if !r.respond_to?(:resource_methods) or r.resource_methods.nil? r.resource_methods.each_pair { |http_type, m| bok['methods'] ||= [] diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 1b5619f87..f672d7262 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -110,7 +110,7 @@ def groom statement_id: "#{@mu_name}-ID-1", } - MU.log trigger_properties, MU::DEBUG +# MU.log "trigger properties", MU::NOTICE, details: trigger_properties begin MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties) rescue Aws::Lambda::Errors::ResourceConflictException @@ -157,11 +157,13 @@ def assume_trigger_arns(svc, name) arn = nil case svc.downcase when 'sns' - arn = "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}" + sib_sns = @deploy.findLitterMate(name: name, type: "notifiers") + arn = sib_sns ? sib_sns.arn : "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}" when 'alarm','events', 'event', 'cloudwatch_event' arn = "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}" when 'apigateway' - arn = "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}" + sib_apig = @deploy.findLitterMate(name: name, type: "endpoints") + arn = sib_apig ? sib_apig.arn : "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}" when 's3' arn = '' end @@ -180,11 +182,12 @@ def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda' when 'sns' # XXX don't do this, use MU::Cloud::AWS::Notification sns_client = MU::Cloud::AWS.sns(region: region, credentials: @config['credentials']) - sns_client.subscribe({ - topic_arn: trig_arn, - protocol: protocol, - endpoint: func_arn - }) + MU.log "Would try to subscribe #{@mu_name} to #{trig_arn} (#{protocol}) here but I shouldn't have to", MU::WARN +# sns_client.subscribe({ +# topic_arn: trig_arn, +# protocol: protocol, +# endpoint: func_arn +# }) when 'event','cloudwatch_event', 'events' # XXX don't do this, use MU::Cloud::AWS::Log MU::Cloud::AWS.cloudwatch_events(region: region, credentials: @config['credentials']).put_targets({ @@ -292,6 +295,20 @@ def toKitten(**_args) bok['timeout'] = cloud_desc.timeout function = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function(function_name: bok['name']) +# event_srcs = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).list_event_source_mappings(function_name: @cloud_id) +# if event_srcs and !event_srcs.event_source_mappings.empty? +# MU.log "dem mappings tho #{@cloud_id}", MU::WARN, details: event_srcs +# end + +# begin +# invoke_cfg = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function_event_invoke_config(function_name: @cloud_id) +# MU.log "invoke config #{@cloud_id}", MU::WARN, details: invoke_cfg +# rescue ::Aws::Lambda::Errors::ResourceNotFoundException +# end + +# MU.log @cloud_id, MU::WARN, details: cloud_desc if @cloud_id == "Espier-Scheduled-Scanner" +# MU.log "configuration #{@cloud_id}", MU::WARN, details: MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_function_configuration(function_name: @cloud_id) if @cloud_id == "Espier-Scheduled-Scanner" + if function.code.repository_type == "S3" bok['code'] = {} @@ -357,8 +374,22 @@ def toKitten(**_args) type: "roles" ) end + + begin + pol = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_policy(function_name: @cloud_id).policy + if pol + bok['triggers'] ||= [] + JSON.parse(pol)["Statement"].each { |s| + bok['triggers'] << { + "service" => s["Principal"]["Service"].sub(/\..*/, ''), + "name" => s["Resource"].sub(/.*?[:\/]([^:\/]+)$/, '\1') + } + } + end + rescue ::Aws::Lambda::Errors::ResourceNotFoundException + end #MU.log @cloud_id, MU::NOTICE, details: function -# XXX triggers, permissions +# XXX permissions bok end @@ -379,7 +410,7 @@ def self.schema(_config) "properties" => { "service" => { "type" => "string", - "enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot}, + "enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot lex}, "description" => "The name of the AWS service that will trigger this function" }, "name" => { @@ -438,6 +469,28 @@ def self.schema(_config) def self.validateConfig(function, configurator) ok = true + if function['triggers'] + function['triggers'].each { |t| + mu_type = if t["service"] == "sns" + "notifiers" + elsif t["service"] == "apigateway" + "endpoints" + elsif t["service"] == "s3" + "buckets" + elsif t["service"] == "dynamodb" + "nosqldbs" + elsif t["service"] == "events" + "jobs" + elsif t["service"] == "sqs" + "msg_queues" + end + + if mu_type + MU::Config.addDependency(function, t['name'], mu_type, no_create_wait: true) + end + } + end + if function['vpc'] fwname = "lambda-#{function['name']}" # default to allowing pings, if no ingress_rules were specified diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index bddd2c227..58b2ec74c 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -50,6 +50,7 @@ def groom if params.size > 0 MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params + params[:name] = @cloud_id MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params) end @@ -133,7 +134,18 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, regi (flags and flags['known'] and flags['known'].include?(id)) MU.log "Deleting CloudWatch Event #{id}" if !noop - # XXX purge all targets first + resp = MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).list_targets_by_rule( + rule: id, + event_bus_name: desc.event_bus_name, + ) + if resp and resp.targets and !resp.targets.empty? + MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).remove_targets( + rule: id, + event_bus_name: desc.event_bus_name, + ids: resp.targets.map { |t| t.id } + ) + end + MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).delete_rule( name: id, event_bus_name: desc.event_bus_name diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index 94a998b2c..b5b9e0450 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -27,8 +27,8 @@ def initialize(**args) # Called automatically by {MU::Deploy#createResources} def create - MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @mu_name) @cloud_id = @mu_name + MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @cloud_id) MU.log "Created SNS topic #{@mu_name}" end @@ -37,8 +37,14 @@ def groom if @config['subscriptions'] @config['subscriptions'].each { |sub| if sub['resource'] and !sub['endpoint'] - sub['endpoint'] = MU::Config::Ref.get(sub['resource']).kitten(@deploy).arn + endpoint_obj = nil + MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { endpoint_obj.nil? }) { + endpoint_obj = MU::Config::Ref.get(sub['resource']).kitten(@deploy) + } + sub['endpoint'] = endpoint_obj.arn end +# XXX guard this + MU.log "Subscribing #{sub['endpoint']} to SNS topic #{arn}", MU::NOTICE MU::Cloud::AWS::Notifier.subscribe( arn: arn, endpoint: sub['endpoint'], @@ -139,7 +145,7 @@ def toKitten(**_args) MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config return nil end - +pp cloud_desc if @cloud_id == "Espier-Publish-Domains" bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"] svcmap = { "lambda" => "functions", @@ -171,6 +177,7 @@ def toKitten(**_args) } end } +pp bok['subscriptions'] if @cloud_id == "Espier-Publish-Domains" bok end @@ -220,7 +227,6 @@ def self.validateConfig(notifier, configurator) elsif sub['resource']['type'] == "msg_queues" "sqs" end - exit elsif sub['endpoint'] if sub["endpoint"].match(/^http:/i) "http" @@ -278,8 +284,8 @@ def self.subscribe(arn: nil, protocol: nil, endpoint: nil, region: MU.curRegion, end unless already_subscribed + MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}" MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint) - MU.log "Subscribed #{endpoint} to SNS topic #{arn}" end end From a894ae68ebbd652d9bea9cd68138fb22928ff6e5 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 23 Jul 2020 02:23:24 -0400 Subject: [PATCH 047/107] AWS::Function: add dynamodb (Dynamo Streams) as a trigger type --- modules/mu/providers/aws/function.rb | 32 ++++++----- modules/mu/providers/aws/job.rb | 3 +- modules/mu/providers/aws/notifier.rb | 80 +++++++++++++--------------- modules/mu/providers/aws/server.rb | 4 +- 4 files changed, 60 insertions(+), 59 deletions(-) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index f672d7262..172a1c357 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -100,7 +100,7 @@ def groom ### triggers must exist prior if @config['triggers'] @config['triggers'].each { |tr| - trigger_arn = assume_trigger_arns(tr['service'], tr['name']) + trigger_arn = resolveARN(tr['service'], tr['name']) trigger_properties = { action: "lambda:InvokeFunction", @@ -151,8 +151,8 @@ def addTrigger(calling_arn, calling_service, calling_name) end # Look up an ARN for a given trigger type and resource name - def assume_trigger_arns(svc, name) - supported_triggers = %w(apigateway sns events event cloudwatch_event) + def resolveARN(svc, name) + supported_triggers = %w(apigateway sns events event cloudwatch_event dynamodb) if supported_triggers.include?(svc.downcase) arn = nil case svc.downcase @@ -160,7 +160,11 @@ def assume_trigger_arns(svc, name) sib_sns = @deploy.findLitterMate(name: name, type: "notifiers") arn = sib_sns ? sib_sns.arn : "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}" when 'alarm','events', 'event', 'cloudwatch_event' - arn = "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}" + sib_event = @deploy.findLitterMate(name: name, type: "job") + arn = sib_event ? sib_event.arn : "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}" + when 'dynamodb' + sib_dynamo = @deploy.findLitterMate(name: name, type: "nosqldb") + arn = sib_dynamo ? sib_dynamo.arn : "arn:aws:dynamodb:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:table/#{name}" when 'apigateway' sib_apig = @deploy.findLitterMate(name: name, type: "endpoints") arn = sib_apig ? sib_apig.arn : "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}" @@ -180,14 +184,18 @@ def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda' case trig_type when 'sns' - # XXX don't do this, use MU::Cloud::AWS::Notification - sns_client = MU::Cloud::AWS.sns(region: region, credentials: @config['credentials']) - MU.log "Would try to subscribe #{@mu_name} to #{trig_arn} (#{protocol}) here but I shouldn't have to", MU::WARN -# sns_client.subscribe({ -# topic_arn: trig_arn, -# protocol: protocol, -# endpoint: func_arn -# }) + MU::Cloud.resourceClass("AWS", "Notifier").subscribe(trig_arn, arn, "lambda", region: @config['region'], credentials: @credentials) + when 'dynamodb' + stream = MU::Cloud::AWS.dynamostream(region: @config['region'], credentials: @config['credentials']).list_streams(table_name: trig_arn.sub(/.*?:table\//, '')).streams.first +# XXX guard this + MU.log "Adding DynamoDB Stream from #{stream.stream_arn} as trigger for #{@cloud_id}" + MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_event_source_mapping( + event_source_arn: stream.stream_arn, + function_name: @cloud_id, + starting_position: "TRIM_HORIZON" # ...whatever that is + ) + +# MU::Cloud.resourceClass("AWS", "NoSQLDB").subscribe(trig_arn, arn, "lambda", region: @config['region'], credentials: @credentials) when 'event','cloudwatch_event', 'events' # XXX don't do this, use MU::Cloud::AWS::Log MU::Cloud::AWS.cloudwatch_events(region: region, credentials: @config['credentials']).put_targets({ diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index 58b2ec74c..83009dca7 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -50,8 +50,7 @@ def groom if params.size > 0 MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params - params[:name] = @cloud_id - MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params) + MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(new_props) end if @config['targets'] diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index b5b9e0450..28fba2078 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -43,19 +43,41 @@ def groom } sub['endpoint'] = endpoint_obj.arn end -# XXX guard this - MU.log "Subscribing #{sub['endpoint']} to SNS topic #{arn}", MU::NOTICE - MU::Cloud::AWS::Notifier.subscribe( - arn: arn, - endpoint: sub['endpoint'], - region: @config['region'], - credentials: @config['credentials'], - protocol: sub['type'] - ) + subscribe(sub['endpoint'], sub['type']) } end end + # Subscribe something to this SNS topic + # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed + # @param protocol [String]: The protocol being subscribed + def subscribe(endpoint, protocol) + self.class.subscribe(arn, endpoint, protocol, region: @config['region'], credentials: @credentials) + end + + # Subscribe something to an SNS topic + # @param cloud_id [String]: The short name or ARN of an existing SNS topic + # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed + # @param protocol [String]: The protocol being subscribed + # @param region [String]: The region of the target SNS topic + # @param credentials [String]: + def self.subscribe(cloud_id, endpoint, protocol, region: nil, credentials: nil) + topic = find(cloud_id: cloud_id, region: region, credentials: credentials).values.first + if !topic + raise MuError, "Failed to find SNS Topic #{cloud_id} in #{region}" + end + arn = topic["TopicArn"] + + resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions + + resp.each { |subscription| + return subscription if subscription.protocol == protocol and subscription.endpoint == endpoint + } + + MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}", MU::NOTICE + MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint) + end + # Does this resource type exist as a global (cloud-wide) artifact, or # is it localized to a region/zone? # @return [Boolean] @@ -111,7 +133,11 @@ def self.find(**args) found = {} if args[:cloud_id] - arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id] + arn = if args[:cloud_id].match(/^arn:/) + args[:cloud_id] + else + "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id] + end desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes found[args[:cloud_id]] = desc if desc else @@ -255,40 +281,6 @@ def self.validateConfig(notifier, configurator) end - # Subscribe to a notifier group. This can either be an email address, SQS queue, application endpoint, etc... - # Will create the subscription only if it doesn't already exist. - # @param arn [String]: The cloud provider's identifier of the notifier group. - # @param protocol [String]: The type of the subscription (eg. email,https, etc..). - # @param endpoint [String]: The endpoint of the subscription. This will depend on the 'protocol' (as an example if protocol is email, endpoint will be the email address) .. - # @param region [String]: The cloud provider region. - def self.subscribe(arn: nil, protocol: nil, endpoint: nil, region: MU.curRegion, credentials: nil) - retries = 0 - begin - resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions - rescue Aws::SNS::Errors::NotFound - if retries < 5 - MU.log "Couldn't find topic #{arn}, retrying several times in case of a lagging resource" - retries += 1 - sleep 30 - retry - else - raise MuError, "Couldn't find topic #{arn}, giving up" - end - end - - already_subscribed = false - if resp && !resp.empty? - resp.each { |subscription| - already_subscribed = true if subscription.protocol == protocol && subscription.endpoint == endpoint - } - end - - unless already_subscribed - MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}" - MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint) - end - end - # Test if a notifier group exists # Create a new notifier group. Will check if the group exists before creating it. # @param topic_name [String]: The cloud provider's name for the notifier group. diff --git a/modules/mu/providers/aws/server.rb b/modules/mu/providers/aws/server.rb index 88beaf8f3..14b1ad35d 100644 --- a/modules/mu/providers/aws/server.rb +++ b/modules/mu/providers/aws/server.rb @@ -2202,8 +2202,10 @@ def setAlarms alarm["dimensions"] = [{:name => "InstanceId", :value => @cloud_id}] if alarm["enable_notifications"] + # XXX vile, this should be a sibling resource generated by the + # parser topic_arn = MU::Cloud.resourceClass("AWS", "Notification").createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials']) - MU::Cloud.resourceClass("AWS", "Notification").subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"], credentials: @config["credentials"]) + MU::Cloud.resourceClass("AWS", "Notification").subscribe(topic_arn, alarm["notification_endpoint"], alarm["notification_type"], region: @config["region"], credentials: @config["credentials"]) alarm["alarm_actions"] = [topic_arn] alarm["ok_actions"] = [topic_arn] end From f1638f639e901c612a00ae3d6114bfee44a5e281 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 23 Jul 2020 14:14:51 -0400 Subject: [PATCH 048/107] AWS::Function: create our log group in case AWS doesn't (may delete later); handle regrooms of dynamo log stream triggers with grace --- modules/mu/providers/aws/function.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 172a1c357..8ce3bb732 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -62,6 +62,11 @@ def create @cloud_id = resp.function_name } + # why do we have to do this? + MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @credentials).create_log_group( + log_group_name: "/aws/lambda/#{@cloud_id}", + tags: @tags + ) end # Called automatically by {MU::Deploy#createResources} @@ -189,11 +194,14 @@ def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda' stream = MU::Cloud::AWS.dynamostream(region: @config['region'], credentials: @config['credentials']).list_streams(table_name: trig_arn.sub(/.*?:table\//, '')).streams.first # XXX guard this MU.log "Adding DynamoDB Stream from #{stream.stream_arn} as trigger for #{@cloud_id}" + begin MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_event_source_mapping( event_source_arn: stream.stream_arn, function_name: @cloud_id, starting_position: "TRIM_HORIZON" # ...whatever that is ) + rescue ::Aws::Lambda::Errors::ResourceConflictException + end # MU::Cloud.resourceClass("AWS", "NoSQLDB").subscribe(trig_arn, arn, "lambda", region: @config['region'], credentials: @credentials) when 'event','cloudwatch_event', 'events' From b36cef8f413828dbc1fe2ec5676389fd61acce92 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Thu, 23 Jul 2020 16:17:35 -0400 Subject: [PATCH 049/107] MommaCat: quit yapping about the daemon not running for non-root users when the root daemon is there --- modules/mu/mommacat/daemon.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/mu/mommacat/daemon.rb b/modules/mu/mommacat/daemon.rb index cd5598eeb..ce8a0f98b 100644 --- a/modules/mu/mommacat/daemon.rb +++ b/modules/mu/mommacat/daemon.rb @@ -306,7 +306,10 @@ def self.start Dir.mkdir(dir) end } - if status or (Process.uid != 0 and $MU_CFG['overridden_keys'].include?("mommacat_port") and status(true)) + if (Process.uid != 0 and + (!$MU_CFG['overridden_keys'] or !$MU_CFG['overridden_keys'].include?("mommacat_port")) and + status(true) + ) or status return 0 end From ca1bf97d3801e1e68110bb6d35e92c04fd6adf9d Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 23 Jul 2020 17:05:52 -0400 Subject: [PATCH 050/107] AWS: make sure notify will return things if there are things to return --- modules/mu/providers/aws/endpoint.rb | 2 +- modules/mu/providers/aws/function.rb | 6 +++++- modules/mu/providers/aws/nosqldb.rb | 1 + modules/mu/providers/aws/notifier.rb | 2 +- modules/mu/providers/aws/search_domain.rb | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 1fb0f4e3d..09a12f379 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -236,7 +236,7 @@ def cloud_desc(use_cache: true) # Return the metadata for this API # @return [Hash] def notify - return nil if !@cloud_id or !cloud_desc + return nil if !@cloud_id or !cloud_desc(use_cache: false) deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) deploy_struct['url'] = "https://"+@cloud_id+".execute-api."+@config['region']+".amazonaws.com/"+@config['deploy_to'] # XXX stages and whatnot diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 8ce3bb732..6cfa6574e 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -638,8 +638,12 @@ def get_properties siblings = @deploy.findLitterMate(return_all: true, type: sib_type, cloud: "AWS") if siblings siblings.each_value { |sibling| + pp sibling.cloud_desc(use_cache: false) metadata = sibling.notify - next if !metadata + if !metadata + MU.log "Failed to extract metadata from sibling #{sibling}", MU::WARN + next + end SIBLING_VARS[sib_type].each { |var| if metadata[var] @config['environment_variables'] ||= [] diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 071668cd8..9c8b44a5a 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -243,6 +243,7 @@ def arn # Return the metadata for this user cofiguration # @return [Hash] def notify + return nil if !@cloud_id or !cloud_desc(use_cache: false) MU.structToHash(cloud_desc, stringify_keys: true) end diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index 28fba2078..ee272280d 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -122,7 +122,7 @@ def arn # Return the metadata for this user cofiguration # @return [Hash] def notify - return nil if !@cloud_id + return nil if !@cloud_id or !cloud_desc(use_cache: false) desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes MU.structToHash(desc) end diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index fcce95c1a..76254d016 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -85,7 +85,7 @@ def arn # Return the metadata for this SearchDomain rule # @return [Hash] def notify - return nil if !cloud_desc + return nil if !cloud_desc(use_cache: false) deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: arn).tag_list deploy_struct['tags'] = tags.map { |t| { t.key => t.value } } From 3802f45851c5d83265394f832a821cd4814281f5 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 23 Jul 2020 21:26:58 -0400 Subject: [PATCH 051/107] AWS::Notifier: don't explode in .find if a topic doesn't exist --- modules/mu/providers/aws/function.rb | 4 ++-- modules/mu/providers/aws/notifier.rb | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 6cfa6574e..82dc4b8a5 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -62,7 +62,8 @@ def create @cloud_id = resp.function_name } - # why do we have to do this? + # the console does this and docs expect it to be there, so mimic the + # behavior MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @credentials).create_log_group( log_group_name: "/aws/lambda/#{@cloud_id}", tags: @tags @@ -638,7 +639,6 @@ def get_properties siblings = @deploy.findLitterMate(return_all: true, type: sib_type, cloud: "AWS") if siblings siblings.each_value { |sibling| - pp sibling.cloud_desc(use_cache: false) metadata = sibling.notify if !metadata MU.log "Failed to extract metadata from sibling #{sibling}", MU::WARN diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index ee272280d..36eaf3f9c 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -138,8 +138,11 @@ def self.find(**args) else "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id] end - desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes - found[args[:cloud_id]] = desc if desc + begin + desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes + found[args[:cloud_id]] = desc if desc + rescue ::Aws::SNS::Errors::NotFound + end else next_token = nil begin From a3a87b43113e5c75df5d3ac147781026ffd11fd7 Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 24 Jul 2020 13:37:27 -0400 Subject: [PATCH 052/107] AWS::NoSQLDB: add a MU::SUMMARY line at the end of groom --- modules/mu/providers/aws/nosqldb.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 9c8b44a5a..61b70c4f8 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -172,6 +172,7 @@ def tagTable # Called automatically by {MU::Deploy#createResources} def groom tagTable if !@config['scrub_mu_isms'] + MU.log "NoSQL Table #{@config['name']}: #{@cloud_id}", MU::SUMMARY end # Does this resource type exist as a global (cloud-wide) artifact, or From 1038be624d8b68f183c3a12a1367eb16a286d3b4 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 27 Jul 2020 03:50:13 -0400 Subject: [PATCH 053/107] AWS::Function: expose ability to invoke a Lambda function after it's groomed --- modules/mu/providers/aws/function.rb | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 82dc4b8a5..cb62c0a94 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -125,6 +125,23 @@ def groom } end + + if @config['invoke_on_completion'] + invoke_params = { + function_name: @cloud_id, + invocation_type: @config['invoke_on_completion']['invocation_type'], + log_type: "Tail" + } + if @config['invoke_on_completion']['payload'] + invoke_params[:payload] = JSON.generate(@config['invoke_on_completion']['payload']) + end + resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).invoke(invoke_params) + if resp.status_code == 200 + MU.log "Invoked #{@cloud_id}", MU::NOTICE, details: Base64.decode64(resp.log_result) + else + MU.log "Invoked #{@cloud_id} and got #{resp.status_code} (#{resp.function_error})", MU::WARN, details: Base64.decode64(resp.log_result) + end + end end # Intended to be called by other Mu resources, such as Endpoints (API @@ -418,6 +435,22 @@ def toKitten(**_args) def self.schema(_config) toplevel_required = ["runtime"] schema = { + "invoke_on_completion" => { + "type" => "object", + "description" => "Setting this will cause this Lambda function to be invoked when its groom phase is complete.", + "required" => ["invocation_type"], + "properties" => { + "invocation_type" => { + "type" => "string", + "enum" => ["RequestResponse", "Event", "Dryrun"], + "default" => "RequestReponse" + }, + "payload" => { + "type" => "object", + "description" => "Optional input to the function, which will be formatted as JSON and sent for execution" + } + } + }, "triggers" => { "type" => "array", "items" => { From 1969b42c3b3c605521081079911a170d6a977164 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 28 Jul 2020 03:58:44 -0400 Subject: [PATCH 054/107] framing out a new CDN type so I have somewhere to put CloudFront later --- modules/mu/cleanup.rb | 2 +- modules/mu/cloud.rb | 14 +++ modules/mu/config/cdn.rb | 48 +++++++++++ modules/mu/providers/aws.rb | 9 ++ modules/mu/providers/aws/cdn.rb | 146 ++++++++++++++++++++++++++++++++ 5 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 modules/mu/config/cdn.rb create mode 100644 modules/mu/providers/aws/cdn.rb diff --git a/modules/mu/cleanup.rb b/modules/mu/cleanup.rb index b1eb86175..84505a491 100644 --- a/modules/mu/cleanup.rb +++ b/modules/mu/cleanup.rb @@ -32,7 +32,7 @@ class Cleanup # Resource types, in the order in which we generally have to clean them up # to disentangle them from one another. - TYPES_IN_ORDER = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "Job", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"] + TYPES_IN_ORDER = ["Collection", "CDN", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "Job", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"] # Purge all resources associated with a deployment. # @param deploy_id [String]: The identifier of the deployment to remove (typically seen in the MU-ID tag on a resource). diff --git a/modules/mu/cloud.rb b/modules/mu/cloud.rb index 515848c32..44e7c0d04 100644 --- a/modules/mu/cloud.rb +++ b/modules/mu/cloud.rb @@ -151,6 +151,9 @@ class NoSQLDB; # Stub base class; real implementations generated at runtime class Job; end + # Stub base class; real implementations generated at runtime + class CDN; + end # Denotes a resource implementation which is missing significant # functionality or is largely untested. @@ -453,6 +456,17 @@ class Job; :waits_on_parent_completion => false, :class => @@generic_class_methods, :instance => @@generic_instance_methods + [:groom] + }, + :CDN => { + :has_multiples => false, + :can_live_in_vpc => false, + :cfg_name => "cdn", + :cfg_plural => "cdns", + :interface => self.const_get("Job"), + :deps_wait_on_my_creation => true, + :waits_on_parent_completion => false, + :class => @@generic_class_methods, + :instance => @@generic_instance_methods + [:groom] } }.freeze diff --git a/modules/mu/config/cdn.rb b/modules/mu/config/cdn.rb new file mode 100644 index 000000000..6e3e5450d --- /dev/null +++ b/modules/mu/config/cdn.rb @@ -0,0 +1,48 @@ +# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved +# +# Licensed under the BSD-3 license (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the root of the project or at +# +# http://egt-labs.com/mu/LICENSE.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module MU + class Config + # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/job.rb + class CDN + + # Base configuration schema for a scheduled job + # @return [Hash] + def self.schema + { + "type" => "object", + "additionalProperties" => false, + "description" => "A cloud provider-specific facility for triggered or scheduled tasks, such as AWS CloudWatch Events or Google Cloud Scheduler.", + "properties" => { + "name" => { + "type" => "string" + }, + "credentials" => MU::Config.credentials_primitive + } + } + end + + # Generic pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated. + # @param _job [Hash]: The resource to process and validate + # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member + # @return [Boolean]: True if validation succeeded, False otherwise + def self.validate(_job, _configurator) + ok = true + + ok + end + + end + end +end diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index be25661a1..7b21ec596 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -1181,6 +1181,14 @@ def self.kms(region: MU.curRegion, credentials: nil) @@kms_api[credentials][region] end + # Amazon's CloudFront API + def self.cloudfront(region: MU.curRegion, credentials: nil) + region ||= myRegion + @@cloudfront_api[credentials] ||= {} + @@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFront", region: region, credentials: credentials) + @@cloudfront_api[credentials][region] + end + # Amazon's Organizations API def self.orgs(credentials: nil) @@organizations_api ||= {} @@ -1618,6 +1626,7 @@ def method_missing(method_sym, *arguments) @@organization_api ={} @@dynamo_api ={} @@dynamostream_api ={} + @@cloudfront_api ={} end end end diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb new file mode 100644 index 000000000..c08a46fae --- /dev/null +++ b/modules/mu/providers/aws/cdn.rb @@ -0,0 +1,146 @@ +# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved +# +# Licensed under the BSD-3 license (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the root of the project or at +# +# http://egt-labs.com/mu/LICENSE.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module MU + class Cloud + class AWS + # A scheduled task facility as configured in {MU::Config::BasketofKittens::cdns} + class CDN < MU::Cloud::CDN + + # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us. + # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat + def initialize(**args) + super + @mu_name ||= @deploy.getResourceName(@config["name"]) + end + + # Called automatically by {MU::Deploy#createResources} + def create + end + + # Called automatically by {MU::Deploy#createResources} + def groom + end + + # Canonical Amazon Resource Number for this resource + # @return [String] + def arn + cloud_desc ? cloud_desc.arn : nil + end + + # Return the metadata for this cdn + # @return [Hash] + def notify + MU.structToHash(cloud_desc, stringify_keys: true) + end + + # Does this resource type exist as a global (cloud-wide) artifact, or + # is it localized to a region/zone? + # @return [Boolean] + def self.isGlobal? + true + end + + # Denote whether this resource implementation is experiment, ready for + # testing, or ready for production use. + def self.quality + MU::Cloud::ALPHA + end + + # Remove all cdns associated with the currently loaded deployment. + # @param noop [Boolean]: If true, will only print what would be done + # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server + # @param region [String]: The cloud provider region + # @return [void] + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + end + + # Locate an existing event. + # @return [Hash]: The cloud provider's complete descriptions of matching CloudWatch Event + def self.find(**args) + found = {} + + MU::Cloud::AWS.cloudfront(credentials: args[:credentials]).list_distributions.distribution_list.items.each { |d| + next if args[:cloud_id] and ![d.id, d.arn].include?(args[:cloud_id]) + found[d.id] = d + } + + found + end + + # Reverse-map our cloud description into a runnable config hash. + # We assume that any values we have in +@config+ are placeholders, and + # calculate our own accordingly based on what's live in the cloud. + def toKitten(**_args) + bok = { + "cloud" => "AWS", + "credentials" => @config['credentials'], + "cloud_id" => @cloud_id + } + + if !cloud_desc + MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config + return nil + end + + resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).list_tags_for_resource(resource: arn) + if resp and resp.tags and resp.tags.items + tags = MU.structToHash(resp.tags.items, stringify_keys: true) + bok['name'] = MU::Adoption.tagsToName(tags) + bok['tags'] = tags + end + + if !bok['name'] + bok['name'] = if cloud_desc.domain_name !~ /\.cloudfront\.net$/ + cloud_desc.domain_name.sub(/\..*/, '') + elsif cloud_desc.aliases and !cloud_desc.aliases.items.empty? + cloud_desc.aliases.items.first.sub(/\..*/, '') + # XXX maybe try to guess from the name of an origin resource? + else + @cloud_id + end + end + + MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc + MU.log @cloud_id+" bok", MU::NOTICE, details: bok + + bok + end + + + # Cloud-specific configuration properties. + # @param _config [MU::Config]: The calling MU::Config object + # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource + def self.schema(_config) + toplevel_required = [] + + [toplevel_required, schema] + end + + # Cloud-specific pre-processing of {MU::Config::BasketofKittens::cdns}, bare and unvalidated. + # @param cdn [Hash]: The resource to process and validate + # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member + # @return [Boolean]: True if validation succeeded, False otherwise + def self.validateConfig(cdn, _configurator) + ok = true + + ok + end + + private + + end + end + end +end From 786ec109d48f6c16fa322714c084fc3b56c04825 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 29 Jul 2020 03:18:37 -0400 Subject: [PATCH 055/107] AWS::CDN: decide on some schema for origins and work on importing them in toKitten --- modules/mu/config/cdn.rb | 16 ++++++++++++++- modules/mu/providers/aws/bucket.rb | 26 ++++++++++++++++++++++--- modules/mu/providers/aws/cdn.rb | 31 +++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/modules/mu/config/cdn.rb b/modules/mu/config/cdn.rb index 6e3e5450d..2e01558b2 100644 --- a/modules/mu/config/cdn.rb +++ b/modules/mu/config/cdn.rb @@ -28,7 +28,21 @@ def self.schema "name" => { "type" => "string" }, - "credentials" => MU::Config.credentials_primitive + "credentials" => MU::Config.credentials_primitive, + "origins" => { + "type" => "array", + "items" => { + "type" => "object", + "properties" => { + "domain_name" => { + "type" => "string" + }, + "origin_path" => { + "type" => "string" + } + } + } + } } } end diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index 58153be0a..9ffa16cba 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -331,9 +331,27 @@ def notify def self.find(**args) found = {} + args[:region] ||= MU::Cloud::AWS.myRegion(args[:credentials]) + + location = Proc.new { |name| + begin + loc_resp = MU::Cloud::AWS.s3(credentials: args[:credentials], region: args[:region]).get_bucket_location(bucket: name) + + if loc_resp.location_constraint and !loc_resp.location_constraint.empty? + loc_resp.location_constraint + else + nil + end + rescue Aws::S3::Errors::AccessDenied + nil + end + } + if args[:cloud_id] begin found[args[:cloud_id]] = describe_bucket(args[:cloud_id], minimal: true, credentials: args[:credentials], region: args[:region]) + found[args[:cloud_id]]['region'] ||= location.call(args[:cloud_id]) + found[args[:cloud_id]]['region'] ||= args[:region] rescue ::Aws::S3::Errors::NoSuchBucket end else @@ -341,11 +359,13 @@ def self.find(**args) if resp and resp.buckets resp.buckets.each { |b| begin - loc_resp = MU::Cloud::AWS.s3(credentials: args[:credentials], region: args[:region]).get_bucket_location(bucket: b.name) - if !loc_resp or loc_resp.location_constraint != args[:region] + bucket_region = location.call(b.name) + if !args[:allregions] and bucket_region != args[:region] next end - found[b.name] = describe_bucket(b.name, minimal: true, credentials: args[:credentials], region: args[:region]) + bucket_region ||= args[:region] + found[b.name] = describe_bucket(b.name, minimal: true, credentials: args[:credentials], region: bucket_region) + found[b.name]["region"] ||= bucket_region rescue Aws::S3::Errors::AccessDenied end } diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index c08a46fae..4e0e778d0 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -112,7 +112,26 @@ def toKitten(**_args) end end - MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc + cloud_desc.origins.items.each { |o| + bok['origins'] ||= [] + origin = { "origin_path" => o.origin_path } + if o.s3_origin_config + buckets = MU::Cloud.resourceClass("AWS", "Bucket").find(credentials: @credentials, allregions: true, cloud_id: o.domain_name.sub(/\..*/, '')) + if buckets and buckets.size == 1 + pp buckets + origin["bucket"] = MU::Config::Ref.get( + id: buckets.keys.first, + type: "buckets", + region: buckets.values.first["region"], + credentials: @credentials, + cloud: "AWS" + ) + end + end + origin["domain_name"] = o.domain_name if !origin["bucket"] + bok['origins'] << origin + } + MU.log @cloud_id+" bok", MU::NOTICE, details: bok bok @@ -125,6 +144,16 @@ def toKitten(**_args) def self.schema(_config) toplevel_required = [] + schema = { + "origins" => { + "items" => { + "properties" => { + "bucket" => MU::Config::Ref.schema(type: "buckets", desc: "Reference an S3 bucket for use as an origin") + } + } + } + } + [toplevel_required, schema] end From 5a02874ad6202921d5e6d50a71b3a7f68ecae186 Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 31 Jul 2020 00:35:52 -0400 Subject: [PATCH 056/107] AWS::CDN: Creation and deletion of S3-backed CloudFront distributions --- modules/mu/config/cdn.rb | 37 ++- modules/mu/providers/aws/bucket.rb | 62 +++-- modules/mu/providers/aws/cdn.rb | 425 ++++++++++++++++++++++++++++- 3 files changed, 496 insertions(+), 28 deletions(-) diff --git a/modules/mu/config/cdn.rb b/modules/mu/config/cdn.rb index 2e01558b2..e08584c8f 100644 --- a/modules/mu/config/cdn.rb +++ b/modules/mu/config/cdn.rb @@ -23,21 +23,54 @@ def self.schema { "type" => "object", "additionalProperties" => false, - "description" => "A cloud provider-specific facility for triggered or scheduled tasks, such as AWS CloudWatch Events or Google Cloud Scheduler.", + "required" => ["origins"], "properties" => { "name" => { "type" => "string" }, + "default_object" => { + "type" => "string", + "default" => "index.html" + }, "credentials" => MU::Config.credentials_primitive, + "aliases" => { + "type" => "array", + "items" => { + "type" => "string" + } + }, "origins" => { "type" => "array", + "minItems" => 1, "items" => { "type" => "object", + "required" => ["name"], "properties" => { + "name" => { + "type" => "string" + }, "domain_name" => { "type" => "string" }, - "origin_path" => { + "path" => { + "type" => "string", + "default" => "" + } + } + } + }, + "behaviors" => { + "type" => "array", + "items" => { + "type" => "object", + "properties" => { + "origin" => { + "type" => "string" + }, + "is_default" => { + "type" => "boolean" + }, + "path_pattern" => { "type" => "string" } } diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index 9ffa16cba..d593c78a3 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -85,6 +85,10 @@ def tagBucket end + def url + "https://#{@cloud_id}.s3.amazonaws.com" + end + # Called automatically by {MU::Deploy#createResources} def groom @@ -111,25 +115,6 @@ def groom } end - if @config['web'] and current["website"].nil? - MU.log "Enabling web service on S3 bucket #{@cloud_id}", MU::NOTICE - MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_website( - bucket: @cloud_id, - website_configuration: { - error_document: { - key: @config['web_error_object'] - }, - index_document: { - suffix: @config['web_index_object'] - } - } - ) - elsif !@config['web'] and !current["website"].nil? - MU.log "Disabling web service on S3 bucket #{@cloud_id}", MU::NOTICE - MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).delete_bucket_website( - bucket: @cloud_id - ) - end if @config['versioning'] and current["versioning"].status != "Enabled" MU.log "Enabling versioning on S3 bucket #{@cloud_id}", MU::NOTICE @@ -172,6 +157,43 @@ def groom } end + if @config['web'] and current["website"].nil? + MU.log "Enabling web service on S3 bucket #{@cloud_id}", MU::NOTICE + MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_website( + bucket: @cloud_id, + website_configuration: { + error_document: { + key: @config['web_error_object'] + }, + index_document: { + suffix: @config['web_index_object'] + } + } + ) + ['web_error_object', 'web_index_object'].each { |key| + begin + MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).head_object( + bucket: @cloud_id, + key: @config[key] + ) + rescue Aws::S3::Errors::NotFound + MU.log "Uploading placeholder #{@config[key]} to bucket #{@cloud_id}" + MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_object( + acl: "public-read", + bucket: @cloud_id, + key: @config[key], + body: "" + ) + end + } +# XXX check if error and index objs exist, and if not provide placeholders + elsif !@config['web'] and !current["website"].nil? + MU.log "Disabling web service on S3 bucket #{@cloud_id}", MU::NOTICE + MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).delete_bucket_website( + bucket: @cloud_id + ) + end + MU.log "Bucket #{@config['name']}: s3://#{@cloud_id}", MU::SUMMARY if @config['web'] MU.log "Bucket #{@config['name']} web access: http://#{@cloud_id}.s3-website-#{@config['region']}.amazonaws.com/", MU::SUMMARY @@ -352,6 +374,7 @@ def self.find(**args) found[args[:cloud_id]] = describe_bucket(args[:cloud_id], minimal: true, credentials: args[:credentials], region: args[:region]) found[args[:cloud_id]]['region'] ||= location.call(args[:cloud_id]) found[args[:cloud_id]]['region'] ||= args[:region] + found[args[:cloud_id]]['name'] ||= args[:cloud_id] rescue ::Aws::S3::Errors::NoSuchBucket end else @@ -366,6 +389,7 @@ def self.find(**args) bucket_region ||= args[:region] found[b.name] = describe_bucket(b.name, minimal: true, credentials: args[:credentials], region: bucket_region) found[b.name]["region"] ||= bucket_region + found[b.name]['name'] ||= b.name rescue Aws::S3::Errors::AccessDenied end } diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index 4e0e778d0..797631a3f 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -27,6 +27,30 @@ def initialize(**args) # Called automatically by {MU::Deploy#createResources} def create + resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_cloud_front_origin_access_identity( + cloud_front_origin_access_identity_config: { + caller_reference: @mu_name, + comment: @mu_name + } + ) + + @origin_access_identity = "origin-access-identity/cloudfront/"+resp.cloud_front_origin_access_identity.id + + params = get_properties + begin + MU.log "Creating CloudFront distribution #{@mu_name}", details: params + resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_distribution_with_tags( + distribution_config_with_tags: { + distribution_config: params, + tags: { + items: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } } + } + } + ) + @cloud_id = resp.distribution.id + rescue ::Aws::CloudFront::Errors::InvalidOrigin => e + raise MuError.new e.message, details: params[:origins] + end end # Called automatically by {MU::Deploy#createResources} @@ -45,6 +69,21 @@ def notify MU.structToHash(cloud_desc, stringify_keys: true) end + # Wait until the distribution is ready (status is +Deployed+) + def ready? + self.class.ready?(@cloud_id, credentials: @credentials) + end + + # Wait until a distribution is ready (status is +Deployed+) + # @param id [String] + # @param credentials [String] + def self.ready?(id, credentials: nil) + desc = nil + MU.retrier([], loop_if: Proc.new { !desc or desc.status != "Deployed" }, wait: 30, max:60) { + desc = MU::Cloud::AWS.cloudfront(credentials: credentials).get_distribution(id: id).distribution + } + end + # Does this resource type exist as a global (cloud-wide) artifact, or # is it localized to a region/zone? # @return [Boolean] @@ -61,9 +100,61 @@ def self.quality # Remove all cdns associated with the currently loaded deployment. # @param noop [Boolean]: If true, will only print what would be done # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server - # @param region [String]: The cloud provider region # @return [void] - def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) + def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) + resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_distributions + if resp and resp.distribution_list and resp.distribution_list.items + ids = Hash[resp.distribution_list.items.map { |distro| [distro.arn, distro] }] + ids.each_key { |arn| + tags = MU::Cloud::AWS.cloudfront(credentials: credentials).list_tags_for_resource(resource: arn).tags.items + + found_muid = found_master = false + name = nil + tags.each { |tag| + name = tag.value if tag.key == "Name" + found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id + found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip + } + + if found_muid and (ignoremaster or found_master) + current = MU::Cloud::AWS.cloudfront(credentials: credentials).get_distribution_config(id: ids[arn].id) + etag = current.etag + + if !noop + + if current.distribution_config.enabled + newcfg = MU.structToHash(current.distribution_config) + newcfg[:enabled] = false + MU.log "Disabling CloudFront distribution #{name ? name : ids[arn].id})", MU::NOTICE + updated = MU::Cloud::AWS.cloudfront(credentials: credentials).update_distribution(id: ids[arn].id, distribution_config: newcfg, if_match: etag) + etag = updated.etag + end + + end + + MU.log "Deleting CloudFront distribution #{name ? name : ids[arn].id})" + if !noop + ready?(ids[arn].id, credentials: credentials) + MU::Cloud::AWS.cloudfront(credentials: credentials).delete_distribution(id: ids[arn].id, if_match: etag) + end + end + } + end + + resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_cloud_front_origin_access_identities + if resp and resp.cloud_front_origin_access_identity_list and + resp.cloud_front_origin_access_identity_list.items.each and + deploy_id =~ /-\d{10}-[A-Z]{2}/ + resp.cloud_front_origin_access_identity_list.items.each { |ident| + if ident.comment =~ /^#{Regexp.quote(deploy_id)}-/ + MU.log "Deleting CloudFront origin access identity #{ident.id} (#{ident.comment})" + if !noop + getresp = MU::Cloud::AWS.cloudfront(credentials: credentials).get_cloud_front_origin_access_identity(id: ident.id) + MU::Cloud::AWS.cloudfront(credentials: credentials).delete_cloud_front_origin_access_identity(id: ident.id, if_match: getresp.etag) + end + end + } + end end # Locate an existing event. @@ -98,7 +189,7 @@ def toKitten(**_args) if resp and resp.tags and resp.tags.items tags = MU.structToHash(resp.tags.items, stringify_keys: true) bok['name'] = MU::Adoption.tagsToName(tags) - bok['tags'] = tags + bok['tags'] = tags if !tags.empty? end if !bok['name'] @@ -114,11 +205,13 @@ def toKitten(**_args) cloud_desc.origins.items.each { |o| bok['origins'] ||= [] - origin = { "origin_path" => o.origin_path } + origin = { + "path" => o.origin_path, + "name" => o.id + } if o.s3_origin_config buckets = MU::Cloud.resourceClass("AWS", "Bucket").find(credentials: @credentials, allregions: true, cloud_id: o.domain_name.sub(/\..*/, '')) if buckets and buckets.size == 1 - pp buckets origin["bucket"] = MU::Config::Ref.get( id: buckets.keys.first, type: "buckets", @@ -129,11 +222,80 @@ def toKitten(**_args) end end origin["domain_name"] = o.domain_name if !origin["bucket"] + if o.custom_origin_config + origin["http_port"] = o.custom_origin_config.http_port + origin["https_port"] = o.custom_origin_config.https_port + origin["protocol_policy"] = o.custom_origin_config.origin_protocol_policy + origin["ssl_protocols"] = o.custom_origin_config.origin_ssl_protocols.items + end + + if o.custom_headers and !o.custom_headers.empty? + end + bok['origins'] << origin } - MU.log @cloud_id+" bok", MU::NOTICE, details: bok + if cloud_desc.aliases and cloud_desc.aliases.items and + !cloud_desc.aliases.items.empty? + bok['aliases'] = cloud_desc.aliases.items + end + + bok['disabled'] = true if !cloud_desc.enabled + + bok['behaviors'] = [] + + add_behavior = Proc.new { |b, default| + behavior = {} + + behavior["origin"] = b.target_origin_id + behavior["is_default"] = true if default + behavior["protocol_policy"] = b.viewer_protocol_policy + if b.lambda_function_associations and !b.lambda_function_associations.items.empty? + b.lambda_function_associations.items.each { |f| + behavior['functions'] ||= [] + f.lambda_function_arn.match(/^arn:.*?:lambda:([^:]+?):(\d*):function:([^:]+)/) + region = Regexp.last_match[1] + acct = Regexp.last_match[2] + id = Regexp.last_match[3] + behavior['functions'] << MU::Config::Ref.get( + id: id, + region: region, + type: "functions", + event_type: f.event_type, + include_body: f.include_body, + cloud: "AWS", + credentials: @credentials, + habitat: MU::Config::Ref.get( + id: acct, + cloud: "AWS", + credentials: @credentials + ) + ) + } + [:min_ttl, :default_ttl, :max_ttl].each { |ttl| + behavior[ttl.to_s] = b.send(ttl) + } + end + bok['behaviors'] << behavior + } + + add_behavior.call(cloud_desc.default_cache_behavior, true) + if cloud_desc.cache_behaviors and + !cloud_desc.cache_behaviors.items.empty? + cloud_desc.cache_behaviors.items.each { |b| + add_behavior.call(b, false) + bok['behaviors'] << { + "origin" => b.target_origin_id, + "path_pattern" => b.path_pattern + } + } + end + +if bok['name'] == "espier" + MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc + MU.log @cloud_id+" bok", MU::NOTICE, details: bok +end bok end @@ -145,15 +307,143 @@ def self.schema(_config) toplevel_required = [] schema = { + "disabled" => { + "type" => "boolean", + "default" => false + }, + "behaviors" => { + "items" => { + "properties" => { + "min_ttl" => { + "type" => "integer", + "default" => 0 + }, + "default_ttl" => { + "type" => "integer", + "default" => 86400 + }, + "max_ttl" => { + "type" => "integer", + "default" => 31536000 + }, + "protocol_policy" => { + "type" => "string", + "enum" => %w{allow-all https-only redirect-to-https}, + "default" => "redirect-to-https" + }, + "functions" => { + "type" => "array", + "items" => MU::Config::Ref.schema(type: "functions", desc: "Add a Lambda function which can be invoked on requests or responses through this distribution.") + }, + "forwarded_values" => { + "type" => "object", + "default" => { + "query_string" => false, + "cookies" => { + "forward" => "none" + } + }, + "properties" => { + "query_string" => { + "type" => "boolean", + "default" => false + }, + "cookies" => { + "type" => "object", + "properties" => { + "forward" => { + "type" => "string", + "enum" => %w{none whitelist all} + }, + "whitelisted_names" => { + "type" => "array", + "items" => { + "type" => "string" + } + }, + } + }, + "headers" => { + "type" => "array", + "items" => { + "type" => "string" + } + }, + "query_string_cache_keys" => { + "type" => "array", + "items" => { + "type" => "string" + } + } + } + } + } + } + }, "origins" => { "items" => { "properties" => { - "bucket" => MU::Config::Ref.schema(type: "buckets", desc: "Reference an S3 bucket for use as an origin") + "bucket" => MU::Config::Ref.schema(type: "buckets", desc: "Reference an S3 bucket for use as an origin"), + "loadbalancer" => MU::Config::Ref.schema(type: "loadbalancers", desc: "Reference a Load Balancer for use as an origin"), + "connection_attempts" => { + "type" => "integer", + "default" => 3 + }, + "connection_timeout" => { + "type" => "integer", + "default" => 10 + }, + "protocol_policy" => { + "type" => "string", + "enum" => %w{http-only https-only match-viewer}, + "default" => "match-viewer" + }, + "ssl_protocols" => { + "type" => "array", + "default" => ["TLSv1.2"], + "items" => { + "type" => "string", + "enum" => %w{SSLv3 TLSv1 TLSv1.1 TLSv1.2}, + } + }, + "http_port" => { + "type" => "integer", + "default" => 80 + }, + "https_port" => { + "type" => "integer", + "default" => 443 + }, + "custom_headers" => { + "type" => "array", + "items" => { + "type" => "object", + "required" => ["key", "value"], + "properties" => { + "key" => { + "type" => "string" + }, + "value" => { + "type" => "string" + }, + } + } + } } } } } + schema["behaviors"]["items"]["properties"]["functions"]["items"]["include_body"] = { + "type" => "boolean", + "default" => false + } + schema["behaviors"]["items"]["properties"]["functions"]["items"]["event_type"] = { + "type" => "string", + "enum" => %w{viewer-request viewer-response origin-request origin-response}, + "default" => "viewer-request" + } + [toplevel_required, schema] end @@ -164,11 +454,132 @@ def self.schema(_config) def self.validateConfig(cdn, _configurator) ok = true + cdn['origins'].each { |o| + if o['bucket'] + target_ref = MU::Config::Ref.get(o['bucket']) + if target_ref.name + MU::Config.addDependency(cdn, target_ref.name, "buckets", phase: "groom") + end + end + } + + cdn['behaviors'].each { |b| + b['path_pattern'] ||= "" + } + ok end private + def get_properties + params = { + default_root_object: @config['default_object'], + caller_reference: @mu_name, # eh, probably should be random + origins: { + quantity: @config['origins'].size, + items: [] + }, + comment: @deploy.deploy_id, + enabled: !(@config['disabled']) + } + + @config['origins'].each { |o| + origin = { + id: o['name'], + } + if o['bucket'] + bucket_obj = MU::Config::Ref.get(o['bucket']).kitten(@deploy, cloud: "AWS") + if !bucket_obj + raise MuError.new "Failed to resolve bucket referenced in CloudFront distribution #{@config['name']}", details: o['bucket'].to_h + end + origin[:domain_name] = bucket_obj.cloud_desc["name"]+".s3.amazonaws.com" + origin[:origin_path] = o['path'] if o['path'] + origin[:s3_origin_config] = { + origin_access_identity: @origin_access_identity + } + else # XXX make sure parser guarantees these are present + origin[:domain_name] = o['domain_name'] + origin[:origin_path] = o['path'] + end + + if o['custom_headers'] + origin[:custom_headers] = { + quantity: o['custom_headers'].size, + items: o['custom_headers'].map { |h| + { + header_name: h['key'], + header_value: h['value'] + } + } + } + end + + [:connection_attempts, :connection_timeout].each { |field| + origin[field] = o[field.to_s] + } + + params[:origins][:items] << origin + } + + if @config['aliases'] + params[:aliases] = { + items: @config['aliases'], + quantity: @config['aliases'].size + } + end + + # XXX config parser should guarantee a default behavior + @config['behaviors'].each { |b| + b['origin'] ||= @config['origins'].first['name'] + behavior = { + target_origin_id: b['origin'], + viewer_protocol_policy: b['protocol_policy'], + min_ttl: b['min_ttl'], + max_ttl: b['max_ttl'], + default_ttl: b['default_ttl'], + } + behavior[:trusted_signers] = { + enabled: false, + quantity: 0, +# items: [] + } + behavior[:forwarded_values] = { + query_string: b['forwarded_values']['query_string'], + cookies: { + forward: b['forwarded_values']['cookies']['forward'] + } + } + if b['forwarded_values']['cookies']['whitelisted_names'] + behavior[:forwarded_values][:cookies][:whitelisted_names] = { + quantity: b['forwarded_values']['cookies']['whitelisted_names'].size, + items: b['forwarded_values']['cookies']['whitelisted_names'] + } + end + ['headers', 'query_string_cache_keys'].each { |field| + if b['forwarded_values'][field] + behavior[:forwarded_values][field.to_sym] = { + quantity: b['forwarded_values'][field].size, + items: b['forwarded_values'][field] + } + end + } + + if b['is_default'] or @config['behaviors'].size == 1 + params[:default_cache_behavior] = behavior + else + behavior[:path_pattern] = b['path_pattern'] + params[:cache_behaviors] ||= { + quantity: (@config['behaviors'].size-1), + items: [] + } + params[:cache_behaviors][:items] << behavior + end + } + + params + end + end end end From 4af743e1acef16b5e469f2b49ca835ad4b61538a Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 31 Jul 2020 21:45:58 -0400 Subject: [PATCH 057/107] AWS::CDN: Stick non-default SSL certs onto CloudFront, and try some autodetection when aliases are specified without a cert --- modules/mu/providers/aws.rb | 12 +++-- modules/mu/providers/aws/cdn.rb | 83 +++++++++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 13 deletions(-) diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index 7b21ec596..0eb7748bb 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -852,21 +852,21 @@ def self.listInstanceTypes(region = myRegion) # @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised. # @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied. # @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired. - def self.findSSLCertificate(name: nil, id: nil, region: myRegion) + def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true) if name.nil? and name.empty? and id.nil? and id.empty? raise MuError, "Can't call findSSLCertificate without specifying either a name or an id" end if !name.nil? and !name.empty? matches = [] - acmcerts = MU::Cloud::AWS.acm(region: region).list_certificates( + acmcerts = MU::Cloud::AWS.acm(region: region, credentials: credentials).list_certificates( certificate_statuses: ["ISSUED"] ) acmcerts.certificate_summary_list.each { |cert| matches << cert.certificate_arn if cert.domain_name == name } begin - iamcert = MU::Cloud::AWS.iam.get_server_certificate( + iamcert = MU::Cloud::AWS.iam(credentials: credentials).get_server_certificate( server_certificate_name: name ) rescue Aws::IAM::Errors::ValidationError, Aws::IAM::Errors::NoSuchEntity @@ -878,7 +878,11 @@ def self.findSSLCertificate(name: nil, id: nil, region: myRegion) if matches.size == 1 return matches.first elsif matches.size == 0 - raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}" + if raise_on_missing + raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}" + else + return nil + end elsif matches.size > 1 raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use." end diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index 797631a3f..f61782dae 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -48,8 +48,11 @@ def create } ) @cloud_id = resp.distribution.id + ready? rescue ::Aws::CloudFront::Errors::InvalidOrigin => e raise MuError.new e.message, details: params[:origins] + rescue ::Aws::CloudFront::Errors::InvalidArgument => e + raise MuError.new e.message, details: params end end @@ -105,6 +108,7 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, cred resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_distributions if resp and resp.distribution_list and resp.distribution_list.items ids = Hash[resp.distribution_list.items.map { |distro| [distro.arn, distro] }] + pp ids.keys ids.each_key { |arn| tags = MU::Cloud::AWS.cloudfront(credentials: credentials).list_tags_for_resource(resource: arn).tags.items @@ -249,6 +253,7 @@ def toKitten(**_args) behavior["origin"] = b.target_origin_id behavior["is_default"] = true if default + behavior["path_pattern"] = b.path_pattern if b.respond_to?(:path_pattern) behavior["protocol_policy"] = b.viewer_protocol_policy if b.lambda_function_associations and !b.lambda_function_associations.items.empty? b.lambda_function_associations.items.each { |f| @@ -285,16 +290,12 @@ def toKitten(**_args) !cloud_desc.cache_behaviors.items.empty? cloud_desc.cache_behaviors.items.each { |b| add_behavior.call(b, false) - bok['behaviors'] << { - "origin" => b.target_origin_id, - "path_pattern" => b.path_pattern - } } end -if bok['name'] == "espier" - MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc - MU.log @cloud_id+" bok", MU::NOTICE, details: bok +if @cloud_id == "E2SVQ5DTYCQ22P" +# MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc +# MU.log @cloud_id+" bok", MU::NOTICE, details: bok end bok end @@ -311,6 +312,7 @@ def self.schema(_config) "type" => "boolean", "default" => false }, + "certificate" => MU::Config::Ref.schema(type: "certificate", desc: "Required if any domains have been specified with +aliases+; parser will attempt to autodetect a valid ACM or IAM certificate if not specified.", omit_fields: ["cloud", "tag", "deploy_id"]), "behaviors" => { "items" => { "properties" => { @@ -463,8 +465,57 @@ def self.validateConfig(cdn, _configurator) end } + if cdn['certificate'] + cdn['certificate']['region'] ||= cdn['region'] if !cdn['certificate']['id'] + cdn['certificate']['credentials'] ||= cdn['credentials'] + cert_arn = MU::Cloud::AWS.findSSLCertificate( + name: cdn['certificate']["name"], + id: cdn['certificate']["id"], + region: cdn['certificate']['region'], + credentials: cdn['certificate']['credentials'] + ) + if !cert_arn + MU.log "Failed to find an ACM or IAM certificate specified in CloudFront distribution #{cdn['name']}", MU::ERR, details: cdn['certificate'].to_h + ok = false + else + cdn['certificate']['id'] ||= cert_arn + end + end + + if cdn['aliases'] + cdn['aliases'].each { |a| + if !cdn['certificate'] + foundcert = MU::Cloud::AWS.findSSLCertificate(name: a, region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false) + if !foundcert + foundcert = MU::Cloud::AWS.findSSLCertificate(name: a.sub(/^[^\.]+\./, '*.'), region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false) + end + if !foundcert + MU.log "Failed to find an ACM or IAM certificate matching #{a} for CloudFront distribution #{cdn['name']}", MU::ERR + ok = false + else + cdn['certificate'] = { + "id" => foundcert, + "credentials" => cdn['credentials'] + } + MU.log "Auto-detected SSL certificate for CloudFront distribution #{cdn['name']} alias #{a}", MU::NOTICE, details: cdn['certificate']['id'] + end + else +# XXX make sure the certificate we have meshes with the alias name, ugh + end + } + end + + path_patterns = {} cdn['behaviors'].each { |b| - b['path_pattern'] ||= "" + b['path_pattern'] ||= "*" + path_patterns[b['path_pattern']] ||= 0 + path_patterns[b['path_pattern']] += 1 + } + path_patterns.each_pair { |pattern, origins| + if origins > 1 + MU.log "CDN #{cdn['name']} has #{origins.to_s} uses of path_pattern '#{pattern}' in its behavior list (must be unique)", MU::ERR, details: cdn['behaviors'] + ok = false + end } ok @@ -484,6 +535,20 @@ def get_properties enabled: !(@config['disabled']) } + if @config['certificate'] + params[:viewer_certificate] = { + ssl_support_method: "sni-only" + } + if @config['certificate']['id'] =~ /^arn:aws(?:-us-gov)?:iam/ + params[:viewer_certificate][:iam_certificate_id] = @config['certificate']['id'] + params[:viewer_certificate][:certificate_source] = "iam" + elsif @config['certificate']['id'] =~ /^arn:aws(?:-us-gov)?:acm/ + params[:viewer_certificate][:acm_certificate_arn] = @config['certificate']['id'] + params[:viewer_certificate][:certificate_source] = "acm" + end + + end + @config['origins'].each { |o| origin = { id: o['name'], @@ -565,7 +630,7 @@ def get_properties end } - if b['is_default'] or @config['behaviors'].size == 1 + if b['is_default'] or @config['behaviors'].size == 1 or b['path_pattern'] == "*" params[:default_cache_behavior] = behavior else behavior[:path_pattern] = b['path_pattern'] From c62cc87a27562c7495d990657d290a9c542512da Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 3 Aug 2020 03:36:12 -0400 Subject: [PATCH 058/107] AWS::CDN: quick and dirty DNS --- modules/mu/config/cache_cluster.rb | 2 +- modules/mu/config/cdn.rb | 1 + modules/mu/config/database.rb | 2 +- modules/mu/config/dnszone.rb | 7 ++++--- modules/mu/config/endpoint.rb | 1 + modules/mu/config/server.rb | 2 +- modules/mu/providers/aws/cdn.rb | 18 ++++++++++++++++++ 7 files changed, 27 insertions(+), 6 deletions(-) diff --git a/modules/mu/config/cache_cluster.rb b/modules/mu/config/cache_cluster.rb index 37456320a..d9faa16e2 100644 --- a/modules/mu/config/cache_cluster.rb +++ b/modules/mu/config/cache_cluster.rb @@ -54,7 +54,7 @@ def self.schema "type" => "string", "default" => "redis" }, - "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true), + "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "cache_cluster"), "dns_sync_wait" => { "type" => "boolean", "description" => "Wait for DNS record to propagate in DNS Zone.", diff --git a/modules/mu/config/cdn.rb b/modules/mu/config/cdn.rb index e08584c8f..8d5d0affc 100644 --- a/modules/mu/config/cdn.rb +++ b/modules/mu/config/cdn.rb @@ -28,6 +28,7 @@ def self.schema "name" => { "type" => "string" }, + "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "cdn"), "default_object" => { "type" => "string", "default" => "index.html" diff --git a/modules/mu/config/database.rb b/modules/mu/config/database.rb index 661c31447..624d0bee1 100644 --- a/modules/mu/config/database.rb +++ b/modules/mu/config/database.rb @@ -62,7 +62,7 @@ def self.schema "default" => false }, "member_of_cluster" => MU::Config::Ref.schema(type: "databases", desc: "Internal use"), - "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true), + "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "database"), "dns_sync_wait" => { "type" => "boolean", "description" => "Wait for DNS record to propagate in DNS Zone.", diff --git a/modules/mu/config/dnszone.rb b/modules/mu/config/dnszone.rb index 91348875d..32e83cb43 100644 --- a/modules/mu/config/dnszone.rb +++ b/modules/mu/config/dnszone.rb @@ -60,7 +60,7 @@ def self.schema # @param default_type [String]: The type of record to make default (e.g. An, CNAME, etc) # @param need_zone [Boolean]: Whether to explicitly require a zone be declared # @return [Hash] - def self.records_primitive(need_target: true, default_type: nil, need_zone: false) + def self.records_primitive(need_target: true, default_type: nil, need_zone: false, embedded_type: nil) dns_records_primitive = { "type" => "array", "maxItems" => 100, @@ -107,8 +107,9 @@ def self.records_primitive(need_target: true, default_type: nil, need_zone: fals }, "mu_type" => { "type" => "string", - "description" => "The Mu resource type to search the deployment for.", - "enum" => ["loadbalancer", "server", "database", "cache_cluster"] + "description" => "The mu type of a resource being targeted.", + "enum" => embedded_type ? [embedded_type] : ["loadbalancer", "server", "database", "cache_cluster", "endpoint", "cdn"], + "default" => embedded_type }, "target_type" => { "description" => "If the target is a public or a private resource. This only applies to servers/server_pools when using automatic DNS registration. If set to public but the target only has a private address, the private address will be used", diff --git a/modules/mu/config/endpoint.rb b/modules/mu/config/endpoint.rb index 9bdd99f06..1adc4c722 100644 --- a/modules/mu/config/endpoint.rb +++ b/modules/mu/config/endpoint.rb @@ -32,6 +32,7 @@ def self.schema "iam_role" => {"type" => "string"}, "region" => MU::Config.region_primitive, "vpc" => MU::Config::VPC.reference(MU::Config::VPC::NO_SUBNETS, MU::Config::VPC::NO_NAT_OPTS), + "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "endpoint"), "methods" => { "type" => "array", "items" => { diff --git a/modules/mu/config/server.rb b/modules/mu/config/server.rb index f9c7fb0d6..96f4111af 100644 --- a/modules/mu/config/server.rb +++ b/modules/mu/config/server.rb @@ -546,7 +546,7 @@ def self.schema "additionalProperties" => false, "description" => "Create individual server instances.", "properties" => { - "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true), + "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true, embedded_type: "server"), "bastion" => { "type" => "boolean", "default" => false, diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index f61782dae..e037bd045 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -58,6 +58,17 @@ def create # Called automatically by {MU::Deploy#createResources} def groom + if !@config['dns_records'].nil? + # XXX this should be a call to @deploy.nameKitten + @config['dns_records'].each { |dnsrec| + dnsrec['name'] ||= @mu_name.downcase + dnsrec['name'] += ".#{MU.environment.downcase}" if dnsrec["append_environment_name"] and dnsrec['name'] !~ /\.#{MU.environment.downcase}$/ + } + + if !MU::Cloud::AWS.isGovCloud? + MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: cloud_desc.domain_name) + end + end end # Canonical Amazon Resource Number for this resource @@ -505,6 +516,13 @@ def self.validateConfig(cdn, _configurator) } end + if cdn['dns_records'] + cdn['dns_records'].each { |rec| +# XXX if this record's domain name jives with our certificate name, add an +# automatic alias for it + } + end + path_patterns = {} cdn['behaviors'].each { |b| b['path_pattern'] ||= "*" From f3486649d6598ffce4bafd24ace50d1e5f30fc73 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 4 Aug 2020 00:45:54 -0400 Subject: [PATCH 059/107] AWS::CDN: do smart things with DNS, SSL, and aliases --- modules/mu/cloud.rb | 2 +- modules/mu/providers/aws.rb | 39 +++++++++++--- modules/mu/providers/aws/cdn.rb | 67 ++++++++++++++++++------ modules/mu/providers/aws/dnszone.rb | 21 +++++++- modules/mu/providers/aws/loadbalancer.rb | 2 +- 5 files changed, 106 insertions(+), 25 deletions(-) diff --git a/modules/mu/cloud.rb b/modules/mu/cloud.rb index 44e7c0d04..3008332f9 100644 --- a/modules/mu/cloud.rb +++ b/modules/mu/cloud.rb @@ -462,7 +462,7 @@ class CDN; :can_live_in_vpc => false, :cfg_name => "cdn", :cfg_plural => "cdns", - :interface => self.const_get("Job"), + :interface => self.const_get("CDN"), :deps_wait_on_my_creation => true, :waits_on_parent_completion => false, :class => @@generic_class_methods, diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index 0eb7748bb..86d4f1be4 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -844,6 +844,8 @@ def self.listInstanceTypes(region = myRegion) @@instance_types end + @@certificates = {} + # AWS can stash API-available certificates in Amazon Certificate Manager # or in IAM. Rather than make people crazy trying to get the syntax # correct in our Baskets of Kittens, let's have a helper that tries to do @@ -853,9 +855,12 @@ def self.listInstanceTypes(region = myRegion) # @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied. # @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired. def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true) - if name.nil? and name.empty? and id.nil? and id.empty? + if (name.nil? or name.empty?) and (id.nil? or id.empty?) raise MuError, "Can't call findSSLCertificate without specifying either a name or an id" end + if id and @@certificates[id] + return [id, @@certificates[id]] + end if !name.nil? and !name.empty? matches = [] @@ -876,7 +881,7 @@ def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: n matches << iamcert.server_certificate.server_certificate_metadata.arn end if matches.size == 1 - return matches.first + id = matches.first elsif matches.size == 0 if raise_on_missing raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}" @@ -888,24 +893,33 @@ def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: n end end + domains = [] + if id.match(/^arn:aws(?:-us-gov)?:acm/) - resp = MU::Cloud::AWS.acm(region: region).get_certificate( + resp = MU::Cloud::AWS.acm(region: region).describe_certificate( certificate_arn: id ) - if resp.nil? + + if resp.nil? or resp.certificate.nil? raise MuError, "No such ACM certificate '#{id}'" end + domains << resp.certificate.domain_name + if resp.certificate.subject_alternative_names + domains.concat(resp.certificate.subject_alternative_names) + end elsif id.match(/^arn:aws(?:-us-gov)?:iam/) resp = MU::Cloud::AWS.iam.list_server_certificates if resp.nil? raise MuError, "No such IAM certificate '#{id}'" end resp.server_certificate_metadata_list.each { |cert| + if cert.arn == id if cert.expiration < Time.now MU.log "IAM SSL certificate #{cert.server_certificate_name} (#{id}) is EXPIRED", MU::WARN end - return id + @@certificates[id] = [cert.server_certificate_name] + return [id, [cert.server_certificate_name]] end } raise MuError, "No such IAM certificate '#{id}'" @@ -913,7 +927,20 @@ def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: n raise MuError, "The format of '#{id}' doesn't look like an ARN for either Amazon Certificate Manager or IAM" end - id + @@certificates[id] = domains.uniq + [id, domains.uniq] + end + + def self.nameMatchesCertificate(name, cert_id) + _id, domains = findSSLCertificate(id: cert_id) + return false if !domains + domains.each { |dom| + if dom == name or + (dom =~ /^\*/ and name =~ /.*#{Regexp.quote(dom[1..-1])}/) + return true + end + } + false end # Amazon Certificate Manager API diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index e037bd045..c7a5a2d78 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -37,6 +37,7 @@ def create @origin_access_identity = "origin-access-identity/cloudfront/"+resp.cloud_front_origin_access_identity.id params = get_properties + begin MU.log "Creating CloudFront distribution #{@mu_name}", details: params resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_distribution_with_tags( @@ -49,6 +50,14 @@ def create ) @cloud_id = resp.distribution.id ready? + rescue ::Aws::CloudFront::Errors::InvalidViewerCertificate => e + cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate( + name: @config['certificate']["name"], + id: @config['certificate']["id"], + region: @config['certificate']['region'], + credentials: @config['certificate']['credentials'] + ) + raise MuError.new e.message, details: { "aliases" => @config['aliases'], "certificate domains" => cert_domains } rescue ::Aws::CloudFront::Errors::InvalidOrigin => e raise MuError.new e.message, details: params[:origins] rescue ::Aws::CloudFront::Errors::InvalidArgument => e @@ -58,17 +67,19 @@ def create # Called automatically by {MU::Deploy#createResources} def groom - if !@config['dns_records'].nil? - # XXX this should be a call to @deploy.nameKitten - @config['dns_records'].each { |dnsrec| - dnsrec['name'] ||= @mu_name.downcase - dnsrec['name'] += ".#{MU.environment.downcase}" if dnsrec["append_environment_name"] and dnsrec['name'] !~ /\.#{MU.environment.downcase}$/ - } + params = get_properties + if !@config['dns_records'].nil? if !MU::Cloud::AWS.isGovCloud? MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: cloud_desc.domain_name) end end + MU.log "CloudFront Distribution #{@config['name']} at #{cloud_desc.domain_name}", MU::SUMMARY + if @config['aliases'] + @config['aliases'].each { |a| + MU.log "Alias for CloudFront Distribution #{@config['name']}: #{a}", MU::SUMMARY + } + end end # Canonical Amazon Resource Number for this resource @@ -119,7 +130,7 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, cred resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_distributions if resp and resp.distribution_list and resp.distribution_list.items ids = Hash[resp.distribution_list.items.map { |distro| [distro.arn, distro] }] - pp ids.keys + ids.each_key { |arn| tags = MU::Cloud::AWS.cloudfront(credentials: credentials).list_tags_for_resource(resource: arn).tags.items @@ -476,10 +487,12 @@ def self.validateConfig(cdn, _configurator) end } + cert_domains = nil + if cdn['certificate'] cdn['certificate']['region'] ||= cdn['region'] if !cdn['certificate']['id'] cdn['certificate']['credentials'] ||= cdn['credentials'] - cert_arn = MU::Cloud::AWS.findSSLCertificate( + cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate( name: cdn['certificate']["name"], id: cdn['certificate']["id"], region: cdn['certificate']['region'], @@ -489,16 +502,17 @@ def self.validateConfig(cdn, _configurator) MU.log "Failed to find an ACM or IAM certificate specified in CloudFront distribution #{cdn['name']}", MU::ERR, details: cdn['certificate'].to_h ok = false else - cdn['certificate']['id'] ||= cert_arn + cdn['certificate']['id'] ||= cert_arn end + cdn['certificate'].delete('region') if cdn['certificate']['region'].nil? end if cdn['aliases'] cdn['aliases'].each { |a| if !cdn['certificate'] - foundcert = MU::Cloud::AWS.findSSLCertificate(name: a, region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false) + foundcert, cert_domains = MU::Cloud::AWS.findSSLCertificate(name: a, region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false) if !foundcert - foundcert = MU::Cloud::AWS.findSSLCertificate(name: a.sub(/^[^\.]+\./, '*.'), region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false) + foundcert, cert_domains = MU::Cloud::AWS.findSSLCertificate(name: a.sub(/^[^\.]+\./, '*.'), region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false) end if !foundcert MU.log "Failed to find an ACM or IAM certificate matching #{a} for CloudFront distribution #{cdn['name']}", MU::ERR @@ -511,15 +525,22 @@ def self.validateConfig(cdn, _configurator) MU.log "Auto-detected SSL certificate for CloudFront distribution #{cdn['name']} alias #{a}", MU::NOTICE, details: cdn['certificate']['id'] end else -# XXX make sure the certificate we have meshes with the alias name, ugh + if !MU::Cloud::AWS.nameMatchesCertificate(a, cdn['certificate']['id']) + MU.log "Alias #{a} in CloudFront distro #{cdn['name']} does not appear to fit any domains on our SSL certificate", MU::ERR, details: cert_domains + ok = false + end end } end - if cdn['dns_records'] + if cdn['dns_records'] and cdn['certificate'] cdn['dns_records'].each { |rec| -# XXX if this record's domain name jives with our certificate name, add an -# automatic alias for it + next if !rec['name'] + dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec) + if MU::Cloud::AWS.nameMatchesCertificate(dnsname, cdn['certificate']['id']) + cdn['aliases'] ||= [] + cdn['aliases'] << dnsname if !cdn['aliases'].include?(dnsname) + end } end @@ -605,6 +626,22 @@ def get_properties params[:origins][:items] << origin } + # if we have any placeholder DNS records that are intended to be + # filled out with our runtime @mu_name, do so, and add an alias if + # applicable + if @config['dns_records'] + @config['dns_records'].each { |rec| + if !rec['name'] + rec['name'] = @mu_name.downcase + dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec) + if @config['certificate'] and MU::Cloud::AWS.nameMatchesCertificate(dnsname, @config['certificate']['id']) + @config['aliases'] ||= [] + @config['aliases'] << dnsname if !@config['aliases'].include?(dnsname) + end + end + } + end + if @config['aliases'] params[:aliases] = { items: @config['aliases'], diff --git a/modules/mu/providers/aws/dnszone.rb b/modules/mu/providers/aws/dnszone.rb index 4269d3807..8fd492bb6 100644 --- a/modules/mu/providers/aws/dnszone.rb +++ b/modules/mu/providers/aws/dnszone.rb @@ -173,11 +173,29 @@ def create return resp.hosted_zone if @config["create_zone"] end + # Resolve a record entry (as in {MU::Config::BasketofKittens::dnszones::record} to the full DNS name we would assign it + def self.recordToName(record) + shortname = record['name'] + shortname += ".#{MU.environment.downcase}" if record["append_environment_name"] + + zone = if record['zone'].has_key?("id") + MU::Cloud::DNSZone.find(cloud_id: record['zone']['id']).values.first + else + MU::Cloud::DNSZone.find(cloud_id: record['zone']['name']).values.first + end + + if zone.nil? + raise MuError.new "Failed to locate Route53 DNS Zone", details: record['zone'] + end + + shortname+"."+zone.name.sub(/\.$/, '') + end + # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all # requested records in background and returns immediately. # @param cfg [Array]: An array of parsed {MU::Config::BasketofKittens::dnszones::records} objects. # @param target [String]: Optional target for the records to be created. Overrides targets embedded in cfg records. - def self.createRecordsFromConfig(cfg, target: nil) + def self.createRecordsFromConfig(cfg, target: nil, name_only: false) return if cfg.nil? record_threads = [] @@ -190,7 +208,6 @@ def self.createRecordsFromConfig(cfg, target: nil) zone = MU::Cloud::DNSZone.find(cloud_id: record['zone']['name']).values.first end - raise MuError, "Failed to locate Route53 DNS Zone for domain #{record['zone']['name']}" if zone.nil? healthcheck_id = nil record['target'] = target if !target.nil? diff --git a/modules/mu/providers/aws/loadbalancer.rb b/modules/mu/providers/aws/loadbalancer.rb index 485c138e2..f30a6d978 100644 --- a/modules/mu/providers/aws/loadbalancer.rb +++ b/modules/mu/providers/aws/loadbalancer.rb @@ -838,7 +838,7 @@ def self.validateConfig(lb, _configurator) (!listener["ssl_certificate_id"].nil? and !listener["ssl_certificate_id"].empty?) if lb['cloud'] != "CloudFormation" # XXX or maybe do this anyway? begin - listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region']) + listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region']).first rescue MuError ok = false next From 14496ef691bb8d4d041dfbe2c3efc8cc6fb596fc Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 5 Aug 2020 03:27:23 -0400 Subject: [PATCH 060/107] AWS::Bucket: Add a method for injecting allow permissions to a bucket's policies; AWS::CDN: use that bucket method --- modules/mu/providers/aws/bucket.rb | 64 ++++++++++++++++++++++-------- modules/mu/providers/aws/cdn.rb | 45 ++++++++++++++------- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index d593c78a3..dbbf7414b 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -85,10 +85,35 @@ def tagBucket end + # @return [String] def url "https://#{@cloud_id}.s3.amazonaws.com" end + # Grant access via our bucket policy to the specified resource + # @param arn [String] + # @param permissions [Array] + # @param paths [Array] + def allowPrincipal(arn, permissions: ["GetObject"], paths: ["/*"]) + @config['policies'] ||= [] + @config['policies'] << { + "name" => arn.sub(/.*?([0-9a-z\-_]+)$/i, '\1'), + "grant_to" => [ { "identifier" => arn } ], + "permissions" => permissions.map { |p| "s3:"+p }, + "flag" => "allow", + "targets" => paths.map { |p| + { + "path" => p, + "type" => "bucket", + "identifier" => @config['name'] + } + } + } + + MU.log "policydoc", MU::NOTICE, details: MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true) + applyPolicies + end + # Called automatically by {MU::Deploy#createResources} def groom @@ -98,23 +123,7 @@ def groom tagBucket if !@config['scrub_mu_isms'] current = cloud_desc - if @config['policies'] - @config['policies'].each { |pol| - pol['grant_to'] ||= [ - { "id" => "*" } - ] - } - - policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true) - policy_docs.each { |doc| - MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first) - MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy( - bucket: @cloud_id, - policy: JSON.generate(doc.values.first) - ) - } - end - + applyPolicies if @config['policies'] if @config['versioning'] and current["versioning"].status != "Enabled" MU.log "Enabling versioning on S3 bucket #{@cloud_id}", MU::NOTICE @@ -480,6 +489,27 @@ def self.describe_bucket(bucket, minimal: false, credentials: nil, region: nil) desc end + private + + def applyPolicies + return if !@config['policies'] + + @config['policies'].each { |pol| + pol['grant_to'] ||= [ + { "id" => "*" } + ] + } + + policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true) + policy_docs.each { |doc| + MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first) + MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy( + bucket: @cloud_id, + policy: JSON.generate(doc.values.first) + ) + } + end + end end end diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index c7a5a2d78..0398f8dbc 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -80,6 +80,17 @@ def groom MU.log "Alias for CloudFront Distribution #{@config['name']}: #{a}", MU::SUMMARY } end + + # Make sure we show up in the bucket policy of our target bucket, + # if it's a sibling in this deploy + cloud_desc(use_cache: false).origins.items.each { |o| + if o.s3_origin_config + id = o.s3_origin_config.origin_access_identity.sub(/^origin-access-identity\/cloudfront\//, '') + bucketref = get_bucketref_from_domain(o.domain_name) + bucketref.kitten.allowPrincipal("arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity "+id) + end + } + end # Canonical Amazon Resource Number for this resource @@ -236,16 +247,7 @@ def toKitten(**_args) "name" => o.id } if o.s3_origin_config - buckets = MU::Cloud.resourceClass("AWS", "Bucket").find(credentials: @credentials, allregions: true, cloud_id: o.domain_name.sub(/\..*/, '')) - if buckets and buckets.size == 1 - origin["bucket"] = MU::Config::Ref.get( - id: buckets.keys.first, - type: "buckets", - region: buckets.values.first["region"], - credentials: @credentials, - cloud: "AWS" - ) - end + origin["bucket"] = get_bucketref_from_domain(domain_name) end origin["domain_name"] = o.domain_name if !origin["bucket"] if o.custom_origin_config @@ -315,8 +317,8 @@ def toKitten(**_args) } end -if @cloud_id == "E2SVQ5DTYCQ22P" -# MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc +if ["espier", "espier-dev-2020080400-zn-front"].include?(bok['name']) + MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc # MU.log @cloud_id+" bok", MU::NOTICE, details: bok end bok @@ -473,9 +475,9 @@ def self.schema(_config) # Cloud-specific pre-processing of {MU::Config::BasketofKittens::cdns}, bare and unvalidated. # @param cdn [Hash]: The resource to process and validate - # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member + # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member # @return [Boolean]: True if validation succeeded, False otherwise - def self.validateConfig(cdn, _configurator) + def self.validateConfig(cdn, configurator) ok = true cdn['origins'].each { |o| @@ -700,6 +702,21 @@ def get_properties params end + def get_bucketref_from_domain(domain_name) + buckets = MU::Cloud.resourceClass("AWS", "Bucket").find(credentials: @credentials, allregions: true, cloud_id: domain_name.sub(/\..*/, '')) + if buckets and buckets.size == 1 + return MU::Config::Ref.get( + id: buckets.keys.first, + type: "buckets", + region: buckets.values.first["region"], + credentials: @credentials, + cloud: "AWS" + ) + end + + nil + end + end end end From 5d9b233898ffb80480a077868719f0d98a1d1af9 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 6 Aug 2020 21:20:40 -0400 Subject: [PATCH 061/107] boatloads of missing YARD comments added --- modules/mu/config/bucket.rb | 2 +- modules/mu/config/cdn.rb | 20 +++++---- modules/mu/providers/aws/bucket.rb | 64 ++++++++++++++++++++++------- modules/mu/providers/aws/cdn.rb | 29 ++++++++++--- modules/mu/providers/aws/dnszone.rb | 2 +- modules/mu/providers/aws/role.rb | 14 +++++-- 6 files changed, 97 insertions(+), 34 deletions(-) diff --git a/modules/mu/config/bucket.rb b/modules/mu/config/bucket.rb index 11ce963f6..893b94b4b 100644 --- a/modules/mu/config/bucket.rb +++ b/modules/mu/config/bucket.rb @@ -52,9 +52,9 @@ def self.schema }, "upload" => { "type" => "array", - "description" => "Upload objects to a bucket, where supported", "items" => { "type" => "object", + "description" => "Upload objects to a bucket, where supported", "required" => ["source", "destination"], "properties" => { "source" => { diff --git a/modules/mu/config/cdn.rb b/modules/mu/config/cdn.rb index 8d5d0affc..61b5c6df0 100644 --- a/modules/mu/config/cdn.rb +++ b/modules/mu/config/cdn.rb @@ -45,17 +45,21 @@ def self.schema "minItems" => 1, "items" => { "type" => "object", + "description" => "One or more back-end sources which this CDN will cache", "required" => ["name"], "properties" => { "name" => { - "type" => "string" + "type" => "string", + "description" => "A unique identifying string which other components of this distribution may use to reference this origin" }, "domain_name" => { - "type" => "string" + "type" => "string", + "description" => "Domain name of the back-end web server or other resource behind this CDN" }, "path" => { "type" => "string", - "default" => "" + "default" => "", + "description" => "Optional path on back-end service to which to map front-end requests" } } } @@ -63,16 +67,16 @@ def self.schema "behaviors" => { "type" => "array", "items" => { + "description" => "Customize the behavior of requests sent to one of this CDN's configured +origins+", "type" => "object", "properties" => { "origin" => { - "type" => "string" - }, - "is_default" => { - "type" => "boolean" + "type" => "string", + "description" => "Which of our +origins+ this set of behaviors should map to, by its +name+ field." }, "path_pattern" => { - "type" => "string" + "type" => "string", + "description" => "The request path or paths for which this behavior should be invoked" } } } diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index dbbf7414b..7908c9e81 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -21,6 +21,8 @@ class Bucket < MU::Cloud::Bucket @@region_cache = {} @@region_cache_semaphore = Mutex.new + # Map some filename extensions to mime types. S3 does most of this on + # its own, add to this for cases it doesn't cover. MIME_MAP = { ".svg" => "image/svg+xml" } @@ -91,14 +93,15 @@ def url end # Grant access via our bucket policy to the specified resource - # @param arn [String] + # @param principal [String] # @param permissions [Array] # @param paths [Array] - def allowPrincipal(arn, permissions: ["GetObject"], paths: ["/*"]) + def allowPrincipal(principal, permissions: ["GetObject", "ListBucket"], paths: [""], doc_id: nil, name: nil) @config['policies'] ||= [] + name ||= principal.sub(/.*?([0-9a-z\-_]+)$/i, '\1') @config['policies'] << { - "name" => arn.sub(/.*?([0-9a-z\-_]+)$/i, '\1'), - "grant_to" => [ { "identifier" => arn } ], + "name" => name, + "grant_to" => [ { "identifier" => principal } ], "permissions" => permissions.map { |p| "s3:"+p }, "flag" => "allow", "targets" => paths.map { |p| @@ -110,8 +113,7 @@ def allowPrincipal(arn, permissions: ["GetObject"], paths: ["/*"]) } } - MU.log "policydoc", MU::NOTICE, details: MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true) - applyPolicies + applyPolicies(doc_id: doc_id) end # Called automatically by {MU::Deploy#createResources} @@ -161,7 +163,7 @@ def groom end Hash[upload_me].each_pair { |file, url| - self.class.upload(url, file: file, credentials: @credentials, region: @config['region']) + self.class.upload(url, file: file, credentials: @credentials, region: @config['region'], acl: batch['acl']) } } end @@ -363,6 +365,10 @@ def self.find(**args) found = {} args[:region] ||= MU::Cloud::AWS.myRegion(args[:credentials]) + if args[:flags] and args[:flags][:allregions] + args[:allregions] = args[:flags][:allregions] + end + minimal = args[:full] ? false : true location = Proc.new { |name| begin @@ -380,7 +386,7 @@ def self.find(**args) if args[:cloud_id] begin - found[args[:cloud_id]] = describe_bucket(args[:cloud_id], minimal: true, credentials: args[:credentials], region: args[:region]) + found[args[:cloud_id]] = describe_bucket(args[:cloud_id], minimal: minimal, credentials: args[:credentials], region: args[:region]) found[args[:cloud_id]]['region'] ||= location.call(args[:cloud_id]) found[args[:cloud_id]]['region'] ||= args[:region] found[args[:cloud_id]]['name'] ||= args[:cloud_id] @@ -396,7 +402,7 @@ def self.find(**args) next end bucket_region ||= args[:region] - found[b.name] = describe_bucket(b.name, minimal: true, credentials: args[:credentials], region: bucket_region) + found[b.name] = describe_bucket(b.name, minimal: minimal, credentials: args[:credentials], region: bucket_region) found[b.name]["region"] ||= bucket_region found[b.name]['name'] ||= b.name rescue Aws::S3::Errors::AccessDenied @@ -408,6 +414,28 @@ def self.find(**args) found end + # Reverse-map our cloud description into a runnable config hash. + # We assume that any values we have in +@config+ are placeholders, and + # calculate our own accordingly based on what's live in the cloud. + def toKitten(**_args) + bok = { + "cloud" => "AWS", + "credentials" => @config['credentials'], + "cloud_id" => @cloud_id + } + +if @cloud_id =~ /espier/i + MU.log @cloud_id, MU::WARN, details: cloud_desc +end + + if !cloud_desc + MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config + return nil + end + + nil + end + # Cloud-specific configuration properties. # @param _config [MU::Config]: The calling MU::Config object # @return [Array]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource @@ -415,10 +443,16 @@ def self.schema(_config) toplevel_required = [] schema = { "policies" => MU::Cloud.resourceClass("AWS", "Role").condition_schema, - "acl" => { - "type" => "string", - "enum" => ["private", "public-read", "public-read-write", "authenticated-read"], - "default" => "private" + "upload" => { + "items" => { + "properties" => { + "acl" => { + "type" => "string", + "enum" => ["private", "public-read", "public-read-write", "authenticated-read"], + "default" => "private" + } + } + } }, "storage_class" => { "type" => "string", @@ -491,7 +525,7 @@ def self.describe_bucket(bucket, minimal: false, credentials: nil, region: nil) private - def applyPolicies + def applyPolicies(doc_id: nil) return if !@config['policies'] @config['policies'].each { |pol| @@ -500,7 +534,7 @@ def applyPolicies ] } - policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true) + policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true, version: "2008-10-17", doc_id: doc_id) policy_docs.each { |doc| MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first) MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy( diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index 0398f8dbc..1b2b05914 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -87,7 +87,11 @@ def groom if o.s3_origin_config id = o.s3_origin_config.origin_access_identity.sub(/^origin-access-identity\/cloudfront\//, '') bucketref = get_bucketref_from_domain(o.domain_name) - bucketref.kitten.allowPrincipal("arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity "+id) + next if !bucketref or !bucketref.kitten + resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).get_cloud_front_origin_access_identity(id: id) + pp resp +# bucketref.kitten.allowPrincipal("arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity "+id, doc_id: "PolicyForCloudFrontPrivateContent", permissions: ["GetObject"]) + bucketref.kitten.allowPrincipal(resp.cloud_front_origin_access_identity.s3_canonical_user_id, doc_id: "PolicyForCloudFrontPrivateContent", permissions: ["GetObject"], name: @mu_name) end } @@ -247,7 +251,7 @@ def toKitten(**_args) "name" => o.id } if o.s3_origin_config - origin["bucket"] = get_bucketref_from_domain(domain_name) + origin["bucket"] = get_bucketref_from_domain(o.domain_name) end origin["domain_name"] = o.domain_name if !origin["bucket"] if o.custom_origin_config @@ -276,7 +280,6 @@ def toKitten(**_args) behavior = {} behavior["origin"] = b.target_origin_id - behavior["is_default"] = true if default behavior["path_pattern"] = b.path_pattern if b.respond_to?(:path_pattern) behavior["protocol_policy"] = b.viewer_protocol_policy if b.lambda_function_associations and !b.lambda_function_associations.items.empty? @@ -317,8 +320,8 @@ def toKitten(**_args) } end -if ["espier", "espier-dev-2020080400-zn-front"].include?(bok['name']) - MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc +if bok["name"] =~ /espier/i + MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc.origins # MU.log @cloud_id+" bok", MU::NOTICE, details: bok end bok @@ -334,6 +337,7 @@ def self.schema(_config) schema = { "disabled" => { "type" => "boolean", + "description" => "Flag this CloudFront distribution as disabled", "default" => false }, "certificate" => MU::Config::Ref.schema(type: "certificate", desc: "Required if any domains have been specified with +aliases+; parser will attempt to autodetect a valid ACM or IAM certificate if not specified.", omit_fields: ["cloud", "tag", "deploy_id"]), @@ -342,14 +346,17 @@ def self.schema(_config) "properties" => { "min_ttl" => { "type" => "integer", + "description" => "The minimum amount of time that you want objects to stay in CloudFront caches before CloudFront forwards another request to your origin to determine whether the object has been updated.", "default" => 0 }, "default_ttl" => { "type" => "integer", + "description" => "The default amount of time that you want objects to stay in CloudFront caches before CloudFront forwards another request to your origin to determine whether the object has been updated.", "default" => 86400 }, "max_ttl" => { "type" => "integer", + "description" => "The maximum amount of time that you want objects to stay in CloudFront caches before CloudFront forwards another request to your origin to determine whether the object has been updated.", "default" => 31536000 }, "protocol_policy" => { @@ -363,6 +370,7 @@ def self.schema(_config) }, "forwarded_values" => { "type" => "object", + "description" => "HTTP request artifacts to include in requests passed to our back-end +origin+", "default" => { "query_string" => false, "cookies" => { @@ -372,18 +380,22 @@ def self.schema(_config) "properties" => { "query_string" => { "type" => "boolean", + "description" => "Indicates whether you want CloudFront to forward query strings to the origin that is associated with this cache behavior and cache based on the query string parameters.", "default" => false }, "cookies" => { "type" => "object", + "description" => "A complex type that specifies whether you want CloudFront to forward cookies to the origin and, if so, which ones.", "properties" => { "forward" => { "type" => "string", + "description" => "Specifies which cookies to forward to the origin for this cache behavior: all, none, or the list of cookies specified in +whitelisted_names+", "enum" => %w{none whitelist all} }, "whitelisted_names" => { "type" => "array", "items" => { + "description" => "Required if you specify whitelist for the value of +forward+", "type" => "string" } }, @@ -392,12 +404,14 @@ def self.schema(_config) "headers" => { "type" => "array", "items" => { + "description" => "Specifies the headers, if any, that you want CloudFront to forward to the origin for this cache behavior (whitelisted headers).", "type" => "string" } }, "query_string_cache_keys" => { "type" => "array", "items" => { + "description" => "Indicates whether you want CloudFront to forward query strings to the origin that is associated with this cache behavior and cache based on the query string parameters", "type" => "string" } } @@ -443,6 +457,7 @@ def self.schema(_config) "custom_headers" => { "type" => "array", "items" => { + "description" => "A list of HTTP header names and values that CloudFront adds to requests it sends to the origin.", "type" => "object", "required" => ["key", "value"], "properties" => { @@ -687,7 +702,7 @@ def get_properties end } - if b['is_default'] or @config['behaviors'].size == 1 or b['path_pattern'] == "*" + if @config['behaviors'].size == 1 or b['path_pattern'] == "*" params[:default_cache_behavior] = behavior else behavior[:path_pattern] = b['path_pattern'] @@ -712,6 +727,8 @@ def get_bucketref_from_domain(domain_name) credentials: @credentials, cloud: "AWS" ) + else + MU.log "Failed to locate or isolate a bucket object from #{domain_name}", MU::WARN, details: buckets.keys end nil diff --git a/modules/mu/providers/aws/dnszone.rb b/modules/mu/providers/aws/dnszone.rb index 8fd492bb6..a1ad41109 100644 --- a/modules/mu/providers/aws/dnszone.rb +++ b/modules/mu/providers/aws/dnszone.rb @@ -173,7 +173,7 @@ def create return resp.hosted_zone if @config["create_zone"] end - # Resolve a record entry (as in {MU::Config::BasketofKittens::dnszones::record} to the full DNS name we would assign it + # Resolve a record entry (as in {MU::Config::BasketofKittens::dnszones::records} to the full DNS name we would assign it def self.recordToName(record) shortname = record['name'] shortname += ".#{MU.environment.downcase}" if record["append_environment_name"] diff --git a/modules/mu/providers/aws/role.rb b/modules/mu/providers/aws/role.rb index 9e86c6faf..ea64961ff 100644 --- a/modules/mu/providers/aws/role.rb +++ b/modules/mu/providers/aws/role.rb @@ -1109,13 +1109,14 @@ def self.validateConfig(role, _configurator) # @param policies [Array]: One or more policy chunks # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources # @return [Array] - def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false) + def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil) if policies name = nil doc = { - "Version" => "2012-10-17", + "Version" => version, "Statement" => [] } + doc["Id"] = doc_id if doc_id policies.each { |policy| policy["flag"] ||= "Allow" statement = { @@ -1156,7 +1157,14 @@ def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false) raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy" end else - bucket_prefix = grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/) ? "Service" : "AWS" + bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/) + "Service" + elsif grantee["identifier"] =~ /^[a-f0-9]+$/ + "CanonicalUser" + else + "AWS" + end + if bucket_style statement["Principal"] << { bucket_prefix => grantee["identifier"] } else From 0931b8047a45ca6bcf5a4854e144b1456e34dd35 Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 7 Aug 2020 01:12:50 -0400 Subject: [PATCH 062/107] more tedious YARDing --- modules/mu/providers/aws.rb | 5 +++++ modules/mu/providers/aws/job.rb | 22 ++++++++++++++++++++-- modules/mu/providers/azure/server.rb | 1 + 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index 86d4f1be4..c284b6126 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -931,6 +931,11 @@ def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: n [id, domains.uniq] end + # Given a domain name and an ACM or IAM certificate identifier, sort out + # whether the domain name is "covered" by the certificate + # @param name [String] + # @param cert_id [String] + # @return [Boolean] def self.nameMatchesCertificate(name, cert_id) _id, domains = findSSLCertificate(id: cert_id) return false if !domains diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index 83009dca7..fc6e97f99 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -278,20 +278,24 @@ def self.schema(_config) }, "run_command_parameters" => { "type" => "object", + "description" => "Parameters used when you are using the rule to invoke Amazon EC2 Run Command", "required" => ["run_command_targets"], "properties" => { "run_command_targets" => { "type" => "array", "items" => { "type" => "object", + "description" => "Currently, AWS supports including only one +run_command_targets+ block, which specifies either an array of InstanceIds or a tag.", "required" => ["key", "values"], "properties" => { "key" => { - "type" => "string" + "type" => "string", + "description" => "Can be either +tag: tag-key+ or +InstanceIds+" }, "values" => { "type" => "array", "items" => { + "description" => "If +key+ is +tag: tag-key+, +values+ is a list of tag values; if +key+ is +InstanceIds+, +values+ is a list of Amazon EC2 instance IDs.", "type" => "string" } } @@ -302,38 +306,48 @@ def self.schema(_config) }, "input_transformer" => { "type" => "object", + "description" => "Settings to enable you to provide custom input to a target based on certain event data. You can extract one or more key-value pairs from the event and then use that data to send customized input to the target.", "required" => ["input_template"], "properties" => { "input_template" => { - "type" => "string" + "type" => "string", + "description" => "Input template where you specify placeholders that will be filled with the values of the keys from +input_paths_map+ to customize the data sent to the target." }, "input_paths_map" => { "type" => "object", + "description" => "Hash representing JSON paths to be extracted from the event" } } }, "batch_parameters" => { "type" => "object", + "description" => "If the event target is an AWS Batch job, this contains the job definition, job name, and other parameters. See: https://docs.aws.amazon.com/batch/latest/userguide/jobs.html", "required" => ["job_definition", "job_name"], "properties" => { "job_definition" => { + "description" => "The ARN or name of the job definition to use if the event target is an AWS Batch job.", "type" => "string" }, "job_name" => { + "description" => "The name to use for this execution of the job, if the target is an AWS Batch job.", "type" => "string" }, "array_properties" => { "type" => "object", + "description" => "The array properties for the submitted job, such as the size of the array.", "properties" => { "size" => { + "description" => "Size of the submitted array", "type" => "integer" } } }, "retry_strategy" => { "type" => "object", + "description" => "The retry strategy to use for failed jobs, if the target is an AWS Batch job.", "properties" => { "attempts" => { + "description" => "Number of retry attempts, valid values from 1-10", "type" => "integer" } } @@ -342,6 +356,7 @@ def self.schema(_config) }, "sqs_parameters" => { "type" => "object", + "description" => "Contains the message group ID to use when the target is an SQS FIFO queue.", "required" => ["message_group_id"], "properties" => { "message_group_id" => { @@ -351,6 +366,7 @@ def self.schema(_config) }, "kinesis_parameters" => { "type" => "object", + "description" => "The custom parameter you can use to control the shard assignment, when the target is a Kinesis data stream.", "required" => ["partition_key_path"], "properties" => { "partition_key_path" => { @@ -360,10 +376,12 @@ def self.schema(_config) }, "http_parameters" => { "type" => "object", + "description" => "Contains the HTTP parameters to use when the target is a API Gateway REST endpoint.", "properties" => { "path_parameter_values" => { "type" => "array", "items" => { + "description" => "The path parameter values to be used to populate API Gateway REST API path wildcards (\"*\").", "type" => "string" } }, diff --git a/modules/mu/providers/azure/server.rb b/modules/mu/providers/azure/server.rb index 9e577b67a..1faa7a242 100644 --- a/modules/mu/providers/azure/server.rb +++ b/modules/mu/providers/azure/server.rb @@ -636,6 +636,7 @@ def self.validateConfig(server, configurator) ok end + # stub def self.diskConfig(config, create = true, disk_as_url = true, credentials: nil) end From d91b003accb407cf7dc6ad7d517f7c65e3588abe Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 7 Aug 2020 02:55:31 -0400 Subject: [PATCH 063/107] add a test to cover a bit of AWS CloudWatch Events and Lambda --- modules/tests/aws-jobs-functions.yaml | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 modules/tests/aws-jobs-functions.yaml diff --git a/modules/tests/aws-jobs-functions.yaml b/modules/tests/aws-jobs-functions.yaml new file mode 100644 index 000000000..4c805bf52 --- /dev/null +++ b/modules/tests/aws-jobs-functions.yaml @@ -0,0 +1,46 @@ +# clouds: AWS +--- +appname: smoketest +jobs: +- name: event1 + schedule: + minute: '0' + hour: '1' + day_of_month: '1' + month: "*" + day_of_week: "?" + year: "*" + targets: + - type: functions + name: python-function +- name: event2 + disabled: true + schedule: + minute: '0' + hour: '2' + day_of_month: '1' + month: "*" + day_of_week: "?" + year: "*" + targets: + - type: functions + name: node-function + +functions: +- name: python-function + handler: lambda_function.lambda_handler + memory: 128 + runtime: python3.6 + timeout: 300 + code: + path: functions/python-function + environment_variable: + - key: foo + value: bar +- name: node-function + runtime: nodejs12.x + handler: lambda_function.lambda_handler + memory: 256 + timeout: 60 + code: + path: functions/node-function From b3fa4f759a2d47aebe90c6139cdddcc831185023 Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 7 Aug 2020 03:33:41 -0400 Subject: [PATCH 064/107] don't forget the lambda code that goes with our new test --- .../tests/functions/node-function/lambda_function.js | 10 ++++++++++ .../functions/python-function/lambda_function.py | 12 ++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 modules/tests/functions/node-function/lambda_function.js create mode 100644 modules/tests/functions/python-function/lambda_function.py diff --git a/modules/tests/functions/node-function/lambda_function.js b/modules/tests/functions/node-function/lambda_function.js new file mode 100644 index 000000000..dbc5ded66 --- /dev/null +++ b/modules/tests/functions/node-function/lambda_function.js @@ -0,0 +1,10 @@ +console.log('Loading function'); + +exports.handler = async (event, context) => { + //console.log('Received event:', JSON.stringify(event, null, 2)); + console.log('value1 =', event.key1); + console.log('value2 =', event.key2); + console.log('value3 =', event.key3); + return event.key1; // Echo back the first key value + // throw new Error('Something went wrong'); +}; diff --git a/modules/tests/functions/python-function/lambda_function.py b/modules/tests/functions/python-function/lambda_function.py new file mode 100644 index 000000000..7a90653e2 --- /dev/null +++ b/modules/tests/functions/python-function/lambda_function.py @@ -0,0 +1,12 @@ +import json + +print('Loading function') + + +def lambda_handler(event, context): + #print("Received event: " + json.dumps(event, indent=2)) + print("value1 = " + event['key1']) + print("value2 = " + event['key2']) + print("value3 = " + event['key3']) + return event['key1'] # Echo back the first key value + #raise Exception('Something went wrong') From 9a4857ef3ba406402d115fa3d1547da4b13e2ba4 Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 7 Aug 2020 23:39:33 -0400 Subject: [PATCH 065/107] AWS::CDN: support API Gateway and Load Balancer origins sanely --- modules/mu/providers/aws/cdn.rb | 89 +++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index 1b2b05914..d8b754fce 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -39,16 +39,19 @@ def create params = get_properties begin +MU.log @config['name'], MU::NOTICE, details: params MU.log "Creating CloudFront distribution #{@mu_name}", details: params - resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_distribution_with_tags( - distribution_config_with_tags: { - distribution_config: params, - tags: { - items: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } } + MU.retrier([Aws::CloudFront::Errors::InvalidOrigin], wait: 10, max: 6) { + resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_distribution_with_tags( + distribution_config_with_tags: { + distribution_config: params, + tags: { + items: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } } + } } - } - ) - @cloud_id = resp.distribution.id + ) + @cloud_id = resp.distribution.id + } ready? rescue ::Aws::CloudFront::Errors::InvalidViewerCertificate => e cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate( @@ -156,7 +159,7 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, cred found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip } - +# XXX launch threads for this so we can keep deleting if found_muid and (ignoremaster or found_master) current = MU::Cloud::AWS.cloudfront(credentials: credentials).get_distribution_config(id: ids[arn].id) etag = current.etag @@ -424,6 +427,7 @@ def self.schema(_config) "items" => { "properties" => { "bucket" => MU::Config::Ref.schema(type: "buckets", desc: "Reference an S3 bucket for use as an origin"), + "endpoint" => MU::Config::Ref.schema(type: "endpoints", desc: "Reference an API Gateway for use as an origin"), "loadbalancer" => MU::Config::Ref.schema(type: "loadbalancers", desc: "Reference a Load Balancer for use as an origin"), "connection_attempts" => { "type" => "integer", @@ -496,12 +500,20 @@ def self.validateConfig(cdn, configurator) ok = true cdn['origins'].each { |o| - if o['bucket'] - target_ref = MU::Config::Ref.get(o['bucket']) - if target_ref.name - MU::Config.addDependency(cdn, target_ref.name, "buckets", phase: "groom") + count = 0 + ['bucket', 'endpoint', 'loadbalancer'].each { |sib_type| + if o[sib_type] + if count > 0 + ok = false + MU.log "Origin in CloudFront distro #{cdn['name']} may specify at most one of bucket, endpoint, or loadbalancer.", MU::ERR + end + target_ref = MU::Config::Ref.get(o[sib_type]) + if target_ref.name + MU::Config.addDependency(cdn, target_ref.name, sib_type, phase: "groom") + end + count += 1 end - end + } } cert_domains = nil @@ -609,16 +621,33 @@ def get_properties origin = { id: o['name'], } - if o['bucket'] - bucket_obj = MU::Config::Ref.get(o['bucket']).kitten(@deploy, cloud: "AWS") - if !bucket_obj - raise MuError.new "Failed to resolve bucket referenced in CloudFront distribution #{@config['name']}", details: o['bucket'].to_h + sib_obj = nil + ['bucket', 'endpoint', 'loadbalancer'].each { |sib_type| + if o[sib_type] + sib_obj = MU::Config::Ref.get(o[sib_type]).kitten(@deploy, cloud: "AWS") + if !sib_obj + raise MuError.new "Failed to resolve #{sib_type} referenced in CloudFront distribution #{@config['name']}", details: o[sib_type].to_h + end + break end - origin[:domain_name] = bucket_obj.cloud_desc["name"]+".s3.amazonaws.com" + } + if o['bucket'] + origin[:domain_name] = sib_obj.cloud_desc["name"]+".s3.amazonaws.com" origin[:origin_path] = o['path'] if o['path'] origin[:s3_origin_config] = { origin_access_identity: @origin_access_identity } + elsif o['endpoint'] + origin[:domain_name] = sib_obj.cloud_id+".execute-api."+sib_obj.config['region']+".amazonaws.com" + origin[:custom_origin_config] = { + origin_protocol_policy: "https-only" + } + if sib_obj.config['deploy_to'] + origin[:origin_path] ||= "/"+sib_obj.config['deploy_to'] + end + elsif o['loadbalancer'] + origin[:domain_name] = sib_obj.cloud_desc.dns_name + origin[:origin_path] = o['path'] if o['path'] else # XXX make sure parser guarantees these are present origin[:domain_name] = o['domain_name'] origin[:origin_path] = o['path'] @@ -637,8 +666,28 @@ def get_properties end [:connection_attempts, :connection_timeout].each { |field| - origin[field] = o[field.to_s] + origin[field] ||= o[field.to_s] } + if !origin[:s3_origin_config] + maplet = { + 'protocol_policy' => :origin_protocol_policy, + 'ssl_protocols' => :origin_ssl_protocols, + 'http_port' => :http_port, + 'https_port' => :https_port + } + maplet.each_pair { |field, paramfield| + next if !o[field] + origin[:custom_origin_config] ||= {} + origin[:custom_origin_config][paramfield] ||= if o[field.to_s].is_a?(Array) + { + quantity: o[field].size, + items: o[field] + } + else + o[field] + end + } + end params[:origins][:items] << origin } From f3d8b2ded165f26c3d5affaf29fe2e8117fba4fa Mon Sep 17 00:00:00 2001 From: John Stange Date: Sat, 8 Aug 2020 01:45:07 -0400 Subject: [PATCH 066/107] AWS::Function: derp less about API Gateway triggers --- modules/mu/providers/aws/endpoint.rb | 2 +- modules/mu/providers/aws/function.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 09a12f379..daa62110e 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -169,7 +169,7 @@ def generate_methods(integrations = true) resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).put_integration(params) - if m['integrate_with']['type'] == "function" + if m['integrate_with']['type'] =~ /^functions?$/ function_obj.addTrigger(method_arn, "apigateway", @config['name']) end diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index cb62c0a94..3f5532455 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -116,10 +116,11 @@ def groom statement_id: "#{@mu_name}-ID-1", } -# MU.log "trigger properties", MU::NOTICE, details: trigger_properties + MU.log "trigger properties", MU::NOTICE, details: trigger_properties begin MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties) rescue Aws::Lambda::Errors::ResourceConflictException + # just means the permission is already there end adjust_trigger(tr['service'], trigger_arn, arn, @mu_name) } @@ -154,7 +155,7 @@ def addTrigger(calling_arn, calling_service, calling_name) source_arn: calling_arn, statement_id: "#{calling_service}-#{calling_name}", } - +MU.log "addTrigger called from #{caller[0]}", MU::WARN, details: trigger begin # XXX There doesn't seem to be an API call to list or view existing # permissions, wtaf. This means we can't intelligently guard this. @@ -233,9 +234,8 @@ def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda' } ] }) -# when 'apigateway' -# XXX this is actually happening in ::Endpoint... maybe... -# MU.log "Creation of API Gateway integrations not yet implemented, you'll have to do this manually", MU::WARN, details: "(because we'll basically have to implement all of APIG for this)" + when 'apigateway' + addTrigger(trig_arn, "lambda", trig_arn.sub(/.*?([a-z0-9\-_]+)$/i, '\1')) end end From a99a528ead27db33808751b4a87bbe7fe5f15972 Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 9 Aug 2020 03:07:23 -0400 Subject: [PATCH 067/107] AWS::Function/Endpoint: don't get triggers front for API Gateway methods with no appended paths; CloudFront: delete more better --- modules/mu/providers/aws/cdn.rb | 45 +++++++++++++++++----------- modules/mu/providers/aws/endpoint.rb | 7 +++-- modules/mu/providers/aws/function.rb | 3 +- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index d8b754fce..fd7551489 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -145,8 +145,10 @@ def self.quality # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server # @return [void] def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {}) + resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_distributions if resp and resp.distribution_list and resp.distribution_list.items + delete_threads = [] ids = Hash[resp.distribution_list.items.map { |distro| [distro.arn, distro] }] ids.each_key { |arn| @@ -159,30 +161,33 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, cred found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip } -# XXX launch threads for this so we can keep deleting + if found_muid and (ignoremaster or found_master) - current = MU::Cloud::AWS.cloudfront(credentials: credentials).get_distribution_config(id: ids[arn].id) - etag = current.etag + delete_threads << Thread.new(arn, name) { |my_arn, my_name| + current = MU::Cloud::AWS.cloudfront(credentials: credentials).get_distribution_config(id: ids[my_arn].id) + etag = current.etag - if !noop + if !noop - if current.distribution_config.enabled - newcfg = MU.structToHash(current.distribution_config) - newcfg[:enabled] = false - MU.log "Disabling CloudFront distribution #{name ? name : ids[arn].id})", MU::NOTICE - updated = MU::Cloud::AWS.cloudfront(credentials: credentials).update_distribution(id: ids[arn].id, distribution_config: newcfg, if_match: etag) - etag = updated.etag - end + if current.distribution_config.enabled + newcfg = MU.structToHash(current.distribution_config) + newcfg[:enabled] = false + MU.log "Disabling CloudFront distribution #{my_name ? my_name : ids[my_arn].id})", MU::NOTICE + updated = MU::Cloud::AWS.cloudfront(credentials: credentials).update_distribution(id: ids[my_arn].id, distribution_config: newcfg, if_match: etag) + etag = updated.etag + end - end + end - MU.log "Deleting CloudFront distribution #{name ? name : ids[arn].id})" - if !noop - ready?(ids[arn].id, credentials: credentials) - MU::Cloud::AWS.cloudfront(credentials: credentials).delete_distribution(id: ids[arn].id, if_match: etag) - end + MU.log "Deleting CloudFront distribution #{my_name ? my_name : ids[my_arn].id})" + if !noop + ready?(ids[my_arn].id, credentials: credentials) + MU::Cloud::AWS.cloudfront(credentials: credentials).delete_distribution(id: ids[my_arn].id, if_match: etag) + end + } end } + delete_threads.each { |t| t.join } end resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_cloud_front_origin_access_identities @@ -194,7 +199,11 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, cred MU.log "Deleting CloudFront origin access identity #{ident.id} (#{ident.comment})" if !noop getresp = MU::Cloud::AWS.cloudfront(credentials: credentials).get_cloud_front_origin_access_identity(id: ident.id) - MU::Cloud::AWS.cloudfront(credentials: credentials).delete_cloud_front_origin_access_identity(id: ident.id, if_match: getresp.etag) + begin + MU::Cloud::AWS.cloudfront(credentials: credentials).delete_cloud_front_origin_access_identity(id: ident.id, if_match: getresp.etag) + rescue ::Aws::CloudFront::Errors::CloudFrontOriginAccessIdentityInUse => e + MU.log "Got #{e.message} deleting #{ident.id}; it likely belongs to a distribution we can't to delete", MU::WARN, details: ident + end end end } diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index daa62110e..ef4d255ca 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -37,6 +37,7 @@ def generate_methods(integrations = true) method_arn = "arn:#{MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws"}:execute-api:#{@config["region"]}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{@cloud_id}/*/#{m['type']}/#{m['path']}" path_part = ["", "/"].include?(m['path']) ? nil : m['path'] + method_arn.sub!(/\/\/$/, '/') resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( rest_api_id: @cloud_id @@ -179,7 +180,7 @@ def generate_methods(integrations = true) :resource_id => parent_id, :http_method => m['type'], :status_code => r['code'].to_s, - :selection_pattern => "" + :selection_pattern => ".*" } if r['headers'] params[:response_parameters] = r['headers'].map { |h| @@ -238,7 +239,8 @@ def cloud_desc(use_cache: true) def notify return nil if !@cloud_id or !cloud_desc(use_cache: false) deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true) - deploy_struct['url'] = "https://"+@cloud_id+".execute-api."+@config['region']+".amazonaws.com/"+@config['deploy_to'] + deploy_struct['url'] = "https://"+@cloud_id+".execute-api."+@config['region']+".amazonaws.com" + deploy_struct['url'] += "/"+@config['deploy_to'] if @config['deploy_to'] # XXX stages and whatnot return deploy_struct end @@ -322,6 +324,7 @@ def toKitten(**_args) resource_id: r.id, http_method: http_type ) +MU.log bok['name']+" "+http_type, MU::WARN, details: m_desc if bok['name'].match(/ESPIER/) method['type'] = http_type method['path'] = r.path_part || r.path if m_desc.method_responses diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 3f5532455..098ecee7b 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -155,7 +155,7 @@ def addTrigger(calling_arn, calling_service, calling_name) source_arn: calling_arn, statement_id: "#{calling_service}-#{calling_name}", } -MU.log "addTrigger called from #{caller[0]}", MU::WARN, details: trigger + begin # XXX There doesn't seem to be an API call to list or view existing # permissions, wtaf. This means we can't intelligently guard this. @@ -411,6 +411,7 @@ def toKitten(**_args) begin pol = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).get_policy(function_name: @cloud_id).policy +MU.log @cloud_id, MU::WARN, details: JSON.parse(pol) if @cloud_id == "ESPIER-DEV-2020080900-LN-ON-DEMAND-SCANNER" if pol bok['triggers'] ||= [] JSON.parse(pol)["Statement"].each { |s| From eae38a798cd22ccdfb7708938081d62cb20ee74b Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 9 Aug 2020 13:21:25 -0400 Subject: [PATCH 068/107] AWS::Endpoint: expose origin timeout parameter; make sure dns_records actually does something --- modules/mu/providers/aws/endpoint.rb | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index ef4d255ca..e6a20d8aa 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -147,7 +147,8 @@ def generate_methods(integrations = true) :resource_id => parent_id, :type => type, # XXX Lambda and Firehose can do AWS_PROXY :content_handling => "CONVERT_TO_TEXT", # XXX expose in BoK - :http_method => m['type'] + :http_method => m['type'], + :timeout_in_millis => m['timeout_in_millis'] # credentials: role_arn } params[:uri] = uri if uri @@ -211,9 +212,25 @@ def groom # this automatically creates a stage with the same name, so we don't # have to deal with that - my_url = "https://"+@cloud_id+".execute-api."+@config['region']+".amazonaws.com/"+@config['deploy_to'] + my_hostname = @cloud_id+".execute-api."+@config['region']+".amazonaws.com" + my_url = "https://"+my_hostname+"/"+@config['deploy_to'] MU.log "API Endpoint #{@config['name']}: "+my_url, MU::SUMMARY + # if we have any placeholder DNS records that are intended to be + # filled out with our runtime @mu_name, do so, and add an alias if + # applicable + if @config['dns_records'] and !MU::Cloud::AWS.isGovCloud? + @config['dns_records'].each { |rec| + if !rec['name'] + rec['name'] = @mu_name.downcase + end + dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec) + MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname+"/"+@config['deploy_to'], MU::SUMMARY + } + MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: my_hostname) + end + + # resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).create_authorizer( # rest_api_id: @cloud_id, # ) @@ -439,6 +456,11 @@ def self.schema(_config) "description" => "The HTTP method to use when contacting our integrated backend. If not specified, this will be set to match our front end.", "enum" => ["GET", "POST", "PUT", "HEAD", "DELETE", "CONNECT", "OPTIONS", "TRACE"], }, + "timeout_in_millis" => { + "type" => "integer", + "description" => "Custom timeout between +50+ and +29,000+ milliseconds.", + "default" => 29000 + }, "url" => { "type" => "string", "description" => "For HTTP or HTTP_PROXY integrations, this should be a fully-qualified URL" From 80327550bcc8876cc22ca2a2b8c4a924eaf1196f Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 9 Aug 2020 23:47:14 -0400 Subject: [PATCH 069/107] AWS::Endpoint: support for their domain name artifact with its various SSL widgets --- modules/mu/providers/aws.rb | 31 +++++ modules/mu/providers/aws/cdn.rb | 13 +- modules/mu/providers/aws/endpoint.rb | 189 +++++++++++++++++++++++---- modules/mu/providers/aws/function.rb | 4 +- 4 files changed, 198 insertions(+), 39 deletions(-) diff --git a/modules/mu/providers/aws.rb b/modules/mu/providers/aws.rb index c284b6126..3efb33eca 100644 --- a/modules/mu/providers/aws.rb +++ b/modules/mu/providers/aws.rb @@ -948,6 +948,37 @@ def self.nameMatchesCertificate(name, cert_id) false end + # Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate, + # look up and validate the specified certificate. This is intended to be + # invoked from resource implementations' +validateConfig+ methods. + # @param certblock [Hash,MU::Config::Ref]: + # @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any + # @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any + # @return [Boolean] + def self.resolveSSLCertificate(certblock, region: nil, credentials: nil) + return false if !certblock + ok = true + + certblock['region'] ||= region if !certblock['id'] + certblock['credentials'] ||= credentials + cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate( + name: certblock["name"], + id: certblock["id"], + region: certblock['region'], + credentials: certblock['credentials'] + ) + + if cert_arn + certblock['id'] ||= cert_arn + end + + ['region', 'credentials'].each { |field| + certblock.delete(field) if certblock[field].nil? + } + + [cert_arn, cert_domains] + end + # Amazon Certificate Manager API def self.acm(region: MU.curRegion, credentials: nil) region ||= myRegion diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index fd7551489..aa34a19e3 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -39,7 +39,6 @@ def create params = get_properties begin -MU.log @config['name'], MU::NOTICE, details: params MU.log "Creating CloudFront distribution #{@mu_name}", details: params MU.retrier([Aws::CloudFront::Errors::InvalidOrigin], wait: 10, max: 6) { resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_distribution_with_tags( @@ -528,21 +527,11 @@ def self.validateConfig(cdn, configurator) cert_domains = nil if cdn['certificate'] - cdn['certificate']['region'] ||= cdn['region'] if !cdn['certificate']['id'] - cdn['certificate']['credentials'] ||= cdn['credentials'] - cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate( - name: cdn['certificate']["name"], - id: cdn['certificate']["id"], - region: cdn['certificate']['region'], - credentials: cdn['certificate']['credentials'] - ) + cert_arn, cert_domains = MU::Cloud::AWS.resolveSSLCertificate(cdn['certificate'], region: cdn['region'], credentials: cdn['credentials']) if !cert_arn MU.log "Failed to find an ACM or IAM certificate specified in CloudFront distribution #{cdn['name']}", MU::ERR, details: cdn['certificate'].to_h ok = false - else - cdn['certificate']['id'] ||= cert_arn end - cdn['certificate'].delete('region') if cdn['certificate']['region'].nil? end if cdn['aliases'] diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index e6a20d8aa..58db6d63e 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -13,7 +13,7 @@ def initialize(**args) # Called automatically by {MU::Deploy#createResources} def create - resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).create_rest_api( + resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_rest_api( name: @mu_name, description: @deploy.deploy_id, endpoint_configuration: { @@ -26,7 +26,7 @@ def create # Create/update all of the methods declared for this endpoint def generate_methods(integrations = true) - resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( + resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_resources( rest_api_id: @cloud_id, ) root_resource = resp.items.first.id @@ -35,11 +35,11 @@ def generate_methods(integrations = true) @config['methods'].each { |m| m["auth"] ||= m["iam_role"] ? "AWS_IAM" : "NONE" - method_arn = "arn:#{MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws"}:execute-api:#{@config["region"]}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{@cloud_id}/*/#{m['type']}/#{m['path']}" + method_arn = "arn:#{MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws"}:execute-api:#{@config["region"]}:#{MU::Cloud::AWS.credToAcct(@credentials)}:#{@cloud_id}/*/#{m['type']}/#{m['path']}" path_part = ["", "/"].include?(m['path']) ? nil : m['path'] method_arn.sub!(/\/\/$/, '/') - resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( + resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_resources( rest_api_id: @cloud_id ) ext_resource = nil @@ -50,11 +50,11 @@ def generate_methods(integrations = true) } resp = if ext_resource -MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resource( +MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_resource( rest_api_id: @cloud_id, resource_id: ext_resource, ) -# MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).update_resource( +# MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).update_resource( # rest_api_id: @cloud_id, # resource_id: ext_resource, # patch_operations: [ @@ -66,7 +66,7 @@ def generate_methods(integrations = true) # ] # ) else - MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).create_resource( + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_resource( rest_api_id: @cloud_id, parent_id: root_resource, path_part: path_part @@ -75,13 +75,13 @@ def generate_methods(integrations = true) parent_id = resp.id resp = begin - MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_method( + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_method( rest_api_id: @cloud_id, resource_id: parent_id, http_method: m['type'] ) rescue Aws::APIGateway::Errors::NotFoundException - resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).put_method( + resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).put_method( rest_api_id: @cloud_id, resource_id: parent_id, authorization_type: m['auth'], @@ -110,7 +110,7 @@ def generate_methods(integrations = true) params[:response_models] = r['body'].map { |b| [b['content_type'], b['is_error'] ? "Error" : "Empty"] }.to_h end - MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).put_method_response(params) + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).put_method_response(params) } rescue Aws::APIGateway::Errors::ConflictException # fine to ignore @@ -169,7 +169,7 @@ def generate_methods(integrations = true) } end - resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).put_integration(params) + resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).put_integration(params) if m['integrate_with']['type'] =~ /^functions?$/ function_obj.addTrigger(method_arn, "apigateway", @config['name']) @@ -189,7 +189,7 @@ def generate_methods(integrations = true) }.to_h end - MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).put_integration_response(params) + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).put_integration_response(params) } @@ -203,7 +203,7 @@ def groom generate_methods MU.log "Deploying API Gateway #{@config['name']} to #{@config['deploy_to']}" - MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).create_deployment( + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_deployment( rest_api_id: @cloud_id, stage_name: @config['deploy_to'] # cache_cluster_enabled: false, @@ -216,26 +216,86 @@ def groom my_url = "https://"+my_hostname+"/"+@config['deploy_to'] MU.log "API Endpoint #{@config['name']}: "+my_url, MU::SUMMARY + print_dns_alias = Proc.new { |rec| + rec['name'] ||= @mu_name.downcase + dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec) + MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname+"/"+@config['deploy_to'], MU::SUMMARY + dnsname + } + # if we have any placeholder DNS records that are intended to be # filled out with our runtime @mu_name, do so, and add an alias if # applicable if @config['dns_records'] and !MU::Cloud::AWS.isGovCloud? @config['dns_records'].each { |rec| - if !rec['name'] - rec['name'] = @mu_name.downcase - end - dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec) - MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname+"/"+@config['deploy_to'], MU::SUMMARY + print_dns_alias.call(rec) } MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: my_hostname) end + if @config['domain_names'] + @config['domain_names'].each { |dom| + dnsname = if dom['dns_record'] + print_dns_alias.call(dom['dns_record']) + else + dom['unmanaged_name'] + end + certfield, dnsfield = if dom['endpoint_type'] == "EDGE" + [:certificate_arn, :distribution_domain_name] + else + [:regional_certificate_arn, :regional_domain_name] + end + + dom_desc = begin + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_domain_name(domain_name: dnsname) + rescue ::Aws::APIGateway::Errors::NotFoundException + + params = { + domain_name: dnsname, + endpoint_configuration: { + types: [dom['endpoint_type']] + }, + security_policy: dom['security_policy'], + tags: @tags + } + if dom['certificate'] + params[certfield] = dom['certificate']['id'] + end + + MU.log "Creating API Gateway Domain Name #{dnsname}", MU::NOTICE, details: params + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_domain_name(params) + end + + mappings = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_base_path_mappings(domain_name: dnsname, limit: 500).items + found = false + mappings.each { |m| +MU.log @cloud_id, MU::WARN, details: m + if m.rest_api_id == @cloud_id and stage == @config['deploy_to'] + found = true + break + end + } + if !found + MU.log "Mapping #{dnsname} to API Gateway #{@mu_name}" + MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_base_path_mapping( + domain_name: dnsname, + rest_api_id: @cloud_id, + stage: @config['deploy_to'] + ) + end + + if dom['dns_record'] + MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig([dom['dns_record']], target: dom_desc.send(dnsfield)) + end + } + end + -# resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).create_authorizer( +# resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_authorizer( # rest_api_id: @cloud_id, # ) -# resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).create_vpc_link( +# resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_vpc_link( # ) end @@ -245,7 +305,7 @@ def groom def cloud_desc(use_cache: true) return @cloud_desc_cache if @cloud_desc_cache and use_cache return nil if !@cloud_id - @cloud_desc_cache = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_rest_api( + @cloud_desc_cache = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_rest_api( rest_api_id: @cloud_id ) @cloud_desc_cache @@ -271,6 +331,27 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, regi MU.log "AWS::Endpoint.cleanup: need to support flags['known']", MU::DEBUG, details: flags MU.log "Placeholder: AWS Endpoint artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster + resp = MU::Cloud::AWS.apig(region: region, credentials: credentials).get_domain_names(limit: 500) + if resp and resp.items + resp.items.each { |d| + next if !d.tags + if d.tags["MU-ID"] == deploy_id and + (ignoremaster or d.tags["MU-MASTER-IP"] == MU.mu_public_ip) + mappings = MU::Cloud::AWS.apig(region: region, credentials: credentials).get_base_path_mappings(domain_name: d.domain_name, limit: 500).items + mappings.each { |m| + MU.log "Deleting API Gateway Domain Name mapping #{d.domain_name} => #{m.rest_api_id} path #{m.base_path}" + if !noop + MU::Cloud::AWS.apig(region: region, credentials: credentials).delete_base_path_mapping(domain_name: d.domain_name, base_path: m.base_path) + end + } + MU.log "Deleting API Gateway Domain Name #{d.domain_name}" + if !noop + MU::Cloud::AWS.apig(region: region, credentials: credentials).delete_domain_name(domain_name: d.domain_name) + end + end + } + end + resp = MU::Cloud::AWS.apig(region: region, credentials: credentials).get_rest_apis if resp and resp.items resp.items.each { |api| @@ -285,6 +366,7 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, regi end } end + end # Locate an existing API. @@ -314,7 +396,7 @@ def self.find(**args) def toKitten(**_args) bok = { "cloud" => "AWS", - "credentials" => @config['credentials'], + "credentials" => @credentials, "cloud_id" => @cloud_id, "region" => @config['region'] } @@ -326,7 +408,7 @@ def toKitten(**_args) bok['name'] = cloud_desc.name - resources = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources( + resources = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_resources( rest_api_id: @cloud_id, ).items @@ -336,7 +418,7 @@ def toKitten(**_args) r.resource_methods.each_pair { |http_type, m| bok['methods'] ||= [] method = {} - m_desc = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_method( + m_desc = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_method( rest_api_id: @cloud_id, resource_id: r.id, http_method: http_type @@ -432,6 +514,33 @@ def toKitten(**_args) def self.schema(_config) toplevel_required = [] schema = { + "domain_names" => { + "type" => "array", + "items" => { + "description" => "Configure optional Custom Domain Names to map to this API endpoint.", + "type" => "object", + "properties" => { + "certificate" => MU::Config::Ref.schema(type: "certificate", desc: "An existing IAM or ACM SSL certificate to bind to this alternate name endpoint.", omit_fields: ["cloud", "tag", "deploy_id"]), + "dns_record" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "CNAME", need_zone: true, embedded_type: "endpoint")["items"], + "unmanaged_name" => { + "type" => "string", + "description" => "If +dns_record+ is not specified, we will map this string as a domain name and assume that an external DNS record will be created pointing to us at a later time." + }, + "endpoint_type" => { + "type" => "string", + "description" => "The type of endpoint to create with this domain name.", + "default" => "REGIONAL", + "enum" => ["REGIONAL", "EDGE", "PRIVATE"] + }, + "security_policy" => { + "type" => "string", + "default" => "TLS_1_2", + "enum" => ["TLS_1_0", "TLS_1_2"], + "description" => "Acceptable TLS cipher suites. +TLS_1_2+ is strongly recommended." + } + } + } + }, "deploy_to" => { "type" => "string", "description" => "The name of an environment under which to deploy our API. If not specified, will deploy to the name of the global Mu environment for this deployment." @@ -604,7 +713,7 @@ def self.quality # Canonical Amazon Resource Number for this resource # @return [String] def arn - "arn:#{MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws"}:execute-api:#{@config["region"]}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{@cloud_id}" + "arn:#{MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws"}:execute-api:#{@config["region"]}:#{MU::Cloud::AWS.credToAcct(@credentials)}:#{@cloud_id}" end @@ -615,6 +724,36 @@ def arn def self.validateConfig(endpoint, configurator) ok = true + if endpoint['domain_names'] + endpoint['domain_names'].each { |dom| + if dom['certificate'] + cert_arn, cert_domains = MU::Cloud::AWS.resolveSSLCertificate(dom['certificate'], region: dom['region'], credentials: dom['credentials']) + if !cert_arn + MU.log "API Gateway #{endpoint['name']}: Failed to resolve SSL certificate in domain_name block", MU::ERR, details: dom + ok = false + end + end + if !dom['unmanaged_name'] and !dom['dns_record'] + MU.log "API Gateway #{endpoint['name']}: Must specify either unmanaged_name or dns_record in domain_name block", MU::ERR, details: dom + ok = false + end + + # Make at least an attempt to catch when we've specified the same + # DNS name to point to both the main gateway and this alternative + # endpoint, because that ish won't work. This check will miss if + # the end user specifies the zone in competing ways. + if dom['dns_record'] and endpoint['dns_records'] + endpoint['dns_records'].each { |rec| + if rec['name'] == dom['dns_record']['name'] and + rec['zone'] == dom['dns_record']['zone'] + MU.log "API Gateway #{endpoint['name']}: Cannot specify same entry in dns_records and domain_names", MU::ERR, details: rec + ok = false + end + } + end + } + end + append = [] endpoint['deploy_to'] ||= MU.environment || $environment || "dev" endpoint['methods'].each { |m| diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 098ecee7b..daa481111 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -72,7 +72,7 @@ def create # Called automatically by {MU::Deploy#createResources} def groom - +return old_props = MU.structToHash(cloud_desc) new_props = get_properties @@ -116,7 +116,7 @@ def groom statement_id: "#{@mu_name}-ID-1", } - MU.log "trigger properties", MU::NOTICE, details: trigger_properties + MU.log "Adding #{tr['service']} #{tr['name']} trigger to Lambda function #{@cloud_id}", details: trigger_properties begin MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties) rescue Aws::Lambda::Errors::ResourceConflictException From 131ba75c512125b88b6f97bb8960d05588d50dbe Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 9 Aug 2020 23:58:22 -0400 Subject: [PATCH 070/107] AWS::Endpoint: mapping buglet --- modules/mu/providers/aws/endpoint.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 58db6d63e..4f3ee8e67 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -268,13 +268,14 @@ def groom mappings = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_base_path_mappings(domain_name: dnsname, limit: 500).items found = false - mappings.each { |m| -MU.log @cloud_id, MU::WARN, details: m - if m.rest_api_id == @cloud_id and stage == @config['deploy_to'] - found = true - break - end - } + if mappings + mappings.each { |m| + if m.rest_api_id == @cloud_id and stage == @config['deploy_to'] + found = true + break + end + } + end if !found MU.log "Mapping #{dnsname} to API Gateway #{@mu_name}" MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_base_path_mapping( From 598d602a61c8098b25749df5236d78cb2173cea6 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 10 Aug 2020 00:25:39 -0400 Subject: [PATCH 071/107] AWS::Function: uncomment the thing you did to short-circuit grooming while testing unrelated stuff, ya dingus --- modules/mu/providers/aws/function.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index daa481111..1d8bea950 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -72,7 +72,6 @@ def create # Called automatically by {MU::Deploy#createResources} def groom -return old_props = MU.structToHash(cloud_desc) new_props = get_properties From 370770368b0de0ffd8f3e28711435753510c43d8 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 10 Aug 2020 03:28:22 -0400 Subject: [PATCH 072/107] AWS::Bucket: crudely expose bucket CORS configuration --- modules/mu/providers/aws/bucket.rb | 74 ++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/modules/mu/providers/aws/bucket.rb b/modules/mu/providers/aws/bucket.rb index 7908c9e81..8ecd9f754 100644 --- a/modules/mu/providers/aws/bucket.rb +++ b/modules/mu/providers/aws/bucket.rb @@ -205,6 +205,34 @@ def groom ) end + symbolify_keys = Proc.new { |parent| + if parent.is_a?(Hash) + newhash = {} + parent.each_pair { |k, v| + newhash[k.to_sym] = symbolify_keys.call(v) + } + newhash + elsif parent.is_a?(Array) + newarr = [] + parent.each { |child| + newarr << symbolify_keys.call(child) + } + newarr + else + parent + end + } + + if @config['cors'] + MU.log "Setting CORS rules on #{@cloud_id}", details: @config['cors'] + MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_cors( + bucket: @cloud_id, + cors_configuration: { + cors_rules: symbolify_keys.call(@config['cors']) + } + ) + end + MU.log "Bucket #{@config['name']}: s3://#{@cloud_id}", MU::SUMMARY if @config['web'] MU.log "Bucket #{@config['name']} web access: http://#{@cloud_id}.s3-website-#{@config['region']}.amazonaws.com/", MU::SUMMARY @@ -458,6 +486,52 @@ def self.schema(_config) "type" => "string", "enum" => ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", "INTELLIGENT_TIERING", "GLACIER"], "default" => "STANDARD" + }, + "cors" => { + "type" => "array", + "items" => { + "type" => "object", + "description" => "AWS S3 Cross-origin resource sharing policy", + "required" => ["allowed_origins"], + "properties" => { + "allowed_headers" => { + "type" => "array", + "default" => ["*"], + "items" => { + "type" => "string", + "description" => "Specifies which headers are allowed in a preflight request through the +Access-Control-Request-Headers+ header." + } + }, + "allowed_methods" => { + "type" => "array", + "default" => ["GET"], + "items" => { + "type" => "string", + "enum" => %w{GET PUT POST DELETE HEAD}, + "description" => "Specifies which HTTP methods for which cross-domain request are permitted" + } + }, + "allowed_origins" => { + "type" => "array", + "items" => { + "type" => "string", + "description" => "Origins (in URL form) for which cross-domain request are permitted" + } + }, + "expose_headers" => { + "type" => "array", + "items" => { + "type" => "string", + "description" => "Headers in the response which should be visible to the requesting application" + } + }, + "max_age_seconds" => { + "type" => "integer", + "default" => 3600, + "description" => "Maximum cache time for preflight requests" + } + } + } } } [toplevel_required, schema] From bf0cab1dfdbcbe43a56732858aa65597bcc93ff7 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 10 Aug 2020 13:26:17 -0400 Subject: [PATCH 073/107] AWS::CDN: parser stuff --- modules/mu/providers/aws/cdn.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index aa34a19e3..9896fba3d 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -383,10 +383,7 @@ def self.schema(_config) "type" => "object", "description" => "HTTP request artifacts to include in requests passed to our back-end +origin+", "default" => { - "query_string" => false, - "cookies" => { - "forward" => "none" - } + "query_string" => false }, "properties" => { "query_string" => { @@ -397,6 +394,9 @@ def self.schema(_config) "cookies" => { "type" => "object", "description" => "A complex type that specifies whether you want CloudFront to forward cookies to the origin and, if so, which ones.", + "default" => { + "forward" => "none" + }, "properties" => { "forward" => { "type" => "string", From 8d2f2f55d395da6ea8a886c014950684526e76fc Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 12 Aug 2020 03:16:59 -0400 Subject: [PATCH 074/107] AWS::Endpoint: wait maybe don't include the stage name in Custom domain name alias listouts --- modules/mu/providers/aws/endpoint.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 4f3ee8e67..b47d5234f 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -219,7 +219,6 @@ def groom print_dns_alias = Proc.new { |rec| rec['name'] ||= @mu_name.downcase dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec) - MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname+"/"+@config['deploy_to'], MU::SUMMARY dnsname } @@ -228,7 +227,8 @@ def groom # applicable if @config['dns_records'] and !MU::Cloud::AWS.isGovCloud? @config['dns_records'].each { |rec| - print_dns_alias.call(rec) + dnsname = print_dns_alias.call(rec) + MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname+"/"+@config['deploy_to'], MU::SUMMARY } MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: my_hostname) end @@ -236,7 +236,8 @@ def groom if @config['domain_names'] @config['domain_names'].each { |dom| dnsname = if dom['dns_record'] - print_dns_alias.call(dom['dns_record']) + dnsname = print_dns_alias.call(dom['dns_record']) + MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname, MU::SUMMARY else dom['unmanaged_name'] end From 666300357e8a3b2919554a87d9f8433cd7bd7743 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 12 Aug 2020 03:31:19 -0400 Subject: [PATCH 075/107] go back to enabling generic CORS on our API Gatway for testing --- modules/mu/providers/aws/endpoint.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index b47d5234f..f68ef6165 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -236,11 +236,12 @@ def groom if @config['domain_names'] @config['domain_names'].each { |dom| dnsname = if dom['dns_record'] - dnsname = print_dns_alias.call(dom['dns_record']) - MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname, MU::SUMMARY + print_dns_alias.call(dom['dns_record']) else dom['unmanaged_name'] end + MU.log "Alias for API Endpoint #{@config['name']}: https://"+dnsname, MU::SUMMARY + certfield, dnsfield = if dom['endpoint_type'] == "EDGE" [:certificate_arn, :distribution_domain_name] else From 5fa94ef38ef8cb1ed9c724b57a684b7d584e03cd Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 12 Aug 2020 07:10:38 -0400 Subject: [PATCH 076/107] AWS::Endpoint: tweak cors schema to expose valid domain list as a string, instead of a boolean toggling on * --- modules/mu/providers/aws/endpoint.rb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index f68ef6165..ef835c13f 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -131,7 +131,7 @@ def generate_methods(integrations = true) uri, type = if m['integrate_with']['type'] == "aws_generic" svc, action = m['integrate_with']['aws_generic_action'].split(/:/) - ["arn:aws:apigateway:"+@config['region']+":#{svc}:action/#{action}", "AWS"] + ["arn:aws:apigateway:"+@config['region']+":#{svc}:action/#{action}", (m['integrate_with']['proxy'] ? "AWS_PROXY" :"AWS")] elsif m['integrate_with']['type'] == "functions" function_obj = nil MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { function_obj.nil? }) { @@ -640,9 +640,8 @@ def self.schema(_config) "description" => "A Mu resource name, for integrations with a sibling resource (e.g. a Function)" }, "cors" => { - "type" => "boolean", - "description" => "When enabled, this will create an +OPTIONS+ method under this path with request and response header mappings that implement Cross-Origin Resource Sharing", - "default" => true + "type" => "string", + "description" => "When enabled, this will create an +OPTIONS+ method under this path with request and response header mappings that implement Cross-Origin Resource Sharing, setting +Access-Control-Allow-Origin+ to the specified value.", }, "type" => { "type" => "string", @@ -776,13 +775,13 @@ def self.validateConfig(endpoint, configurator) r['headers'] ||= [] r['headers'] << { "header" => "Access-Control-Allow-Origin", - "value" => "*", + "value" => m['cors'], "required" => true } r['headers'].uniq! } - append << cors_option_integrations(m['path']) + append << cors_option_integrations(m['path'], m['cors']) end if !m['iam_role'] @@ -824,7 +823,7 @@ def self.validateConfig(endpoint, configurator) ok end - def self.cors_option_integrations(path) + def self.cors_option_integrations(path, origins) { "type" => "OPTIONS", "path" => path, @@ -845,7 +844,7 @@ def self.cors_option_integrations(path) }, { "header" => "Access-Control-Allow-Origin", - "value" => "*", + "value" => origins, "required" => true } ], From 74ed7d18bd55e2236f51fa53ecaa0f80bd713162 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 12 Aug 2020 07:58:01 -0400 Subject: [PATCH 077/107] AWS::Endpoint: make the proxy flag on method integrations actually do something --- modules/mu/providers/aws/endpoint.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index ef835c13f..cf5c43f42 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -128,16 +128,17 @@ def generate_methods(integrations = true) # end function_obj = nil + aws_int_type = m['integrate_with']['proxy'] ? "AWS_PROXY" : "AWS" uri, type = if m['integrate_with']['type'] == "aws_generic" svc, action = m['integrate_with']['aws_generic_action'].split(/:/) - ["arn:aws:apigateway:"+@config['region']+":#{svc}:action/#{action}", (m['integrate_with']['proxy'] ? "AWS_PROXY" :"AWS")] + ["arn:aws:apigateway:"+@config['region']+":#{svc}:action/#{action}", aws_int_type] elsif m['integrate_with']['type'] == "functions" function_obj = nil MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { function_obj.nil? }) { function_obj = @deploy.findLitterMate(name: m['integrate_with']['name'], type: "functions") } - ["arn:aws:apigateway:"+@config['region']+":lambda:path/2015-03-31/functions/"+function_obj.cloudobj.arn+"/invocations", "AWS"] + ["arn:aws:apigateway:"+@config['region']+":lambda:path/2015-03-31/functions/"+function_obj.cloudobj.arn+"/invocations", aws_int_type] elsif m['integrate_with']['type'] == "mock" [nil, "MOCK"] end @@ -483,6 +484,7 @@ def toKitten(**_args) if m_desc.method_integration.http_method method['integrate_with']['backend_http_method'] = m_desc.method_integration.http_method end + method['proxy'] = true if m_desc.method_integration.type == "AWS_PROXY" elsif m_desc.method_integration.type == "MOCK" method['integrate_with'] = { "type" => "mock" From 9e5ce6c09c42b924c860fa5e075b316822f9baff Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 12 Aug 2020 14:39:48 -0400 Subject: [PATCH 078/107] tag APIGs because we can --- modules/mu/cloud/resource_base.rb | 5 +++++ modules/mu/providers/aws/endpoint.rb | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/mu/cloud/resource_base.rb b/modules/mu/cloud/resource_base.rb index 88f09d8a8..1940fb286 100644 --- a/modules/mu/cloud/resource_base.rb +++ b/modules/mu/cloud/resource_base.rb @@ -227,6 +227,10 @@ class << self } end + MU::MommaCat.listOptionalTags.each_pair { |k, v| + @tags[k] ||= v if v + } + if @cloudparentclass.respond_to?(:resourceInitHook) @cloudparentclass.resourceInitHook(self, @deploy) end @@ -265,6 +269,7 @@ class << self attr_accessor :mu_windows_name # XXX might be ok as reader now end end + @tags["Name"] ||= @mu_name if @mu_name end end diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index cf5c43f42..4fe927d1a 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -18,7 +18,8 @@ def create description: @deploy.deploy_id, endpoint_configuration: { types: ["REGIONAL"] # XXX expose in BoK ["REGIONAL", "EDGE", "PRIVATE"] - } + }, + tags: @tags ) @cloud_id = resp.id generate_methods(false) From 829b31229822bfee80534f71f63c75b56f2a5a56 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 12 Aug 2020 22:13:43 -0400 Subject: [PATCH 079/107] AWS::Endpoint: let's see if we can get working access logs --- modules/mu/providers/aws/endpoint.rb | 77 +++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 4fe927d1a..6066f54dd 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -204,13 +204,19 @@ def generate_methods(integrations = true) def groom generate_methods - MU.log "Deploying API Gateway #{@config['name']} to #{@config['deploy_to']}" - MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_deployment( - rest_api_id: @cloud_id, - stage_name: @config['deploy_to'] + deployment = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_deployments( + rest_api_id: @cloud_id + ).items.sort { |a, b| a.created_date <=> b.created_date }.last + + if !deployment + MU.log "Deploying API Gateway #{@config['name']} to #{@config['deploy_to']}" + deployment = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_deployment( + rest_api_id: @cloud_id, + stage_name: @config['deploy_to'] # cache_cluster_enabled: false, # cache_cluster_size: 0.5, - ) + ) + end # this automatically creates a stage with the same name, so we don't # have to deal with that @@ -274,7 +280,7 @@ def groom found = false if mappings mappings.each { |m| - if m.rest_api_id == @cloud_id and stage == @config['deploy_to'] + if m.rest_api_id == @cloud_id and m.stage == @config['deploy_to'] found = true break end @@ -295,6 +301,39 @@ def groom } end + # The creation of our deployment should have created a matching stage, + # which we're now going to mess with. + stage = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_stage( + rest_api_id: @cloud_id, + stage_name: @config['deploy_to'] + ) + + if @config['access_logs'] and !stage.access_log_settings + log_ref = MU::Config::Ref.get(@config['access_logs']) + MU.log "Enabling API Gateway access logs to CloudWatch Log Group #{log_ref.cloud_id}" + stage = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).update_stage( + rest_api_id: @cloud_id, + stage_name: @config['deploy_to'], + patch_operations: [ + { + op: "replace", + path: "/accessLogSettings/destinationArn", + value: log_ref.kitten.arn.sub(/:\*$/, '') + }, + { + op: "replace", + path: "/accessLogSettings/format", + value: '$context.identity.sourceIp $context.identity.caller $context.identity.user [$context.requestTime] "$context.httpMethod $context.resourcePath $context.protocol" $context.status $context.responseLength $context.requestId' + }, + { + op: "replace", + path: "/description", + value: @deploy.deploy_id + } + ] + ) + end + # resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).create_authorizer( # rest_api_id: @cloud_id, @@ -551,6 +590,12 @@ def self.schema(_config) "type" => "string", "description" => "The name of an environment under which to deploy our API. If not specified, will deploy to the name of the global Mu environment for this deployment." }, + "log_requests" => { + "type" => "boolean", + "description" => "Log all requests to CloudWatch Logs to the log group specified by +access_logs+. If +access_logs+ is unspecified, a reasonable group will be created automatically.", + "default" => true + }, + "access_logs" => MU::Config::Ref.schema(type: "logs", desc: "A pre-existing or sibling Mu Cloudwatch Log group reference. If +log_requests+ is specified and this is not, a log group will be generated automatically. Setting this parameter explicitly automatically enables +log_requests+."), "methods" => { "items" => { "type" => "object", @@ -729,6 +774,26 @@ def arn def self.validateConfig(endpoint, configurator) ok = true + if endpoint['log_requests'] and !endpoint['access_logs'] + logdesc = { + "name" => endpoint['name']+"accesslogs", + } + logdesc["tags"] = endpoint["tags"] if endpoint['tags'] + configurator.insertKitten(logdesc, "logs") + endpoint['access_logs'] = MU::Config::Ref.get( + name: endpoint['name']+"accesslogs", + type: "log", + cloud: "AWS", + credentials: endpoint['credentials'], + region: endpoint['region'] + ) + end + + if endpoint['access_logs'] and endpoint["access_logs"]["name"] + endpoint['log_requests'] = true + MU::Config.addDependency(endpoint, endpoint["access_logs"]["name"], "log") + end + if endpoint['domain_names'] endpoint['domain_names'].each { |dom| if dom['certificate'] From 5caa3eca18508d557279ddfe043fc44a4106c69d Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 13 Aug 2020 01:54:35 -0400 Subject: [PATCH 080/107] AWS::Endpoint: even more gooder CloudWatch logging --- modules/mu/providers/aws/endpoint.rb | 39 ++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 6066f54dd..2c510103e 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -329,6 +329,16 @@ def groom op: "replace", path: "/description", value: @deploy.deploy_id + }, + { + op: "replace", + path: "/*/*/logging/dataTrace", + value: "true" + }, + { + op: "replace", + path: "/*/*/logging/loglevel", + value: "INFO" } ] ) @@ -457,7 +467,6 @@ def toKitten(**_args) ).items resources.each { |r| - bok['deploy_to'] ||= r.path if r.path != "/" # XXX this is wrong/inadequate next if !r.respond_to?(:resource_methods) or r.resource_methods.nil? r.resource_methods.each_pair { |http_type, m| bok['methods'] ||= [] @@ -550,6 +559,32 @@ def toKitten(**_args) } } + deployment = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_deployments( + rest_api_id: @cloud_id + ).items.sort { |a, b| a.created_date <=> b.created_date }.last + stages = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).get_stages( + rest_api_id: @cloud_id, + deployment_id: deployment.id + ) + + # XXX we only support a single stage right now, which is a dumb + # limitation + stage = stages.item.first + if stage + bok['deploy_to'] = stage.stage_name + if stage.access_log_settings + bok['log_requests'] = true + bok['access_logs'] = MU::Config::Ref.get( + id: stage.access_log_settings.destination_arn.sub(/.*?:([^:]+)$/, '\1'), + credentials: @credentials, + region: @config['region'], + type: "logs", + cloud: "AWS" + ) + end + end + + bok end @@ -592,7 +627,7 @@ def self.schema(_config) }, "log_requests" => { "type" => "boolean", - "description" => "Log all requests to CloudWatch Logs to the log group specified by +access_logs+. If +access_logs+ is unspecified, a reasonable group will be created automatically.", + "description" => "Log custom access requests to CloudWatch Logs to the log group specified by +access_logs+, as well as enabling built-in CloudWatch Logs at +INFO+ level. If +access_logs+ is unspecified, a reasonable group will be created automatically.", "default" => true }, "access_logs" => MU::Config::Ref.schema(type: "logs", desc: "A pre-existing or sibling Mu Cloudwatch Log group reference. If +log_requests+ is specified and this is not, a log group will be generated automatically. Setting this parameter explicitly automatically enables +log_requests+."), From 01e4834e68571237b921c9919f8ac86c9015bf00 Mon Sep 17 00:00:00 2001 From: John Stange Date: Sun, 16 Aug 2020 21:57:57 -0400 Subject: [PATCH 081/107] AWS::Endpoint: expose integration request parameters, and provide a simple boolean for async Lambdas (which is what I really wanted) --- modules/mu/providers/aws/endpoint.rb | 78 ++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index 2c510103e..da5fdd662 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -170,6 +170,11 @@ def generate_methods(integrations = true) params[:request_templates][rt['content_type']] = rt['template'] } end + if m['integrate_with']['parameters'] + params[:request_parameters] = Hash[m['integrate_with']['parameters'].map { |p| + ["integration.request.#{p['type']}.#{p['name']}", p['value']] + }] + end resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @credentials).put_integration(params) @@ -476,7 +481,7 @@ def toKitten(**_args) resource_id: r.id, http_method: http_type ) -MU.log bok['name']+" "+http_type, MU::WARN, details: m_desc if bok['name'].match(/ESPIER/) + method['type'] = http_type method['path'] = r.path_part || r.path if m_desc.method_responses @@ -549,10 +554,31 @@ def toKitten(**_args) if m_desc.method_integration.request_templates and !m_desc.method_integration.request_templates.empty? - method['integrate_with'] = m_desc.method_integration.request_templates.keys.map { |rt_content_type, template| + method['integrate_with']['request_templates'] = m_desc.method_integration.request_templates.keys.map { |rt_content_type, template| { "content_type" => rt_content_type, "template" => template } } end + + if m_desc.method_integration.request_parameters + m_desc.method_integration.request_parameters.each_pair { |k, v| + if !k.match(/^integration\.request\.(header|querystring|path)\.(.*)/) + MU.log "Don't know how to handle integration request parameter '#{k}', skipping", MU::WARN + next + end + if Regexp.last_match[1] == "header" and + Regexp.last_match[2] == "X-Amz-Invocation-Type" and + v == "'Event'" + method['integrate_with']['async'] = true + else + method['integrate_with']['parameters'] ||= [] + method['integrate_with']['parameters'] << { + "type" => Regexp.last_match[1], + "name" => Regexp.last_match[2], + "value" => v + } + end + } + end end bok['methods'] << method @@ -641,10 +667,37 @@ def self.schema(_config) "type" => "object", "description" => "Specify what application backend to invoke under this path/method combination", "properties" => { + "async" => { + "type" => "boolean", + "default" => false, + "description" => "For non-proxy Lambda integrations, adds a static +X-Amz-Invocation-Type+ with value +'Event'+ to invoke the function asynchronously. See also https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-integration-async.html" + }, + "parameters" => { + "type" => "array", + "items" => { + "description" => "One or headers, paths, or query string parameters to pass as request parameters to our back end. See also: https://docs.aws.amazon.com/apigateway/latest/developerguide/request-response-data-mappings.html", + "type" => "object", + "properties" => { + "name" => { + "type" => "string", + "description" => "A valid and unique integration request parameter name." + }, + "value" => { + "type" => "string", + "description" => "The name of a method request parameter, or a static value contained in single quotes (+'foo'+)." + }, + "type" => { + "type" => "string", + "description" => "Which HTTP artifact to use when presenting the parameter to the back end. ", + "enum" => ["header", "querystring", "path"] + } + } + } + }, "proxy" => { "type" => "boolean", "default" => false, - "description" => "For HTTP or AWS integrations, specify whether the target is a proxy (((docs unclear, is that actually what this means?)))" # XXX is that actually what this means? + "description" => "Sets HTTP integrations to HTTP_PROXY and AWS/LAMBDA integrations to AWS_PROXY/LAMBDA_PROXY" }, "backend_http_method" => { "type" => "string", @@ -862,6 +915,22 @@ def self.validateConfig(endpoint, configurator) append = [] endpoint['deploy_to'] ||= MU.environment || $environment || "dev" endpoint['methods'].each { |m| + if m['integrate_with']['async'] + if m['integrate_with']['type'] == "functions" and + m['integrate_with']['async'] + m['integrate_with']['parameters'] ||= [] + m['integrate_with']['parameters'] << { + "name" => "X-Amz-Invocation-Type", + "value" => "'Event'", # yes the single quotes are required + "type" => "header" + } + if m['integrate_with']['proxy'] + MU.log "Cannot specify both of proxy and async for API Gateway method integration", MU::ERR + ok = false + end + end + end + if m['integrate_with'] and m['integrate_with']['name'] if m['integrate_with']['type'] != "aws_generic" MU::Config.addDependency(endpoint, m['integrate_with']['name'], m['integrate_with']['type']) @@ -887,6 +956,7 @@ def self.validateConfig(endpoint, configurator) append << cors_option_integrations(m['path'], m['cors']) end + if !m['iam_role'] m['uri'] ||= "*" if m['integrate_with']['type'] == "aws_generic" @@ -908,7 +978,7 @@ def self.validateConfig(endpoint, configurator) "targets" => [{ "identifier" => m['uri'] }] } ] - elsif m['integrate_with']['type'] == "function" + elsif m['integrate_with']['type'] == "functions" roledesc["import"] = ["AWSLambdaBasicExecutionRole"] end configurator.insertKitten(roledesc, "roles") From 62423162d6dcb008f0a30341ce3b32de265e74d4 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 20 Aug 2020 03:17:30 -0400 Subject: [PATCH 082/107] AWS::SearchDomain: don't try to set access_policies until groom phase --- modules/mu/providers/aws/search_domain.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 76254d016..153aa116f 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -36,6 +36,7 @@ def create MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params @cloud_id = @config['domain_name'] + pp params # "Resource": "arn:aws:es:us-west-1:987654321098:domain/test-domain/*" MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).create_elasticsearch_domain(params).domain_status tagDomain @@ -599,7 +600,7 @@ def genParams(ext = nil) params[:snapshot_options][:automated_snapshot_start_hour] = @config['snapshot_hour'] end - if @config['access_policies'] + if @config['access_policies'] and ext # TODO check against ext.access_policies.options params[:access_policies] = JSON.generate(@config['access_policies']) end From f4966f0745eec4e8fe34acac563685b046cc7055 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 2 Sep 2020 01:26:38 -0400 Subject: [PATCH 083/107] AWS::MsgQueue:.find: look up short queue names to urls; return graceful when the thing doesn't exist --- modules/mu/providers/aws/msg_queue.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/mu/providers/aws/msg_queue.rb b/modules/mu/providers/aws/msg_queue.rb index ed1aebe01..e122ab3b2 100644 --- a/modules/mu/providers/aws/msg_queue.rb +++ b/modules/mu/providers/aws/msg_queue.rb @@ -195,7 +195,15 @@ def self.find(**args) # Go fetch its attributes fetch = if args[:cloud_id] - [args[:cloud_id]] + if args[:cloud_id] !~ /^https?:\/\// + [begin + MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).get_queue_url(queue_name: args[:cloud_id]).queue_url + rescue Aws::SQS::Errors::NonExistentQueue + return found + end] + else + [args[:cloud_id]] + end else resp = MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).list_queues resp.queue_urls From 11186aa186ad712bc189e97401773b23ef8f37e7 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 2 Sep 2020 01:40:55 -0400 Subject: [PATCH 084/107] AWS: ditch a lot of debugging noise --- modules/mu/providers/aws/cdn.rb | 5 ----- modules/mu/providers/aws/job.rb | 3 --- modules/mu/providers/aws/nosqldb.rb | 6 +++--- modules/mu/providers/aws/notifier.rb | 4 ++-- modules/mu/providers/aws/search_domain.rb | 1 - modules/mu/providers/google.rb | 1 + 6 files changed, 6 insertions(+), 14 deletions(-) diff --git a/modules/mu/providers/aws/cdn.rb b/modules/mu/providers/aws/cdn.rb index 9896fba3d..6f265e7cf 100644 --- a/modules/mu/providers/aws/cdn.rb +++ b/modules/mu/providers/aws/cdn.rb @@ -91,7 +91,6 @@ def groom bucketref = get_bucketref_from_domain(o.domain_name) next if !bucketref or !bucketref.kitten resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).get_cloud_front_origin_access_identity(id: id) - pp resp # bucketref.kitten.allowPrincipal("arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity "+id, doc_id: "PolicyForCloudFrontPrivateContent", permissions: ["GetObject"]) bucketref.kitten.allowPrincipal(resp.cloud_front_origin_access_identity.s3_canonical_user_id, doc_id: "PolicyForCloudFrontPrivateContent", permissions: ["GetObject"], name: @mu_name) end @@ -331,10 +330,6 @@ def toKitten(**_args) } end -if bok["name"] =~ /espier/i - MU.log @cloud_id+" cloud_desc", MU::NOTICE, details: cloud_desc.origins -# MU.log @cloud_id+" bok", MU::NOTICE, details: bok -end bok end diff --git a/modules/mu/providers/aws/job.rb b/modules/mu/providers/aws/job.rb index fc6e97f99..9f8250a9a 100644 --- a/modules/mu/providers/aws/job.rb +++ b/modules/mu/providers/aws/job.rb @@ -221,7 +221,6 @@ def toKitten(**_args) targets.each { |t| bok['targets'] ||= [] _arn, _plat, service, region, account, resource = t.arn.split(/:/, 6) - MU.log service+" in "+region+" under "+account, MU::WARN, details: resource target_type = if service == "lambda" resource.sub!(/^function:/, '') "functions" @@ -253,8 +252,6 @@ def toKitten(**_args) bok['targets'] << MU::Config::Ref.get(ref_params) } - MU.log @cloud_id, MU::NOTICE, details: bok - # XXX cloud_desc.event_pattern - what do we want to do with this? bok diff --git a/modules/mu/providers/aws/nosqldb.rb b/modules/mu/providers/aws/nosqldb.rb index 61b70c4f8..fc6d0d110 100644 --- a/modules/mu/providers/aws/nosqldb.rb +++ b/modules/mu/providers/aws/nosqldb.rb @@ -315,10 +315,10 @@ def toKitten(**_args) } if cloud_desc.stream_specification and cloud_desc.stream_specification.stream_enabled -MU.log @cloud_id, MU::NOTICE, details: cloud_desc + bok['stream'] = cloud_desc.stream_specification.stream_view_type - cloud_desc.latest_stream_arn - pp MU::Cloud::AWS.dynamostream(credentials: @credentials, region: @config['region']).list_streams +# cloud_desc.latest_stream_arn +# MU::Cloud::AWS.dynamostream(credentials: @credentials, region: @config['region']).list_streams end bok["populate"] = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @config['region']).scan( diff --git a/modules/mu/providers/aws/notifier.rb b/modules/mu/providers/aws/notifier.rb index 36eaf3f9c..1835f0e11 100644 --- a/modules/mu/providers/aws/notifier.rb +++ b/modules/mu/providers/aws/notifier.rb @@ -174,7 +174,7 @@ def toKitten(**_args) MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config return nil end -pp cloud_desc if @cloud_id == "Espier-Publish-Domains" + bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"] svcmap = { "lambda" => "functions", @@ -182,6 +182,7 @@ def toKitten(**_args) } MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub| bok['subscriptions'] ||= [] + bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/) _wholestring, service, region, account, id = Regexp.last_match.to_a { @@ -206,7 +207,6 @@ def toKitten(**_args) } end } -pp bok['subscriptions'] if @cloud_id == "Espier-Publish-Domains" bok end diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 153aa116f..eca94d23b 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -36,7 +36,6 @@ def create MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params @cloud_id = @config['domain_name'] - pp params # "Resource": "arn:aws:es:us-west-1:987654321098:domain/test-domain/*" MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).create_elasticsearch_domain(params).domain_status tagDomain diff --git a/modules/mu/providers/google.rb b/modules/mu/providers/google.rb index 30ae31340..2f2573e31 100644 --- a/modules/mu/providers/google.rb +++ b/modules/mu/providers/google.rb @@ -1024,6 +1024,7 @@ def self.getDomains(credentials = nil) # @return [Array],nil] def self.getOrg(credentials = nil, with_id: nil) creds = MU::Cloud::Google.credConfig(credentials) + return nil if !creds credname = if creds and creds['name'] creds['name'] else From 533c69c47fa03f50273cbe1410d726cf24d5e8fb Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 2 Sep 2020 02:46:56 -0400 Subject: [PATCH 085/107] mu-run-tests: add retries to --dryrun mode because AWS is a spaz --- bin/mu-run-tests | 16 ++++++++++++++-- modules/mu/config/container_cluster.rb | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bin/mu-run-tests b/bin/mu-run-tests index b71e1bcb6..546600e60 100755 --- a/bin/mu-run-tests +++ b/bin/mu-run-tests @@ -34,6 +34,7 @@ Usage: #{$0} [-m <#>] [-f] [-v] [specific test BoK to run [...]] EOS opt :max_threads, "Environment to set on creation.", :require => false, :default => 3, :type => :integer + opt :max_retries, "Number of times to retry failed tests in --dryrun mode.", :require => false, :default => 2, :type => :integer opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false opt :verbose, "Show more information while running", :require => false, :default => false end @@ -121,8 +122,19 @@ def execCommand(cmd, results_stash) } ok = true - output = %x{#{cmd} 2>&1} - ok = false if $?.exitstatus != 0 + retries = 0 + begin + output = %x{#{cmd} 2>&1} + if $?.exitstatus != 0 + ok = false + retries += 1 + if $opts[:verbose] and !$opts[:full] and retries <= $opts[:max_retries] + puts "#{cmd} RETRY #{retries.to_s}".light_red + end + else + ok = true + end + end while !ok and !$opts[:full] and retries <= $opts[:max_retries] results_stash["output"] += output diff --git a/modules/mu/config/container_cluster.rb b/modules/mu/config/container_cluster.rb index c7dc192cf..cbe198962 100644 --- a/modules/mu/config/container_cluster.rb +++ b/modules/mu/config/container_cluster.rb @@ -48,7 +48,7 @@ def self.schema "properties" => { "version" => { "type" => "string", - "default" => "1.14", + "default" => "1.15", "description" => "Version of Kubernetes control plane to deploy", }, "max_pods" => { From b476281410880b296f3fcfbae5eabfe6d3e1e1c7 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 2 Sep 2020 16:48:06 -0400 Subject: [PATCH 086/107] gem updates, including security fix for rack --- modules/Gemfile.lock | 143 ++++++++++++++++++++++--------------------- 1 file changed, 73 insertions(+), 70 deletions(-) diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index 7bd023b58..adbbff95c 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -54,10 +54,10 @@ GEM public_suffix (>= 2.0.2, < 4.0) ast (2.4.1) aws-eventstream (1.1.0) - aws-sdk-core (2.11.540) + aws-sdk-core (2.11.578) aws-sigv4 (~> 1.0) jmespath (~> 1.0) - aws-sigv4 (1.2.1) + aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) azure-core (0.1.15) faraday (~> 0.9) @@ -132,7 +132,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_api_management (0.19.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_appconfiguration (0.17.2) + azure_mgmt_appconfiguration (0.17.3) ms_rest_azure (~> 0.12.0) azure_mgmt_attestation (0.17.1) ms_rest_azure (~> 0.12.0) @@ -142,6 +142,8 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_azurestack (0.17.2) ms_rest_azure (~> 0.12.0) + azure_mgmt_azurestack_hci (0.17.0) + ms_rest_azure (~> 0.12.0) azure_mgmt_batch (0.18.1) ms_rest_azure (~> 0.12.0) azure_mgmt_batchai (0.17.1) @@ -150,7 +152,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_bot_service (0.17.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_cdn (0.17.4) + azure_mgmt_cdn (0.17.6) ms_rest_azure (~> 0.12.0) azure_mgmt_cognitive_services (0.19.2) ms_rest_azure (~> 0.12.0) @@ -160,19 +162,19 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_consumption (0.18.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_container_instance (0.17.5) + azure_mgmt_container_instance (0.17.6) ms_rest_azure (~> 0.12.0) azure_mgmt_container_registry (0.18.4) ms_rest_azure (~> 0.12.0) - azure_mgmt_container_service (0.20.2) + azure_mgmt_container_service (0.20.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_cosmosdb (0.21.2) + azure_mgmt_cosmosdb (0.22.0) ms_rest_azure (~> 0.12.0) azure_mgmt_cost_management (0.17.1) ms_rest_azure (~> 0.12.0) azure_mgmt_customer_insights (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_data_factory (0.18.3) + azure_mgmt_data_factory (0.18.4) ms_rest_azure (~> 0.12.0) azure_mgmt_data_migration (0.18.1) ms_rest_azure (~> 0.12.0) @@ -202,11 +204,11 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_hanaonazure (0.18.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_hdinsight (0.18.0) + azure_mgmt_hdinsight (0.18.2) ms_rest_azure (~> 0.12.0) azure_mgmt_import_export (0.17.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_iot_central (0.19.1) + azure_mgmt_iot_central (0.19.2) ms_rest_azure (~> 0.12.0) azure_mgmt_iot_hub (0.17.4) ms_rest_azure (~> 0.12.0) @@ -214,7 +216,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_kubernetes_configuration (0.17.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_kusto (0.19.2) + azure_mgmt_kusto (0.19.3) ms_rest_azure (~> 0.12.0) azure_mgmt_labservices (0.17.2) ms_rest_azure (~> 0.12.0) @@ -248,13 +250,13 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_mysql (0.17.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_netapp (0.20.0) + azure_mgmt_netapp (0.20.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_network (0.23.4) + azure_mgmt_network (0.24.0) ms_rest_azure (~> 0.12.0) azure_mgmt_notification_hubs (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_operational_insights (0.18.0) + azure_mgmt_operational_insights (0.19.1) ms_rest_azure (~> 0.12.0) azure_mgmt_operations_management (0.17.1) ms_rest_azure (~> 0.12.0) @@ -284,13 +286,13 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_relay (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_reservations (0.19.2) + azure_mgmt_reservations (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_resource_health (0.17.1) ms_rest_azure (~> 0.12.0) azure_mgmt_resourcegraph (0.17.2) ms_rest_azure (~> 0.12.0) - azure_mgmt_resources (0.17.9) + azure_mgmt_resources (0.18.0) ms_rest_azure (~> 0.12.0) azure_mgmt_resources_management (0.17.2) ms_rest_azure (~> 0.12.0) @@ -314,7 +316,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_stor_simple8000_series (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_storage (0.21.1) + azure_mgmt_storage (0.21.2) ms_rest_azure (~> 0.12.0) azure_mgmt_storagecache (0.18.1) ms_rest_azure (~> 0.12.0) @@ -326,7 +328,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_support (0.17.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_synapse (0.17.1) + azure_mgmt_synapse (0.17.2) ms_rest_azure (~> 0.12.0) azure_mgmt_time_series_insights (0.17.1) ms_rest_azure (~> 0.12.0) @@ -336,7 +338,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_web (0.17.6) ms_rest_azure (~> 0.12.0) - azure_sdk (0.59.0) + azure_sdk (0.62.0) azure-storage (~> 0.14.0.preview) azure_cognitiveservices_anomalydetector (~> 0.17.1) azure_cognitiveservices_autosuggest (~> 0.17.2) @@ -370,27 +372,28 @@ GEM azure_mgmt_alerts_management (~> 0.17.1) azure_mgmt_analysis_services (~> 0.17.3) azure_mgmt_api_management (~> 0.19.1) - azure_mgmt_appconfiguration (~> 0.17.2) + azure_mgmt_appconfiguration (~> 0.17.3) azure_mgmt_attestation (~> 0.17.1) azure_mgmt_authorization (~> 0.18.5) azure_mgmt_automation (~> 0.17.3) azure_mgmt_azurestack (~> 0.17.2) + azure_mgmt_azurestack_hci (~> 0.17.0) azure_mgmt_batch (~> 0.18.1) azure_mgmt_batchai (~> 0.17.1) azure_mgmt_billing (~> 0.17.3) azure_mgmt_bot_service (~> 0.17.1) - azure_mgmt_cdn (~> 0.17.4) + azure_mgmt_cdn (~> 0.17.6) azure_mgmt_cognitive_services (~> 0.19.2) azure_mgmt_commerce (~> 0.17.2) azure_mgmt_compute (~> 0.19.3) azure_mgmt_consumption (~> 0.18.1) - azure_mgmt_container_instance (~> 0.17.5) + azure_mgmt_container_instance (~> 0.17.6) azure_mgmt_container_registry (~> 0.18.4) - azure_mgmt_container_service (~> 0.20.2) - azure_mgmt_cosmosdb (~> 0.21.2) + azure_mgmt_container_service (~> 0.20.3) + azure_mgmt_cosmosdb (~> 0.22.0) azure_mgmt_cost_management (~> 0.17.1) azure_mgmt_customer_insights (~> 0.17.3) - azure_mgmt_data_factory (~> 0.18.3) + azure_mgmt_data_factory (~> 0.18.4) azure_mgmt_data_migration (~> 0.18.1) azure_mgmt_databox (~> 0.17.1) azure_mgmt_datalake_analytics (~> 0.17.3) @@ -405,13 +408,13 @@ GEM azure_mgmt_event_hub (~> 0.18.1) azure_mgmt_features (~> 0.17.4) azure_mgmt_hanaonazure (~> 0.18.1) - azure_mgmt_hdinsight (~> 0.18.0) + azure_mgmt_hdinsight (~> 0.18.2) azure_mgmt_import_export (~> 0.17.1) - azure_mgmt_iot_central (~> 0.19.1) + azure_mgmt_iot_central (~> 0.19.2) azure_mgmt_iot_hub (~> 0.17.4) azure_mgmt_key_vault (~> 0.17.6) azure_mgmt_kubernetes_configuration (~> 0.17.1) - azure_mgmt_kusto (~> 0.19.2) + azure_mgmt_kusto (~> 0.19.3) azure_mgmt_labservices (~> 0.17.2) azure_mgmt_links (~> 0.17.3) azure_mgmt_locks (~> 0.17.4) @@ -428,10 +431,10 @@ GEM azure_mgmt_monitor (~> 0.17.6) azure_mgmt_msi (~> 0.17.2) azure_mgmt_mysql (~> 0.17.1) - azure_mgmt_netapp (~> 0.20.0) - azure_mgmt_network (~> 0.23.4) + azure_mgmt_netapp (~> 0.20.1) + azure_mgmt_network (~> 0.24.0) azure_mgmt_notification_hubs (~> 0.17.3) - azure_mgmt_operational_insights (~> 0.18.0) + azure_mgmt_operational_insights (~> 0.19.1) azure_mgmt_operations_management (~> 0.17.1) azure_mgmt_peering (~> 0.17.1) azure_mgmt_policy (~> 0.17.9) @@ -446,10 +449,10 @@ GEM azure_mgmt_recovery_services_site_recovery (~> 0.17.3) azure_mgmt_redis (~> 0.17.4) azure_mgmt_relay (~> 0.17.3) - azure_mgmt_reservations (~> 0.19.2) + azure_mgmt_reservations (~> 0.20.0) azure_mgmt_resource_health (~> 0.17.1) azure_mgmt_resourcegraph (~> 0.17.2) - azure_mgmt_resources (~> 0.17.9) + azure_mgmt_resources (~> 0.18.0) azure_mgmt_resources_management (~> 0.17.2) azure_mgmt_scheduler (~> 0.17.2) azure_mgmt_search (~> 0.17.3) @@ -461,13 +464,13 @@ GEM azure_mgmt_sql (~> 0.19.1) azure_mgmt_sqlvirtualmachine (~> 0.18.2) azure_mgmt_stor_simple8000_series (~> 0.17.3) - azure_mgmt_storage (~> 0.21.1) + azure_mgmt_storage (~> 0.21.2) azure_mgmt_storagecache (~> 0.18.1) azure_mgmt_storagesync (~> 0.18.1) azure_mgmt_stream_analytics (~> 0.17.3) azure_mgmt_subscriptions (~> 0.18.4) azure_mgmt_support (~> 0.17.1) - azure_mgmt_synapse (~> 0.17.1) + azure_mgmt_synapse (~> 0.17.2) azure_mgmt_time_series_insights (~> 0.17.1) azure_mgmt_traffic_manager (~> 0.17.4) azure_mgmt_vmware_cloudsimple (~> 0.17.1) @@ -547,7 +550,7 @@ GEM winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.0) - chef-sugar (5.1.9) + chef-sugar (5.1.11) chef-vault (3.4.3) chef-zero (14.0.17) ffi-yajl (~> 2.2) @@ -563,16 +566,16 @@ GEM cleanroom (1.0.0) color (1.8) colorize (0.8.1) - concurrent-ruby (1.1.6) - cookbook-omnifetch (0.9.1) + concurrent-ruby (1.1.7) + cookbook-omnifetch (0.11.1) mixlib-archive (>= 0.4, < 2.0) - cucumber-core (7.1.0) - cucumber-gherkin (~> 14.0, >= 14.0.1) - cucumber-messages (~> 12.2, >= 12.2.0) + cucumber-core (8.0.1) + cucumber-gherkin (~> 15.0, >= 15.0.2) + cucumber-messages (~> 13.0, >= 13.0.1) cucumber-tag-expressions (~> 2.0, >= 2.0.4) - cucumber-gherkin (14.0.1) - cucumber-messages (~> 12.2, >= 12.2.0) - cucumber-messages (12.2.0) + cucumber-gherkin (15.0.2) + cucumber-messages (~> 13.0, >= 13.0.1) + cucumber-messages (13.0.1) protobuf-cucumber (~> 3.10, >= 3.10.8) cucumber-tag-expressions (2.0.4) daemons (1.3.1) @@ -586,15 +589,15 @@ GEM eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) - faraday-cookie_jar (0.0.6) - faraday (>= 0.7.4) + faraday-cookie_jar (0.0.7) + faraday (>= 0.8.0) http-cookie (~> 1.0.0) faraday_middleware (0.14.0) faraday (>= 0.7.4, < 1.0) ffi (1.13.1) - ffi-libarchive (1.0.3) + ffi-libarchive (1.0.4) ffi (~> 1.0) - ffi-yajl (2.3.3) + ffi-yajl (2.3.4) libyajl2 (~> 1.2) foodcritic (14.1.0) cucumber-core (>= 1.3) @@ -613,7 +616,7 @@ GEM representable (~> 3.0) retriable (>= 2.0, < 4.0) signet (~> 0.12) - googleauth (0.13.0) + googleauth (0.13.1) faraday (>= 0.17.3, < 2.0) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -629,7 +632,7 @@ GEM http-cookie (1.0.3) domain_name (~> 0.5) httpclient (2.8.3) - i18n (1.8.3) + i18n (1.8.5) concurrent-ruby (~> 1.0) inifile (3.0.0) iniparse (1.5.0) @@ -637,29 +640,29 @@ GEM jmespath (1.4.0) json-schema (2.8.1) addressable (>= 2.4) - jwt (2.2.1) + jwt (2.2.2) libyajl2 (1.2.0) little-plugger (1.1.4) - logging (2.2.2) + logging (2.3.0) little-plugger (~> 1.1) - multi_json (~> 1.10) + multi_json (~> 1.14) memoist (0.16.2) middleware (0.1.0) mini_mime (1.0.2) mini_portile2 (2.4.0) minitar (0.9) - minitest (5.14.1) - mixlib-archive (1.0.5) + minitest (5.14.2) + mixlib-archive (1.0.7) mixlib-log mixlib-authentication (2.1.1) mixlib-cli (1.7.0) - mixlib-config (3.0.6) + mixlib-config (3.0.9) tomlrb - mixlib-install (3.12.1) + mixlib-install (3.12.3) mixlib-shellout mixlib-versioning thor - mixlib-log (3.0.8) + mixlib-log (3.0.9) mixlib-shellout (2.4.4) mixlib-versioning (1.2.12) molinillo (0.6.6) @@ -672,10 +675,10 @@ GEM faraday (>= 0.9, < 2.0.0) faraday-cookie_jar (~> 0.0.6) ms_rest (~> 0.7.6) - multi_json (1.14.1) + multi_json (1.15.0) multipart-post (2.1.1) mysql2 (0.5.2) - net-ldap (0.16.2) + net-ldap (0.16.3) net-scp (1.2.1) net-ssh (>= 2.6.5) net-sftp (2.1.2) @@ -688,7 +691,7 @@ GEM net-ssh-gateway (>= 1.2.0) net-telnet (0.1.1) netaddr (2.0.4) - nokogiri (1.10.9) + nokogiri (1.10.10) mini_portile2 (~> 2.4.0) nori (2.6.0) numerizer (0.1.1) @@ -709,7 +712,7 @@ GEM wmi-lite (~> 1.0) openssl-oaep (0.1.0) optimist (3.0.1) - os (1.1.0) + os (1.1.1) paint (1.0.1) parallel (1.19.2) parser (2.7.1.4) @@ -753,17 +756,17 @@ GEM rspec_junit_formatter (0.2.3) builder (< 4) rspec-core (>= 2, < 4, != 2.12.0) - rubocop (0.86.0) + rubocop (0.90.0) parallel (~> 1.10) - parser (>= 2.7.0.1) + parser (>= 2.7.1.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.7) rexml - rubocop-ast (>= 0.0.3, < 1.0) + rubocop-ast (>= 0.3.0, < 1.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (0.1.0) - parser (>= 2.7.0.1) + rubocop-ast (0.3.0) + parser (>= 2.7.1.4) ruby-graphviz (1.2.5) rexml ruby-progressbar (1.10.1) @@ -788,10 +791,10 @@ GEM multi_json (~> 1.10) simple-password-gen (0.1.5) slack-notifier (2.3.2) - solve (4.0.3) + solve (4.0.4) molinillo (~> 0.6) semverse (>= 1.1, < 4.0) - specinfra (2.82.17) + specinfra (2.82.19) net-scp net-ssh (>= 2.7) net-telnet (= 0.1.1) @@ -836,7 +839,7 @@ GEM winrm (~> 2.0) wmi-lite (1.0.5) yard (0.9.25) - zeitwerk (2.3.1) + zeitwerk (2.4.0) PLATFORMS ruby From 0399cc233707e064344fa99d29734b62788ef1b5 Mon Sep 17 00:00:00 2001 From: John Stange Date: Wed, 2 Sep 2020 22:12:06 -0400 Subject: [PATCH 087/107] add an infra-only version of sitemonitor as a test item --- modules/tests/microservice_app.yaml | 288 ++++++++++++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 modules/tests/microservice_app.yaml diff --git a/modules/tests/microservice_app.yaml b/modules/tests/microservice_app.yaml new file mode 100644 index 000000000..44bbbe01c --- /dev/null +++ b/modules/tests/microservice_app.yaml @@ -0,0 +1,288 @@ +# Old Sitemonitor, with serial numbers and code filed off. This will *only* +# work on our own Labs sandbox, unless you feed it a different domain name to +# play in. +# clouds: AWS +--- +appname: espier +parameters: +- name: domain + default: "sandbox.egt-labs.com" # this must exist as a Route53 zone and have a corresponding wildcard ACM or IAM SSL certificate +jobs: +- name: clear-scan-data + schedule: + minute: '0' + hour: '1' + day_of_month: '*' + month: "*" + day_of_week: "?" + year: "*" + targets: + - type: functions + name: empty-out-table +- name: run-scans + schedule: + minute: '0' + hour: '2' + day_of_month: '*' + month: "*" + day_of_week: "?" + year: "*" + targets: + - type: functions + name: queue-domains + +cdns: +- name: front + origins: + - name: default + bucket: + name: bucket + certificate: + name: "*.<%= domain %>" + dns_records: + - zone: + name: <%= domain %> + behaviors: + - origin: default + forwarded_values: + headers: + - Origin + - Access-Control-Request-Headers + - Access-Control-Request-Method + - Access-Control-Allow-Origin + +roles: +- name: dynamostream-to-es + can_assume: + - assume_method: basic + entity_type: service + entity_id: lambda.amazonaws.com + attachable_policies: + - id: AWSLambdaInvocation-DynamoDB + - id: AWSLambdaBasicExecutionRole + policies: + - name: allow_es_posting + permissions: + - es:ESHttpPost + targets: + - identifier: domains-scan-data + type: search_domain + path: "/*" +- name: empty-out-table + can_assume: + - assume_method: basic + entity_type: service + entity_id: lambda.amazonaws.com + attachable_policies: + - id: AmazonDynamoDBFullAccess + - id: AWSLambdaBasicExecutionRole +- name: on-demand-scanner + can_assume: + - assume_method: basic + entity_type: service + entity_id: lambda.amazonaws.com + attachable_policies: + - id: AmazonDynamoDBFullAccess + - id: AWSLambdaBasicExecutionRole +- name: queue-domains + can_assume: + - assume_method: basic + entity_type: service + entity_id: lambda.amazonaws.com + attachable_policies: + - id: AmazonDynamoDBFullAccess + - id: AmazonSNSFullAccess + - id: AWSLambdaBasicExecutionRole +- name: scheduled-scanner + can_assume: + - assume_method: basic + entity_type: service + entity_id: lambda.amazonaws.com + attachable_policies: + - id: AmazonDynamoDBFullAccess + - id: AWSLambdaBasicExecutionRole + +notifiers: +- name: publish-domains + subscriptions: + - type: lambda + resource: + type: functions + name: scheduled-scanner + +functions: +- name: dynamostream-to-es + handler: lambda_function.lambda_handler + memory: 128 + runtime: python2.7 + timeout: 900 + code: + path: functions/python-function + role: + name: dynamostream-to-es + type: roles + triggers: + - service: dynamodb + name: scan-data + dependencies: + - type: search_domain + name: domains-scan-data + phase: groom +- name: empty-out-table + handler: lambda_function.lambda_handler + memory: 128 + runtime: python3.6 + timeout: 300 + code: + path: functions/python-function + environment_variable: + - key: table + value: scandata + role: + name: empty-out-table + type: roles + dependencies: + - type: nosqldb + name: scan-data + - type: nosqldb + name: domain-list +- name: on-demand-scanner + handler: lambda_function.lambda_handler + memory: 128 + runtime: python3.6 + timeout: 900 + code: + path: functions/python-function + role: + name: on-demand-scanner + type: roles + dependencies: + - type: nosqldb + name: scan-data + triggers: + - service: apigateway + name: api +- name: queue-domains + handler: lambda_function.lambda_handler + memory: 128 + runtime: python3.6 + timeout: 900 + code: + path: functions/python-function + role: + name: queue-domains + type: roles + invoke_on_completion: + invocation_type: "RequestResponse" + permissions: + - basic + - dynamo + dependencies: + - type: function + name: dynamostream-to-es + - type: nosqldb + name: domain-list + - type: nosqldb + name: scan-data + - type: notifier + name: publish-domains + phase: groom +- name: scheduled-scanner + handler: lambda_function.lambda_handler + memory: 256 + runtime: python3.6 + timeout: 900 + code: + path: functions/python-function + role: + name: scheduled-scanner + type: roles + dependencies: + - type: nosqldb + name: scan-data + triggers: + - service: sns + name: publish-domains + +endpoints: +- name: api + deploy_to: production + log_requests: true + methods: + - path: "/" + type: POST + cors: "*" + responses: + - code: 200 + body: + - is_error: false + content_type: application/json + integrate_with: + name: on-demand-scanner + type: functions + integration_http_method: POST + async: true + backend_http_method: POST + passthrough_behavior: WHEN_NO_MATCH + domain_names: + - dns_record: + zone: + name: <%= domain %> + certificate: + name: "*.<%= domain %>" + +buckets: +- name: bucket + web: false + cors: + - allowed_methods: + - GET + - POST + allowed_origins: + - "*" + upload: +# - source: "code/build" + - source: "functions" + destination: "/" + +search_domains: +- name: domains-scan-data + elasticsearch_version: '7.4' + instance_count: 1 + instance_type: r5.large.elasticsearch + ebs_size: 10 + ebs_type: gp2 + access_policies: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: "*" + Action: es:ESHttp* +nosqldbs: +- name: scan-data + read_capacity: 25 + write_capacity: 25 + attributes: + - name: domain + type: S + primary_partition: true + - name: last_scanned_date + type: S + primary_sort: true + stream: NEW_IMAGE +- name: domain-list + read_capacity: 100 + write_capacity: 1 + attributes: + - name: business_owner + type: S + primary_sort: true + - name: domain + type: S + primary_partition: true + populate: + - business_owner: TetraTech + staff_division: eGT + operational_division: eGTLabs + domain: egt-labs.com From 7fd0fce2e61dd7e0f6a38733a88b8e1d71ff7b93 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 3 Sep 2020 01:55:51 -0400 Subject: [PATCH 088/107] AWS::Endpoint: catch misconfigured cloudwatch log write role at parse time, so nobody else has to waste an afternoon on that --- modules/mu/providers/aws/endpoint.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index da5fdd662..fc339bec6 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -882,6 +882,20 @@ def self.validateConfig(endpoint, configurator) MU::Config.addDependency(endpoint, endpoint["access_logs"]["name"], "log") end + if endpoint['access_logs'] + resp = MU::Cloud::AWS.apig(credentials: endpoint['credentials'], region: endpoint['region']).get_account + if !resp.cloudwatch_role_arn + MU.log "Endpoint '#{endpoint['name']}' is configured to use CloudWatch Logs, but the account-wide API Gateway log role is not configured", MU::ERR, details: "https://aws.amazon.com/premiumsupport/knowledge-center/api-gateway-cloudwatch-logs/" + ok = false + else + roles = MU::Cloud::AWS::Role.find(cloud_id: resp.cloudwatch_role_arn, credentials: endpoint['credentials'], region: endpoint['region']) + if roles.empty? + MU.log "Endpoint '#{endpoint['name']}' is configured to use CloudWatch Logs, but the configured account-wide API Gateway log role does not exist", MU::ERR, details: resp.cloudwatch_role_arn + ok = false + end + end + end + if endpoint['domain_names'] endpoint['domain_names'].each { |dom| if dom['certificate'] From 1c60a402c8f2b64b820bc99a287d957265d51070 Mon Sep 17 00:00:00 2001 From: John Stange Date: Thu, 3 Sep 2020 17:46:43 -0400 Subject: [PATCH 089/107] AWS::Endpoint: purge the AWS-created CloudWatch Log Group(s) associated with API Gateways that we're deleting --- modules/mu/providers/aws/endpoint.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/mu/providers/aws/endpoint.rb b/modules/mu/providers/aws/endpoint.rb index fc339bec6..50bbfe2ee 100644 --- a/modules/mu/providers/aws/endpoint.rb +++ b/modules/mu/providers/aws/endpoint.rb @@ -416,6 +416,15 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, regi resp.items.each { |api| # The stupid things don't have tags if api.description == deploy_id + logs = MU::Cloud.resourceClass("AWS", "Log").find(region: region, credentials: credentials) + logs.each_pair { |log_id, log_desc| + if log_id =~ /^API-Gateway-Execution-Logs_#{api.id}\// + MU.log "Deleting CloudWatch Log Group #{log_id}" + if !noop + MU::Cloud::AWS.cloudwatchlogs(region: region, credentials: credentials).delete_log_group(log_group_name: log_id) + end + end + } MU.log "Deleting API Gateway #{api.name} (#{api.id})" if !noop MU::Cloud::AWS.apig(region: region, credentials: credentials).delete_rest_api( From 2bd142ae9314a983a200e71b4005699d7bd5694e Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 4 Sep 2020 04:25:31 -0400 Subject: [PATCH 090/107] AWS::SearchDomain: add BoK parameter to expose access policies in a non-raw way --- modules/mu/providers/aws/role.rb | 2 +- modules/mu/providers/aws/search_domain.rb | 28 +++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/modules/mu/providers/aws/role.rb b/modules/mu/providers/aws/role.rb index ea64961ff..6d02c468c 100644 --- a/modules/mu/providers/aws/role.rb +++ b/modules/mu/providers/aws/role.rb @@ -1093,7 +1093,7 @@ def self.validateConfig(role, _configurator) role['policies'].each { |policy| policy['targets'].each { |target| if target['type'] - MU::Config.addDependency(role, target['identifier'], target['type']) + MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true) end } } diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index eca94d23b..7d3c05286 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -305,6 +305,8 @@ def self.schema(_config) ).elasticsearch_instance_types end + polschema = MU::Config::Role.schema["properties"]["policies"]["items"] + polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema["items"]) schema = { "name" => { @@ -326,9 +328,10 @@ def self.schema(_config) "default" => 0, "description" => "Separate, dedicated master node(s), over and above the search instances specified in instance_count." }, + "policy" => polschema, "access_policies" => { "type" => "object", - "description" => "An IAM policy document for access to ElasticSearch. Our parser expects this to be defined inline like the rest of your YAML/JSON Basket of Kittens, not as raw JSON. For guidance on ElasticSearch IAM capabilities, see: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html" + "description" => "An IAM policy document for access to ElasticSearch (see {policies} for setting complex access policies with runtime dependencies). Our parser expects this to be defined inline like the rest of your YAML/JSON Basket of Kittens, not as raw JSON. For guidance on ElasticSearch IAM capabilities, see: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html" }, "master_instance_type" => { "type" => "string", @@ -599,9 +602,26 @@ def genParams(ext = nil) params[:snapshot_options][:automated_snapshot_start_hour] = @config['snapshot_hour'] end - if @config['access_policies'] and ext - # TODO check against ext.access_policies.options - params[:access_policies] = JSON.generate(@config['access_policies']) + if ext + policy = nil + + if @config['access_policies'] + # TODO check against ext.access_policy.options + policy = @config['access_policies'] + end + + if @config['policy'] + parsed = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument([@config['policy']], deploy_obj: @deploy, bucket_style: true).first.values.first + + if policy and policy["Statement"] + policy["Statement"].concat(parsed["Statement"]) + else + policy = parsed + end + end + + params[:access_policies] = JSON.generate(policy) if policy +MU.log "ES access policies would be:", MU::WARN, details: policy end if @config['slow_logs'] From 8bdbf2465c8ba2a8786cdfcae7948c2d81757a88 Mon Sep 17 00:00:00 2001 From: John Stange Date: Fri, 4 Sep 2020 16:45:13 -0400 Subject: [PATCH 091/107] AWS::SearchDomain: meld policies and access_policies into a single policy with multiple statements; add magic to sub #SELF with the domain id in paths so we can target the default index --- modules/mu/providers/aws/search_domain.rb | 54 +++++++++++++++++------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 7d3c05286..963e19d7a 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -79,7 +79,7 @@ def cloud_desc(use_cache: true) # Canonical Amazon Resource Number for this resource # @return [String] def arn - cloud_desc.arn + cloud_desc.arn.dup end # Return the metadata for this SearchDomain rule @@ -305,8 +305,8 @@ def self.schema(_config) ).elasticsearch_instance_types end - polschema = MU::Config::Role.schema["properties"]["policies"]["items"] - polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema["items"]) + polschema = MU::Config::Role.schema["properties"]["policies"] + polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema) schema = { "name" => { @@ -328,7 +328,7 @@ def self.schema(_config) "default" => 0, "description" => "Separate, dedicated master node(s), over and above the search instances specified in instance_count." }, - "policy" => polschema, + "policies" => polschema, "access_policies" => { "type" => "object", "description" => "An IAM policy document for access to ElasticSearch (see {policies} for setting complex access policies with runtime dependencies). Our parser expects this to be defined inline like the rest of your YAML/JSON Basket of Kittens, not as raw JSON. For guidance on ElasticSearch IAM capabilities, see: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html" @@ -603,25 +603,51 @@ def genParams(ext = nil) end if ext + # Despite being called access_policies, this parameter actually + # only accepts one policy. So, we'll munge everything we have + # together into one policy with multiple Statements. policy = nil + # TODO check against ext.access_policy.options if @config['access_policies'] - # TODO check against ext.access_policy.options policy = @config['access_policies'] + # ensure the "Statement" key is cased in a predictable way + statement_key = nil + policy.each_pair { |k, v| + if k.downcase == "statement" and k != "Statement" + statement_key = k + break + end + } + if statement_key + policy["Statement"] = policy.delete(statement_key) + end + if !policy["Statement"].is_a?(Array) + policy["Statement"] = [policy["Statement"]] + end end - if @config['policy'] - parsed = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument([@config['policy']], deploy_obj: @deploy, bucket_style: true).first.values.first + if @config['policies'] + @config['policies'].each { |p| + p['targets'].each { |t| + if t['path'] + t['path'].gsub!(/#SELF/, @cloud_id) + end + } + parsed = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument([p], deploy_obj: @deploy, bucket_style: true).first.values.first + pp parsed - if policy and policy["Statement"] - policy["Statement"].concat(parsed["Statement"]) - else - policy = parsed - end + if policy and policy["Statement"] + policy["Statement"].concat(parsed["Statement"]) + else + policy = parsed + end + } end - params[:access_policies] = JSON.generate(policy) if policy -MU.log "ES access policies would be:", MU::WARN, details: policy + if policy + params[:access_policies] = JSON.generate(policy) + end end if @config['slow_logs'] From 17714078104ff978141f3e5e890954f448ba546b Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 8 Sep 2020 01:58:51 -0400 Subject: [PATCH 092/107] Function: do grotesque things to make sure these get a final groom so their environment variables are in line with reality --- modules/mu/deploy.rb | 13 +++++++++++++ modules/mu/providers/aws/function.rb | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/modules/mu/deploy.rb b/modules/mu/deploy.rb index 694dc4474..27e385ab6 100644 --- a/modules/mu/deploy.rb +++ b/modules/mu/deploy.rb @@ -312,6 +312,17 @@ def run @mommacat.save! + # XXX Functions have a special behavior where we re-invoke their groom + # methods one more time at the end, so we can guarantee their + # environments are fully populated with all sibling resource idents + # regardless of dependency order. This is, obviously, a disgusting + # hack, and we should revisit our dependency language in the next big + # release. + if !@main_config["functions"].nil? and + @main_config["functions"].size > 0 + createResources(@main_config["functions"], "groom") + end + rescue StandardError => e MU.log e.class.name, MU::ERR, details: caller @@ -733,7 +744,9 @@ def createResources(services, mode="create") sleep 10+Random.rand(20) retry end + end + end end #class diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index 1d8bea950..a2fa33b3b 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -152,13 +152,16 @@ def addTrigger(calling_arn, calling_service, calling_name) function_name: @mu_name, principal: "#{calling_service}.amazonaws.com", source_arn: calling_arn, - statement_id: "#{calling_service}-#{calling_name}", + statement_id: "#{calling_service}-#{calling_name.gsub(/[^a-zA-Z0-9-_]/, '_')}", } begin # XXX There doesn't seem to be an API call to list or view existing # permissions, wtaf. This means we can't intelligently guard this. MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger) + rescue Aws::Lambda::Errors::ValidationException => e + MU.log e.message+" (calling_arn: #{calling_arn}, calling_service: #{calling_service}, calling_name: #{calling_name})", MU::ERR, details: trigger + raise e rescue Aws::Lambda::Errors::ResourceConflictException => e if e.message.match(/already exists/) MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).remove_permission( From 94f3ad150ab4a6cb67946bcc85b264b9ec67c89f Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 8 Sep 2020 01:59:53 -0400 Subject: [PATCH 093/107] clean up a regex --- modules/mu/providers/aws/function.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mu/providers/aws/function.rb b/modules/mu/providers/aws/function.rb index a2fa33b3b..7bd13f3af 100644 --- a/modules/mu/providers/aws/function.rb +++ b/modules/mu/providers/aws/function.rb @@ -152,7 +152,7 @@ def addTrigger(calling_arn, calling_service, calling_name) function_name: @mu_name, principal: "#{calling_service}.amazonaws.com", source_arn: calling_arn, - statement_id: "#{calling_service}-#{calling_name.gsub(/[^a-zA-Z0-9-_]/, '_')}", + statement_id: "#{calling_service}-#{calling_name.gsub(/[^a-z0-9\-_]/i, '_')}", } begin From fbb766b78350b6cda0fbca2410f4b8286ee4e7e8 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Tue, 8 Sep 2020 12:57:09 -0400 Subject: [PATCH 094/107] AWS::SearchDomain: guard unavailable cloud_descs in #arn --- modules/mu/providers/aws/search_domain.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 963e19d7a..6ed4497bc 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -79,6 +79,7 @@ def cloud_desc(use_cache: true) # Canonical Amazon Resource Number for this resource # @return [String] def arn + return nil if !cloud_desc cloud_desc.arn.dup end From 27744c5bee02b29213b90b2eedfe1754f0d03ff5 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Tue, 8 Sep 2020 15:38:21 -0400 Subject: [PATCH 095/107] AWS::SearchDomain: downcase that hacky little #SELF resource name substitution in access policies --- modules/mu/providers/aws/search_domain.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index 6ed4497bc..ed3926bea 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -632,11 +632,10 @@ def genParams(ext = nil) @config['policies'].each { |p| p['targets'].each { |t| if t['path'] - t['path'].gsub!(/#SELF/, @cloud_id) + t['path'].gsub!(/#SELF/, @mu_name.downcase) end } parsed = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument([p], deploy_obj: @deploy, bucket_style: true).first.values.first - pp parsed if policy and policy["Statement"] policy["Statement"].concat(parsed["Statement"]) From 9e360420e9091a0af64a54cf521311f44b7208aa Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 15 Sep 2020 19:03:04 -0400 Subject: [PATCH 096/107] test, linux bootstrap, and base image updates --- modules/mu/defaults/AWS.yaml | 34 ++++++++++----------- modules/mu/providers/aws/userdata/linux.erb | 9 +++--- modules/tests/centos6.yaml | 4 +++ modules/tests/centos7.yaml | 4 +++ modules/tests/ecs.yaml | 4 +-- modules/tests/eks.yaml | 2 +- modules/tests/server-with-scrub-muisms.yaml | 2 +- modules/tests/super_complex_bok.yml | 2 +- modules/tests/super_simple_bok.yml | 2 +- 9 files changed, 36 insertions(+), 27 deletions(-) diff --git a/modules/mu/defaults/AWS.yaml b/modules/mu/defaults/AWS.yaml index e0cb2316a..1ab11b977 100644 --- a/modules/mu/defaults/AWS.yaml +++ b/modules/mu/defaults/AWS.yaml @@ -34,23 +34,23 @@ centos6: &3 us-west-1: ami-0b05ec54412b9f8b0 us-west-2: ami-0447e036b102b2ca0 centos7: - us-east-1: ami-067256ca1497c924d - ap-northeast-1: ami-07c1e51354fdfd362 - ap-northeast-2: ami-042b761c93d6df2f1 - ap-south-1: ami-02e879f52322e7c98 - ap-southeast-1: ami-0487e9f84d0ffde89 - ap-southeast-2: ami-0e854dab39fd6a427 - ca-central-1: ami-05a27d311b585a70b - eu-central-1: ami-0e396d00c787b4f47 - eu-north-1: ami-087763a2ba60b2bfe - eu-west-1: ami-04e3bd9335a14e635 - eu-west-2: ami-0efd34a8d1fc2b104 - eu-west-3: ami-08d0bcbc780448cf8 - sa-east-1: ami-0284f4a0968263cf0 - us-east-2: ami-0292786917d1e3015 - us-west-1: ami-0ba622529dcdff2bb - us-west-2: ami-079a309ca6261d7f6 -ubuntu16: &2 + us-east-1: ami-06b883b9ecc1074c1 + us-east-2: ami-0f78094f535fcda86 + ap-northeast-1: ami-0e26532edad7a0883 + ap-northeast-2: ami-002234be57e1ce95d + ap-south-1: ami-0757c60f4d83634e1 + ap-southeast-1: ami-0b18b48ffe1f9d1fd + ap-southeast-2: ami-0f42dde537263530c + ca-central-1: ami-0f628f82dffa915fb + eu-central-1: ami-0cca69b51ca8f81c0 + eu-north-1: ami-058b02745349f80e1 + eu-west-1: ami-0bc0ed56941a45c66 + eu-west-2: ami-0c472cd66ed768e9c + eu-west-3: ami-0faa3c3363365dce7 + sa-east-1: ami-097e704125faa6711 + us-west-1: ami-0f6d2a56903c96c3c + us-west-2: ami-0806af0e25c0e52d1 +ubuntu16: &3 us-east-1: ami-bcdc16c6 us-west-1: ami-1b17257b us-west-2: ami-19e92861 diff --git a/modules/mu/providers/aws/userdata/linux.erb b/modules/mu/providers/aws/userdata/linux.erb index 7247c4a63..225054a58 100644 --- a/modules/mu/providers/aws/userdata/linux.erb +++ b/modules/mu/providers/aws/userdata/linux.erb @@ -42,7 +42,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then <% if !$mu.skipApplyUpdates %> set +e if [ ! -f /.mu-installer-ran-updates ];then - service ssh stop + echo "Applying package updates" > /etc/nologin apt-get --fix-missing -y upgrade touch /.mu-installer-ran-updates if [ $? -eq 0 ] @@ -58,7 +58,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then else echo "FAILED PACKAGE UPDATE" >&2 fi - service ssh start + rm -f /etc/nologin fi <% end %> elif [ -x /usr/bin/yum ];then @@ -94,7 +94,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then <% if !$mu.skipApplyUpdates %> set +e if [ ! -f /.mu-installer-ran-updates ];then - service sshd stop + echo "Applying package updates" > /etc/nologin kernel_update=`yum list updates | grep kernel` yum -y update touch /.mu-installer-ran-updates @@ -108,7 +108,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then else echo "FAILED PACKAGE UPDATE" >&2 fi - service sshd start + rm -f /etc/nologin fi <% end %> fi @@ -116,6 +116,7 @@ else /bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata" touch /.mu-installer-ran-updates fi +rm -f /etc/nologin AWSCLI='command -v aws' PIP='command -v pip' diff --git a/modules/tests/centos6.yaml b/modules/tests/centos6.yaml index 3f6f544e2..a1f28c898 100644 --- a/modules/tests/centos6.yaml +++ b/modules/tests/centos6.yaml @@ -1,8 +1,12 @@ # groomers: Chef --- appname: smoketest +vpcs: +- name: wrapper servers: - name: centos6 + vpc: + name: wrapper platform: centos6 size: m3.medium run_list: diff --git a/modules/tests/centos7.yaml b/modules/tests/centos7.yaml index 4f16b6046..958488ffb 100644 --- a/modules/tests/centos7.yaml +++ b/modules/tests/centos7.yaml @@ -1,9 +1,13 @@ # groomers: Chef --- appname: smoketest +vpcs: +- name: wrapper servers: - name: centos7 platform: centos7 + vpc: + name: wrapper size: m3.medium run_list: - recipe[mu-tools::apply_security] diff --git a/modules/tests/ecs.yaml b/modules/tests/ecs.yaml index 941c16df8..05b9b0b79 100644 --- a/modules/tests/ecs.yaml +++ b/modules/tests/ecs.yaml @@ -7,7 +7,7 @@ vpcs: container_clusters: - name: ecsplain flavor: ECS - instance_type: t2.medium + instance_type: t3.medium vpc: name: ecs containers: @@ -15,7 +15,7 @@ container_clusters: image: "nginx:1.8" - name: ecsfargate flavor: Fargate - instance_type: t2.medium + instance_type: t3.medium vpc: name: ecs containers: diff --git a/modules/tests/eks.yaml b/modules/tests/eks.yaml index 06c3e37ad..ac61afe81 100644 --- a/modules/tests/eks.yaml +++ b/modules/tests/eks.yaml @@ -10,7 +10,7 @@ container_clusters: vpc: vpc_name: eksvpc instance_count: 3 - instance_type: t2.medium + instance_type: t3.medium kubernetes_resources: - apiVersion: apps/v1 kind: Deployment diff --git a/modules/tests/server-with-scrub-muisms.yaml b/modules/tests/server-with-scrub-muisms.yaml index 71aad5724..bc186583a 100644 --- a/modules/tests/server-with-scrub-muisms.yaml +++ b/modules/tests/server-with-scrub-muisms.yaml @@ -13,7 +13,7 @@ servers: ssh_user: centos cloud: <%= cloud %> <% if cloud == "AWS" %> - size: t2.medium + size: t3.medium <% elsif cloud == "Azure" %> size: Standard_DS1_v2 <% elsif cloud == "Google" %> diff --git a/modules/tests/super_complex_bok.yml b/modules/tests/super_complex_bok.yml index 14e8f0364..f817fe3c3 100644 --- a/modules/tests/super_complex_bok.yml +++ b/modules/tests/super_complex_bok.yml @@ -9,7 +9,7 @@ parameters: - name: vpc_name required: false - name: instance_type - default: t2.medium + default: t3.medium - name: db_size default: db.t2.small - name: vpc_name diff --git a/modules/tests/super_simple_bok.yml b/modules/tests/super_simple_bok.yml index 069678d31..8a8aa93da 100644 --- a/modules/tests/super_simple_bok.yml +++ b/modules/tests/super_simple_bok.yml @@ -8,7 +8,7 @@ parameters: - name: vpc_name required: false - name: instance_type - default: t2.medium + default: t3.medium - name: db_size default: db.t2.small - name: vpc_name From 07e44cf0072ab5c98fd8566f558f6e24ebc53b0d Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 15 Sep 2020 19:03:49 -0400 Subject: [PATCH 097/107] more base image updates --- modules/mu/defaults/AWS.yaml | 82 ++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/modules/mu/defaults/AWS.yaml b/modules/mu/defaults/AWS.yaml index 1ab11b977..3e918137b 100644 --- a/modules/mu/defaults/AWS.yaml +++ b/modules/mu/defaults/AWS.yaml @@ -1,5 +1,5 @@ --- -rhel71: &4 +rhel71: &5 us-east-1: ami-0f05fce24aa75ba9f ap-northeast-1: ami-0c0ec19eb19055763 ap-northeast-2: ami-0717ac5c67c99f745 @@ -16,40 +16,40 @@ rhel71: &4 us-east-2: ami-02f6682c7816b3cfc us-west-1: ami-04898e596c06e802b us-west-2: ami-02db5457189a8a8c2 -centos6: &3 - us-east-1: ami-0ccdc671f12147a1d - us-east-2: ami-00d0e8bc2f05ab949 - ap-northeast-1: ami-0726801ceef87f5f8 - ap-northeast-2: ami-05fa4afc4a0493b0a - ap-south-1: ami-0d6e4f3b6592b3139 - ap-southeast-1: ami-0c988e3dc80b14653 - ap-southeast-2: ami-02ac856fd094675ef - ca-central-1: ami-0ce7e343953af2292 - eu-central-1: ami-0ce8317423cea27b8 - eu-north-1: ami-0a923b493d5fc9743 - eu-west-1: ami-06e0f02328921c865 - eu-west-2: ami-07ae118c8814df140 - eu-west-3: ami-03c1017cd1ccc6e9d - sa-east-1: ami-05212ae133b9c3ba1 - us-west-1: ami-0b05ec54412b9f8b0 - us-west-2: ami-0447e036b102b2ca0 +centos6: &4 + us-east-1: ami-009723c5c7f8fbc75 + us-east-2: ami-0781f11395714cd39 + ap-northeast-1: ami-07fa5a8795da2b6bc + ap-northeast-2: ami-0219f0a7c979ff63f + ap-south-1: ami-0f24817242c401740 + ap-southeast-1: ami-042ef2e0643e8e207 + ap-southeast-2: ami-09fc51de648afa168 + ca-central-1: ami-0dc643db74edc5aa5 + eu-central-1: ami-0628759cb297569d5 + eu-north-1: ami-0aed023791f886315 + eu-west-1: ami-0f87f0f252ff03622 + eu-west-2: ami-00abb555d5a460afe + eu-west-3: ami-0ccd93d454c2418a2 + sa-east-1: ami-01e10ea6ea72534ae + us-west-1: ami-01fee56b9ee690ffe + us-west-2: ami-08bcdb944f185e2a8 centos7: - us-east-1: ami-06b883b9ecc1074c1 - us-east-2: ami-0f78094f535fcda86 - ap-northeast-1: ami-0e26532edad7a0883 - ap-northeast-2: ami-002234be57e1ce95d - ap-south-1: ami-0757c60f4d83634e1 - ap-southeast-1: ami-0b18b48ffe1f9d1fd - ap-southeast-2: ami-0f42dde537263530c - ca-central-1: ami-0f628f82dffa915fb - eu-central-1: ami-0cca69b51ca8f81c0 - eu-north-1: ami-058b02745349f80e1 - eu-west-1: ami-0bc0ed56941a45c66 - eu-west-2: ami-0c472cd66ed768e9c - eu-west-3: ami-0faa3c3363365dce7 - sa-east-1: ami-097e704125faa6711 - us-west-1: ami-0f6d2a56903c96c3c - us-west-2: ami-0806af0e25c0e52d1 + us-east-1: ami-067256ca1497c924d + ap-northeast-1: ami-07c1e51354fdfd362 + ap-northeast-2: ami-042b761c93d6df2f1 + ap-south-1: ami-02e879f52322e7c98 + ap-southeast-1: ami-0487e9f84d0ffde89 + ap-southeast-2: ami-0e854dab39fd6a427 + ca-central-1: ami-05a27d311b585a70b + eu-central-1: ami-0e396d00c787b4f47 + eu-north-1: ami-087763a2ba60b2bfe + eu-west-1: ami-04e3bd9335a14e635 + eu-west-2: ami-0efd34a8d1fc2b104 + eu-west-3: ami-08d0bcbc780448cf8 + sa-east-1: ami-0284f4a0968263cf0 + us-east-2: ami-0292786917d1e3015 + us-west-1: ami-0ba622529dcdff2bb + us-west-2: ami-079a309ca6261d7f6 ubuntu16: &3 us-east-1: ami-bcdc16c6 us-west-1: ami-1b17257b @@ -89,7 +89,7 @@ win2k12r2: &1 ap-northeast-2: ami-0368c224de1d20502 ap-southeast-1: ami-028ef74e1edc3943a ap-southeast-2: ami-09e03eab1b1bc151b -win2k16: &5 +win2k16: &2 us-east-1: ami-02801a2c8dcbfb883 us-east-2: ami-0ca4f779a2a58a7ea ca-central-1: ami-05d3854d9d6e9bcc5 @@ -137,9 +137,9 @@ amazon: ap-southeast-1: ami-b953f2da ap-southeast-2: ami-db704cb8 win2k12: *1 -windows: *5 -ubuntu: *2 -centos: *3 -rhel7: *4 -rhel: *4 -linux: *3 +windows: *2 +ubuntu: *3 +centos: *4 +rhel7: *5 +rhel: *5 +linux: *4 From 06a1eaca4500f68cb663e24875045c6584c0323e Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Tue, 15 Sep 2020 19:09:24 -0400 Subject: [PATCH 098/107] gem sync bidness --- modules/Gemfile.lock | 63 +++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index adbbff95c..56b6eafba 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -44,7 +44,7 @@ PATH GEM remote: https://rubygems.org/ specs: - activesupport (6.0.3.2) + activesupport (6.0.3.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -54,7 +54,7 @@ GEM public_suffix (>= 2.0.2, < 4.0) ast (2.4.1) aws-eventstream (1.1.0) - aws-sdk-core (2.11.578) + aws-sdk-core (2.11.586) aws-sigv4 (~> 1.0) jmespath (~> 1.0) aws-sigv4 (1.2.2) @@ -120,7 +120,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_graph_rbac (0.17.2) ms_rest_azure (~> 0.12.0) - azure_key_vault (0.17.4) + azure_key_vault (0.18.0) ms_rest_azure (~> 0.12.0) azure_mgmt_adhybridhealth_service (0.17.1) ms_rest_azure (~> 0.12.0) @@ -132,7 +132,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_api_management (0.19.1) ms_rest_azure (~> 0.12.0) - azure_mgmt_appconfiguration (0.17.3) + azure_mgmt_appconfiguration (0.17.4) ms_rest_azure (~> 0.12.0) azure_mgmt_attestation (0.17.1) ms_rest_azure (~> 0.12.0) @@ -166,7 +166,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_container_registry (0.18.4) ms_rest_azure (~> 0.12.0) - azure_mgmt_container_service (0.20.3) + azure_mgmt_container_service (0.20.4) ms_rest_azure (~> 0.12.0) azure_mgmt_cosmosdb (0.22.0) ms_rest_azure (~> 0.12.0) @@ -174,7 +174,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_customer_insights (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_data_factory (0.18.4) + azure_mgmt_data_factory (0.18.5) ms_rest_azure (~> 0.12.0) azure_mgmt_data_migration (0.18.1) ms_rest_azure (~> 0.12.0) @@ -206,6 +206,8 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_hdinsight (0.18.2) ms_rest_azure (~> 0.12.0) + azure_mgmt_hybrid_compute (0.17.0) + ms_rest_azure (~> 0.12.0) azure_mgmt_import_export (0.17.1) ms_rest_azure (~> 0.12.0) azure_mgmt_iot_central (0.19.2) @@ -234,7 +236,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_managed_applications (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_mariadb (0.17.3) + azure_mgmt_mariadb (0.17.4) ms_rest_azure (~> 0.12.0) azure_mgmt_marketplace_ordering (0.17.5) ms_rest_azure (~> 0.12.0) @@ -248,15 +250,15 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_msi (0.17.2) ms_rest_azure (~> 0.12.0) - azure_mgmt_mysql (0.17.1) + azure_mgmt_mysql (0.17.2) ms_rest_azure (~> 0.12.0) - azure_mgmt_netapp (0.20.1) + azure_mgmt_netapp (0.20.2) ms_rest_azure (~> 0.12.0) - azure_mgmt_network (0.24.0) + azure_mgmt_network (0.24.1) ms_rest_azure (~> 0.12.0) azure_mgmt_notification_hubs (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_operational_insights (0.19.1) + azure_mgmt_operational_insights (0.19.2) ms_rest_azure (~> 0.12.0) azure_mgmt_operations_management (0.17.1) ms_rest_azure (~> 0.12.0) @@ -324,7 +326,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_stream_analytics (0.17.3) ms_rest_azure (~> 0.12.0) - azure_mgmt_subscriptions (0.18.4) + azure_mgmt_subscriptions (0.18.5) ms_rest_azure (~> 0.12.0) azure_mgmt_support (0.17.1) ms_rest_azure (~> 0.12.0) @@ -338,7 +340,7 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_web (0.17.6) ms_rest_azure (~> 0.12.0) - azure_sdk (0.62.0) + azure_sdk (0.63.0) azure-storage (~> 0.14.0.preview) azure_cognitiveservices_anomalydetector (~> 0.17.1) azure_cognitiveservices_autosuggest (~> 0.17.2) @@ -366,13 +368,13 @@ GEM azure_cognitiveservices_websearch (~> 0.18.2) azure_event_grid (~> 0.18.1) azure_graph_rbac (~> 0.17.2) - azure_key_vault (~> 0.17.4) + azure_key_vault (~> 0.18.0) azure_mgmt_adhybridhealth_service (~> 0.17.1) azure_mgmt_advisor (~> 0.17.2) azure_mgmt_alerts_management (~> 0.17.1) azure_mgmt_analysis_services (~> 0.17.3) azure_mgmt_api_management (~> 0.19.1) - azure_mgmt_appconfiguration (~> 0.17.3) + azure_mgmt_appconfiguration (~> 0.17.4) azure_mgmt_attestation (~> 0.17.1) azure_mgmt_authorization (~> 0.18.5) azure_mgmt_automation (~> 0.17.3) @@ -389,11 +391,11 @@ GEM azure_mgmt_consumption (~> 0.18.1) azure_mgmt_container_instance (~> 0.17.6) azure_mgmt_container_registry (~> 0.18.4) - azure_mgmt_container_service (~> 0.20.3) + azure_mgmt_container_service (~> 0.20.4) azure_mgmt_cosmosdb (~> 0.22.0) azure_mgmt_cost_management (~> 0.17.1) azure_mgmt_customer_insights (~> 0.17.3) - azure_mgmt_data_factory (~> 0.18.4) + azure_mgmt_data_factory (~> 0.18.5) azure_mgmt_data_migration (~> 0.18.1) azure_mgmt_databox (~> 0.17.1) azure_mgmt_datalake_analytics (~> 0.17.3) @@ -409,6 +411,7 @@ GEM azure_mgmt_features (~> 0.17.4) azure_mgmt_hanaonazure (~> 0.18.1) azure_mgmt_hdinsight (~> 0.18.2) + azure_mgmt_hybrid_compute (~> 0.17.0) azure_mgmt_import_export (~> 0.17.1) azure_mgmt_iot_central (~> 0.19.2) azure_mgmt_iot_hub (~> 0.17.4) @@ -423,18 +426,18 @@ GEM azure_mgmt_machine_learning_services (~> 0.17.3) azure_mgmt_maintenance (~> 0.17.1) azure_mgmt_managed_applications (~> 0.17.3) - azure_mgmt_mariadb (~> 0.17.3) + azure_mgmt_mariadb (~> 0.17.4) azure_mgmt_marketplace_ordering (~> 0.17.5) azure_mgmt_media_services (~> 0.20.1) azure_mgmt_migrate (~> 0.17.1) azure_mgmt_mixedreality (~> 0.17.3) azure_mgmt_monitor (~> 0.17.6) azure_mgmt_msi (~> 0.17.2) - azure_mgmt_mysql (~> 0.17.1) - azure_mgmt_netapp (~> 0.20.1) - azure_mgmt_network (~> 0.24.0) + azure_mgmt_mysql (~> 0.17.2) + azure_mgmt_netapp (~> 0.20.2) + azure_mgmt_network (~> 0.24.1) azure_mgmt_notification_hubs (~> 0.17.3) - azure_mgmt_operational_insights (~> 0.19.1) + azure_mgmt_operational_insights (~> 0.19.2) azure_mgmt_operations_management (~> 0.17.1) azure_mgmt_peering (~> 0.17.1) azure_mgmt_policy (~> 0.17.9) @@ -468,7 +471,7 @@ GEM azure_mgmt_storagecache (~> 0.18.1) azure_mgmt_storagesync (~> 0.18.1) azure_mgmt_stream_analytics (~> 0.17.3) - azure_mgmt_subscriptions (~> 0.18.4) + azure_mgmt_subscriptions (~> 0.18.5) azure_mgmt_support (~> 0.17.1) azure_mgmt_synapse (~> 0.17.2) azure_mgmt_time_series_insights (~> 0.17.1) @@ -575,7 +578,7 @@ GEM cucumber-tag-expressions (~> 2.0, >= 2.0.4) cucumber-gherkin (15.0.2) cucumber-messages (~> 13.0, >= 13.0.1) - cucumber-messages (13.0.1) + cucumber-messages (13.1.0) protobuf-cucumber (~> 3.10, >= 3.10.8) cucumber-tag-expressions (2.0.4) daemons (1.3.1) @@ -756,16 +759,16 @@ GEM rspec_junit_formatter (0.2.3) builder (< 4) rspec-core (>= 2, < 4, != 2.12.0) - rubocop (0.90.0) + rubocop (0.91.0) parallel (~> 1.10) parser (>= 2.7.1.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.7) rexml - rubocop-ast (>= 0.3.0, < 1.0) + rubocop-ast (>= 0.4.0, < 1.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (0.3.0) + rubocop-ast (0.4.0) parser (>= 2.7.1.4) ruby-graphviz (1.2.5) rexml @@ -809,7 +812,7 @@ GEM thread_safe (0.3.6) timeliness (0.3.10) tomlrb (1.3.0) - treetop (1.6.10) + treetop (1.6.11) polyglot (~> 0.3) tzinfo (1.2.7) thread_safe (~> 0.1) @@ -828,11 +831,11 @@ GEM logging (>= 1.6.1, < 3.0) nori (~> 2.0) rubyntlm (~> 0.6.0, >= 0.6.1) - winrm-elevated (1.2.1) + winrm-elevated (1.2.2) erubi (~> 1.8) winrm (~> 2.0) winrm-fs (~> 1.0) - winrm-fs (1.3.4) + winrm-fs (1.3.5) erubi (~> 1.8) logging (>= 1.6.1, < 3.0) rubyzip (~> 2.0) From 6d28539067b022e5839e34167f2b1427709e1a45 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Wed, 16 Sep 2020 13:42:43 -0400 Subject: [PATCH 099/107] AWS::ContainerCluster: EKS image lookups are really slow, work around as best we can; Cleanup: dump status every couple minutes for really long-running cleanups, fix failure reporting --- modules/mu/cleanup.rb | 66 +++++++++++------ modules/mu/providers/aws/container_cluster.rb | 70 ++++++++++++------- 2 files changed, 86 insertions(+), 50 deletions(-) diff --git a/modules/mu/cleanup.rb b/modules/mu/cleanup.rb index 84505a491..9d626e709 100644 --- a/modules/mu/cleanup.rb +++ b/modules/mu/cleanup.rb @@ -218,51 +218,69 @@ def self.cleanRegion(cloud, credset, region, global_vs_region_semaphore, global_ cloudclass = MU::Cloud.cloudClass(cloud) habitatclass = MU::Cloud.resourceClass(cloud, "Habitat") - projects = [] - if habitats - projects = habitats - else + if !habitats + habitats = [] if $MU_CFG and $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase][credset] and $MU_CFG[cloud.downcase][credset]["project"] # XXX GCP credential schema needs an array for projects - projects << $MU_CFG[cloud.downcase][credset]["project"] + habitats << $MU_CFG[cloud.downcase][credset]["project"] end begin - projects.concat(cloudclass.listHabitats(credset, use_cache: false)) + habitats.concat(cloudclass.listHabitats(credset, use_cache: false)) rescue NoMethodError end end - if projects == [] - projects << "" # dummy + if habitats == [] + habitats << "" # dummy MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{region}", MU::NOTICE end - projects.uniq! + habitats.uniq! # We do these in an order that unrolls dependent resources # sensibly, and we hit :Collection twice because AWS # CloudFormation sometimes fails internally. - projectthreads = [] - projects.each { |project| - if habitats and !habitats.empty? and project != "" - next if !habitats.include?(project) + habitat_threads = [] + habitats.each { |habitat| + if habitats and !habitats.empty? and habitat != "" + next if !habitats.include?(habitat) end - if @habitatsused and !@habitatsused.empty? and project != "" - next if !@habitatsused.include?(project) + if @habitatsused and !@habitatsused.empty? and habitat != "" + next if !@habitatsused.include?(habitat) end - next if !habitatclass.isLive?(project, credset) + next if !habitatclass.isLive?(habitat, credset) - projectthreads << Thread.new { + habitat_threads << Thread.new { + Thread.current.thread_variable_set("name", "#{cloud}/#{credset}/#{habitat}/#{region}") Thread.abort_on_exception = false - if !cleanHabitat(cloud, credset, region, project, global_vs_region_semaphore, global_done) + if !cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done) had_failures = true end } # TYPES_IN_ORDER.each { |t| - } # projects.each { |project| - projectthreads.each do |t| - t.join - end + } # habitats.each { |habitat| + + last_checkin = Time.now + begin + deletia = [] + habitat_threads.each { |t| + if !t.status + t.join + deletia << t + end + } + deletia.each { |t| + habitat_threads.delete(t) + } + if (Time.now - last_checkin) > 120 + list = habitat_threads.map { |t| + t.thread_variable_get("name") + (t.thread_variable_get("type") ? "/"+t.thread_variable_get("type") : "") + } + MU.log "Waiting on #{habitat_threads.size.to_s} habitat#{habitat_threads.size > 1 ? "s" : ""} in region #{region}", MU::NOTICE, details: list + last_checkin = Time.now + end + sleep 10 if !habitat_threads.empty? + end while !habitat_threads.empty? had_failures end @@ -312,7 +330,8 @@ def self.cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphor next end } - had_failures = true + + had_failures end private_class_method :cleanHabitat @@ -323,6 +342,7 @@ def self.cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphor # @param flags [Hash]: # @param region [String]: def self.call_cleanup(type, credset, provider, flags, region) + Thread.current.thread_variable_set("type", type) if @mommacat.nil? or @mommacat.numKittens(types: [type]) > 0 if @mommacat diff --git a/modules/mu/providers/aws/container_cluster.rb b/modules/mu/providers/aws/container_cluster.rb index 7d997297e..375893f37 100644 --- a/modules/mu/providers/aws/container_cluster.rb +++ b/modules/mu/providers/aws/container_cluster.rb @@ -327,7 +327,7 @@ def notify end @@eks_versions = {} - @@eks_version_semaphore = Mutex.new + @@eks_version_semaphores = {} # Use the AWS SSM API to fetch the current version of the Amazon Linux # ECS-optimized AMI, so we can use it as a default AMI for ECS deploys. # @param flavor [String]: ECS or EKS @@ -340,24 +340,22 @@ def self.getStandardImage(flavor = "ECS", region = MU.myRegion, version: nil, gp names: ["/aws/service/#{flavor.downcase}/optimized-ami/amazon-linux/recommended"] ) else - @@eks_version_semaphore.synchronize { + @@eks_version_semaphores[region] ||= Mutex.new + + @@eks_version_semaphores[region].synchronize { if !@@eks_versions[region] @@eks_versions[region] ||= [] versions = {} - resp = nil - next_token = nil - begin - resp = MU::Cloud::AWS.ssm(region: region).get_parameters_by_path( - path: "/aws/service/#{flavor.downcase}", - recursive: true, - next_token: next_token - ) - resp.parameters.each { |p| - p.name.match(/\/aws\/service\/eks\/optimized-ami\/([^\/]+?)\//) - versions[Regexp.last_match[1]] = true - } - next_token = resp.next_token - end while !next_token.nil? + resp = MU::Cloud::AWS.ssm(region: region).get_parameters_by_path( + path: "/aws/service/#{flavor.downcase}/optimized-ami", + recursive: true, + max_results: 10 # as high as it goes, ugh + ) + + resp.parameters.each { |p| + p.name.match(/\/aws\/service\/eks\/optimized-ami\/([^\/]+?)\//) + versions[Regexp.last_match[1]] = true + } @@eks_versions[region] = versions.keys.sort { |a, b| MU.version_sort(a, b) } end } @@ -377,16 +375,31 @@ def self.getStandardImage(flavor = "ECS", region = MU.myRegion, version: nil, gp nil end + @@supported_eks_region_cache = [] + @@eks_region_semaphore = Mutex.new + # Return the list of regions where we know EKS is supported. - def self.EKSRegions(credentials = nil, region: nil) - eks_regions = [] - check_regions = region ? [region] : MU::Cloud::AWS.listRegions(credentials: credentials) - check_regions.each { |r| - ami = getStandardImage("EKS", r) - eks_regions << r if ami - } + def self.EKSRegions(credentials = nil) + @@eks_region_semaphore.synchronize { + if @@supported_eks_region_cache and !@@supported_eks_region_cache.empty? + return @@supported_eks_region_cache + end +start = Time.now + # the SSM API is painfully slow for large result sets, so thread + # these and do them in parallel + @@supported_eks_region_cache = [] + region_threads = [] + MU::Cloud::AWS.listRegions(credentials: credentials).each { |region| + region_threads << Thread.new(region) { |r| +r_start = Time.now + ami = getStandardImage("EKS", r) + @@supported_eks_region_cache << r if ami + } + } + region_threads.each { |t| t.join } - eks_regions + @@supported_eks_region_cache + } end # Does this resource type exist as a global (cloud-wide) artifact, or @@ -418,12 +431,14 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, regi end def self.purge_eks_clusters(noop: false, region: MU.curRegion, credentials: nil, deploy_id: MU.deploy_id) - return if !MU::Cloud::AWS::ContainerCluster.EKSRegions(credentials, region: region).include?(region) resp = begin MU::Cloud::AWS.eks(credentials: credentials, region: region).list_clusters rescue Aws::EKS::Errors::AccessDeniedException # EKS isn't actually live in this region, even though SSM lists # base images for it + if @@supported_eks_region_cache + @@supported_eks_region_cache.delete(region) + end return end @@ -475,6 +490,7 @@ def self.purge_eks_clusters(noop: false, region: MU.curRegion, credentials: nil, private_class_method :purge_eks_clusters def self.purge_ecs_clusters(noop: false, region: MU.curRegion, credentials: nil, deploy_id: MU.deploy_id) +start = Time.now resp = MU::Cloud::AWS.ecs(credentials: credentials, region: region).list_clusters return if !resp or !resp.cluster_arns or resp.cluster_arns.empty? @@ -1222,12 +1238,12 @@ def self.validateConfig(cluster, configurator) cluster["flavor"] = "EKS" if cluster["flavor"].match(/^Kubernetes$/i) - if cluster["flavor"] == "ECS" and cluster["kubernetes"] and !MU::Cloud::AWS.isGovCloud?(cluster["region"]) and !cluster["containers"] and MU::Cloud::AWS::ContainerCluster.EKSRegions(cluster['credentials'], region: cluster['region']).include?(cluster['region']) + if cluster["flavor"] == "ECS" and cluster["kubernetes"] and !MU::Cloud::AWS.isGovCloud?(cluster["region"]) and !cluster["containers"] and MU::Cloud::AWS::ContainerCluster.EKSRegions(cluster['credentials']).include?(cluster['region']) cluster["flavor"] = "EKS" MU.log "Setting flavor of ContainerCluster '#{cluster['name']}' to EKS ('kubernetes' stanza was specified)", MU::NOTICE end - if cluster["flavor"] == "EKS" and !MU::Cloud::AWS::ContainerCluster.EKSRegions(cluster['credentials'], region: cluster['region']).include?(cluster['region']) + if cluster["flavor"] == "EKS" and !MU::Cloud::AWS::ContainerCluster.EKSRegions(cluster['credentials']).include?(cluster['region']) MU.log "EKS is only available in some regions", MU::ERR, details: MU::Cloud::AWS::ContainerCluster.EKSRegions ok = false end From a02e632fc04099057b4a16a490e744547b59222c Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Thu, 17 Sep 2020 03:00:54 -0400 Subject: [PATCH 100/107] AWS::Group: handle missing users attribute on cloud descriptor more gracefully (did AWS change behavior?) --- modules/mu/providers/aws/group.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/mu/providers/aws/group.rb b/modules/mu/providers/aws/group.rb index 52e3131f7..b29a40943 100644 --- a/modules/mu/providers/aws/group.rb +++ b/modules/mu/providers/aws/group.rb @@ -275,14 +275,15 @@ def toKitten(**_args) MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config return nil end - - bok["name"] = cloud_desc.group.group_name - if cloud_desc.group.path != "/" - bok["path"] = cloud_desc.group.path + group_desc = cloud_desc(use_cache: false).respond_to?(:group) ? cloud_desc.group : cloud_desc + bok["name"] = group_desc.group_name + + if group_desc.path != "/" + bok["path"] = group_desc.path end - if cloud_desc.users and cloud_desc.users.size > 0 + if cloud_desc.respond_to?(:users) and cloud_desc.users and cloud_desc.users.size > 0 bok["members"] = cloud_desc.users.map { |u| u.user_name } end From ae6503dc4ceeaf3859a79894c3df9dbc840761fe Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Fri, 18 Sep 2020 03:58:59 -0400 Subject: [PATCH 101/107] AWS::ContainerCluster: a bucket of fixes for ECS --- modules/mu/mommacat.rb | 1 + modules/mu/providers/aws/container_cluster.rb | 7 ++-- modules/mu/providers/aws/role.rb | 6 +++- modules/mu/providers/aws/server_pool.rb | 34 +++++++++++++------ modules/tests/microservice_app.yaml | 2 +- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/modules/mu/mommacat.rb b/modules/mu/mommacat.rb index 66ce5fd51..7697baa55 100644 --- a/modules/mu/mommacat.rb +++ b/modules/mu/mommacat.rb @@ -811,6 +811,7 @@ def syncLitter(nodeclasses = [], triggering_node: nil, save_only: false) threads = [] update_servers.each { |sibling| + next if sibling.config.has_key?("groom") and !sibling.config["groom"] threads << Thread.new { Thread.abort_on_exception = true Thread.current.thread_variable_set("name", "sync-"+sibling.mu_name.downcase) diff --git a/modules/mu/providers/aws/container_cluster.rb b/modules/mu/providers/aws/container_cluster.rb index 375893f37..536020b53 100644 --- a/modules/mu/providers/aws/container_cluster.rb +++ b/modules/mu/providers/aws/container_cluster.rb @@ -1501,7 +1501,8 @@ def self.validateConfig(cluster, configurator) worker_pool[k] = cluster[k] end } - + else + worker_pool["groom"] = false # don't meddle with ECS workers unnecessarily end configurator.insertKitten(worker_pool, "server_pools") @@ -1748,7 +1749,7 @@ def manage_ecs_workers @deploy.findLitterMate(type: "server_pools", name: @config["name"]+"workers") end serverpool.listNodes.each { |mynode| - resources = resource_lookup[node.cloud_desc.instance_type] + resources = resource_lookup[mynode.cloud_desc.instance_type] threads << Thread.new(mynode) { |node| ident_doc = nil ident_doc_sig = nil @@ -1949,6 +1950,8 @@ def register_ecs_task(container_definitions, service_name, cpu_total = 2, mem_to task_params[:network_mode] = "awsvpc" task_params[:cpu] = cpu_total.to_i.to_s task_params[:memory] = mem_total.to_i.to_s + elsif @config['vpc'] + task_params[:network_mode] = "awsvpc" end MU.log "Registering task definition #{service_name} with #{container_definitions.size.to_s} containers" diff --git a/modules/mu/providers/aws/role.rb b/modules/mu/providers/aws/role.rb index 6d02c468c..22efeff00 100644 --- a/modules/mu/providers/aws/role.rb +++ b/modules/mu/providers/aws/role.rb @@ -459,7 +459,11 @@ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, cred # For some dumb reason, the list output that .find gets doesn't # include the tags, so we need to fetch each role individually to # check tags. Hardly seems efficient. - desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name) + desc = begin + MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name) + rescue Aws::IAM::Errors::NoSuchEntity + next + end if desc.role and desc.role.tags and desc.role.tags master_match = false deploy_match = false diff --git a/modules/mu/providers/aws/server_pool.rb b/modules/mu/providers/aws/server_pool.rb index f3d59c84e..c550a57c0 100644 --- a/modules/mu/providers/aws/server_pool.rb +++ b/modules/mu/providers/aws/server_pool.rb @@ -193,9 +193,10 @@ def setScaleInProtection(need_instances = @config['min_size']) # @return [Array] def listNodes nodes = [] - me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id) - if me and me.first and me.first.instances - me.first.instances.each { |instance| + me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id).values.first + pp me + if me and me.instances + me.instances.each { |instance| found = MU::MommaCat.findStray("AWS", "server", cloud_id: instance.instance_id, region: @config["region"], dummy_ok: true) nodes.concat(found) } @@ -532,14 +533,25 @@ def toKitten(**_args) if cloud_desc.vpc_zone_identifier and !cloud_desc.vpc_zone_identifier.empty? nets = cloud_desc.vpc_zone_identifier.split(/,/) - resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first - bok['vpc'] = MU::Config::Ref.get( - id: resp.vpc_id, - cloud: "AWS", - credentials: @credentials, - type: "vpcs", - subnets: nets.map { |s| { "subnet_id" => s } } - ) + begin + resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first + bok['vpc'] = MU::Config::Ref.get( + id: resp.vpc_id, + cloud: "AWS", + credentials: @credentials, + type: "vpcs", + subnets: nets.map { |s| { "subnet_id" => s } } + ) + rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e + if e.message.match(/The subnet ID '(subnet-[a-f0-9]+)' does not exist/) + nets.delete(Regexp.last_match[1]) + if nets.empty? + MU.log "Autoscale Group #{@cloud_id} was configured for a VPC, but the configuration held no valid subnets", MU::WARN, details: cloud_desc.vpc_zone_identifier.split(/,/) + end + else + raise e + end + end end # MU.log @cloud_id, MU::NOTICE, details: cloud_desc diff --git a/modules/tests/microservice_app.yaml b/modules/tests/microservice_app.yaml index 44bbbe01c..1560194a2 100644 --- a/modules/tests/microservice_app.yaml +++ b/modules/tests/microservice_app.yaml @@ -3,7 +3,7 @@ # play in. # clouds: AWS --- -appname: espier +appname: SMOKETEST parameters: - name: domain default: "sandbox.egt-labs.com" # this must exist as a Route53 zone and have a corresponding wildcard ACM or IAM SSL certificate From 2d7497de55c4a96bbcfbd0770f2fe95f1e0a5ea1 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 21 Sep 2020 02:25:55 -0400 Subject: [PATCH 102/107] RDS tests back to passing --- modules/tests/rds.yaml | 10 +++++----- modules/tests/regrooms/rds.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/tests/rds.yaml b/modules/tests/rds.yaml index 18ad23008..1c9449a50 100644 --- a/modules/tests/rds.yaml +++ b/modules/tests/rds.yaml @@ -28,7 +28,7 @@ databases: # name: rdstests - name: maria-base - size: db.t2.small + size: db.t3.small engine: mariadb db_parameter_group_parameters: - name: autocommit @@ -44,7 +44,7 @@ databases: multi_az_on_create: true master_user: Bob - name: maria-from-snap - size: db.t2.small + size: db.t3.small engine: mariadb vpc: name: rdstests @@ -86,12 +86,12 @@ databases: name: oracle-base - name: sqlserver-base - size: db.t2.small + size: db.t3.small engine: sqlserver-ex vpc: name: rdstests - name: sqlserver-from-snap - size: db.t2.small + size: db.t3.small engine: sqlserver-ex vpc: name: rdstests @@ -99,7 +99,7 @@ databases: source: name: sqlserver-base - name: sqlserver-point-in-time - size: db.t2.small + size: db.t3.small engine: sqlserver-ex vpc: name: rdstests diff --git a/modules/tests/regrooms/rds.yaml b/modules/tests/regrooms/rds.yaml index 3bb5d763f..802f5ff35 100644 --- a/modules/tests/regrooms/rds.yaml +++ b/modules/tests/regrooms/rds.yaml @@ -39,7 +39,7 @@ databases: # name: rdstests - name: maria-base - size: db.t2.small + size: db.t3.small engine: mariadb db_parameter_group_parameters: - name: autocommit @@ -55,7 +55,7 @@ databases: multi_az_on_create: true master_user: Stoki - name: maria-from-snap - size: db.t2.small + size: db.t3.small engine: mariadb port: 3307 vpc: @@ -101,12 +101,12 @@ databases: name: oracle-base - name: sqlserver-base - size: db.t2.small + size: db.t3.small engine: sqlserver-ex vpc: name: rdstests - name: sqlserver-from-snap - size: db.t2.small + size: db.t3.small engine: sqlserver-ex vpc: name: rdstests @@ -114,7 +114,7 @@ databases: source: name: sqlserver-base - name: sqlserver-point-in-time - size: db.t2.small + size: db.t3.small engine: sqlserver-ex vpc: name: rdstests From 0a778be88a183bdc8ffd1b6ad1aa8ae96db03cb0 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 21 Sep 2020 12:54:04 -0400 Subject: [PATCH 103/107] fix simple/complex test sets --- modules/mu/providers/aws/search_domain.rb | 2 +- modules/tests/super_complex_bok.yml | 2 +- modules/tests/super_simple_bok.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mu/providers/aws/search_domain.rb b/modules/mu/providers/aws/search_domain.rb index ed3926bea..84cd84c83 100644 --- a/modules/mu/providers/aws/search_domain.rb +++ b/modules/mu/providers/aws/search_domain.rb @@ -219,7 +219,7 @@ def toKitten(**_args) bok['master_instance_type'] = cloud_desc.elasticsearch_cluster_config.dedicated_master_type end - if cloud_desc.access_policies + if cloud_desc.access_policies and !cloud_desc.access_policies.empty? bok['access_policies'] = JSON.parse(cloud_desc.access_policies) end diff --git a/modules/tests/super_complex_bok.yml b/modules/tests/super_complex_bok.yml index f817fe3c3..28b85871b 100644 --- a/modules/tests/super_complex_bok.yml +++ b/modules/tests/super_complex_bok.yml @@ -11,7 +11,7 @@ parameters: - name: instance_type default: t3.medium - name: db_size - default: db.t2.small + default: db.t3.small - name: vpc_name default: superBoK_VPC - name: logs_name diff --git a/modules/tests/super_simple_bok.yml b/modules/tests/super_simple_bok.yml index 8a8aa93da..ca1ba58fb 100644 --- a/modules/tests/super_simple_bok.yml +++ b/modules/tests/super_simple_bok.yml @@ -10,7 +10,7 @@ parameters: - name: instance_type default: t3.medium - name: db_size - default: db.t2.small + default: db.t3.small - name: vpc_name default: superBoK_VPC - name: logs_name From fe00f2c8126a242aa1a886f0338f1fd8ed1ea239 Mon Sep 17 00:00:00 2001 From: John Stange Date: Mon, 21 Sep 2020 16:40:19 -0400 Subject: [PATCH 104/107] Google::Role: guard nil binding list from folders --- modules/mu/providers/google/role.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/mu/providers/google/role.rb b/modules/mu/providers/google/role.rb index e199843c3..0fb942039 100644 --- a/modules/mu/providers/google/role.rb +++ b/modules/mu/providers/google/role.rb @@ -925,7 +925,9 @@ def self.insertBinding(scopetype, scope, binding = nil, member_type: nil, member } MU::Cloud.resourceClass("Google", "Folder").find(credentials: credentials).keys.each { |folder| - MU::Cloud.resourceClass("Google", "Folder").bindings(folder, credentials: credentials).each { |binding| + folder_bindings = MU::Cloud.resourceClass("Google", "Folder").bindings(folder, credentials: credentials) + next if !folder_bindings + folder_bindings.each { |binding| insertBinding("folders", folder, binding) } } From 2a2280e19705031d49b0936f9f6a60dd738e9393 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Mon, 21 Sep 2020 21:21:28 -0400 Subject: [PATCH 105/107] move to prerelease version --- cloud-mu.gemspec | 4 ++-- modules/Gemfile.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud-mu.gemspec b/cloud-mu.gemspec index 8c4fa0112..a9dc8f37d 100644 --- a/cloud-mu.gemspec +++ b/cloud-mu.gemspec @@ -17,8 +17,8 @@ end Gem::Specification.new do |s| s.name = 'cloud-mu' - s.version = '3.2.0' - s.date = '2020-06-16' + s.version = '3.3.0beta1' + s.date = '2020-09-21' s.require_paths = ['modules'] s.required_ruby_version = '>= 2.4' s.summary = "The eGTLabs Mu toolkit for unified cloud deployments" diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index 56b6eafba..65b748d92 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -10,7 +10,7 @@ GIT PATH remote: .. specs: - cloud-mu (3.2.0) + cloud-mu (3.3.0beta1) addressable (~> 2.5) aws-sdk-core (< 3) azure_sdk (~> 0.52) From 39fb9991a899d34d528d580480ec1555abe86097 Mon Sep 17 00:00:00 2001 From: John Stange Date: Tue, 22 Sep 2020 02:08:36 -0400 Subject: [PATCH 106/107] last call for minor support gem updates --- modules/Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index 65b748d92..a2bad0cad 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -54,7 +54,7 @@ GEM public_suffix (>= 2.0.2, < 4.0) ast (2.4.1) aws-eventstream (1.1.0) - aws-sdk-core (2.11.586) + aws-sdk-core (2.11.590) aws-sigv4 (~> 1.0) jmespath (~> 1.0) aws-sigv4 (1.2.2) @@ -732,7 +732,7 @@ GEM rack (2.2.3) rainbow (3.0.0) rake (13.0.1) - regexp_parser (1.7.1) + regexp_parser (1.8.0) representable (3.0.4) declarative (< 0.1.0) declarative-option (< 0.2.0) @@ -768,7 +768,7 @@ GEM rubocop-ast (>= 0.4.0, < 1.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (0.4.0) + rubocop-ast (0.4.2) parser (>= 2.7.1.4) ruby-graphviz (1.2.5) rexml From 151ce1695659faa93931fa2ed764211df56f0206 Mon Sep 17 00:00:00 2001 From: Mu Administrator Date: Tue, 22 Sep 2020 21:30:37 -0400 Subject: [PATCH 107/107] release 3.3.0 --- cloud-mu.gemspec | 4 ++-- modules/Gemfile.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud-mu.gemspec b/cloud-mu.gemspec index a9dc8f37d..8e37ff486 100644 --- a/cloud-mu.gemspec +++ b/cloud-mu.gemspec @@ -17,8 +17,8 @@ end Gem::Specification.new do |s| s.name = 'cloud-mu' - s.version = '3.3.0beta1' - s.date = '2020-09-21' + s.version = '3.3.0' + s.date = '2020-09-22' s.require_paths = ['modules'] s.required_ruby_version = '>= 2.4' s.summary = "The eGTLabs Mu toolkit for unified cloud deployments" diff --git a/modules/Gemfile.lock b/modules/Gemfile.lock index a2bad0cad..faf6c59db 100644 --- a/modules/Gemfile.lock +++ b/modules/Gemfile.lock @@ -10,7 +10,7 @@ GIT PATH remote: .. specs: - cloud-mu (3.3.0beta1) + cloud-mu (3.3.0) addressable (~> 2.5) aws-sdk-core (< 3) azure_sdk (~> 0.52)