From eb388c72f36fbd98a0cfe9ab4f53bef34214c8b0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 26 Jun 2024 21:03:36 -0600 Subject: [PATCH 01/75] ci: PR update checklist GH- anchors to align w/ later template (#5449) --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3b7983c76a7..1f6890f71e0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,7 +10,7 @@ To ease the process of reviewing your PR, do make sure to complete the following - Test files should map to source files i.e. a source file ``cloudinit/example.py`` should be tested by ``tests/unittests/test_example.py`` - Run unit tests with ``tox -e py3`` - [ ] I have kept the change small, avoiding unnecessary whitespace or non-functional changes. -- [ ] I have added a reference to issues that this PR relates to in the PR message (Refs #1234, Closes #1234) +- [ ] I have added a reference to issues that this PR relates to in the PR message (Refs GH-1234, Fixes GH-1234) - [ ] I have updated the documentation with the changed behavior. - If the change doesn't change the user interface and is trivial, this step may be skipped. - Cloud-config documentation is generated from the jsonschema. From 8470af001b9da6ae0a9ece72ca3e05434c34af58 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 27 Jun 2024 06:44:48 -0600 Subject: [PATCH 02/75] tests: hard-code curtin-dev ppa instead of canonical-kernel-team (#5450) Avoid integration test dependencies on PPAs that we don't own. The canonical-kernel-team PPA will trail publication of packages to the devel series sometimes by many weeks. Also, their process may change in the future and the daily canonical-kernel-team PPA may not be used at some point in the future. Some of our apt-related integration tests are only asserting that different formats of user-data can setup and assert valid PPA signing keys to 3 separate PPAs on a single instance launch to avoid the costs of launching three separate instances to test different user-data formats. For these tests, we can hard-code a known series and PPA without degrading the value of the simple test. Shift our canonical-kernel-team PPA reference to curtin-dev daily PPA for two reasons: - we have tighter coupling with the curtin dev team to keep in step with any process changes - ppa:curtin-dev/daily already has a 4096-bit signing key instead of the stale 1024-bit key in canonical-kernel-team's PPA which may change in the near future. --- .../modules/test_apt_functionality.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 06c6ef28ef7..1b49722fd03 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -48,9 +48,10 @@ deb-src $SECURITY $RELEASE-security multiverse sources: test_keyserver: - keyid: 110E21D8B0E2A1F0243AF6820856F197B892ACEA + keyid: 1BC30F715A3B861247A81A5E55FE7C8C0165013E keyserver: keyserver.ubuntu.com - source: "deb http://ppa.launchpad.net/canonical-kernel-team/ppa/ubuntu $RELEASE main" + # Hard-code noble as devel releases may not see new packages for some time + source: "deb http://ppa.launchpad.net/curtin-dev/daily/ubuntu noble main" test_ppa: keyid: 441614D8 keyserver: keyserver.ubuntu.com @@ -249,7 +250,7 @@ def test_keyserver(self, class_client: IntegrationInstance): ) assert ( - "http://ppa.launchpad.net/canonical-kernel-team/ppa/ubuntu" + "http://ppa.launchpad.net/curtin-dev/daily/ubuntu" in test_keyserver_contents ) @@ -452,9 +453,10 @@ def test_apt_proxy(client: IntegrationInstance): apt: sources: test_keyserver: - keyid: 110E21D8B0E2A1F0243AF6820856F197B892ACEA + keyid: 1BC30F715A3B861247A81A5E55FE7C8C0165013E keyserver: keyserver.ubuntu.com - source: "deb http://ppa.launchpad.net/canonical-kernel-team/ppa/ubuntu $RELEASE main" + # Hard-code noble as devel releases may not see new packages for some time + source: "deb http://ppa.launchpad.net/curtin-dev/daily/ubuntu noble main" test_ppa: keyid: 441614D8 keyserver: keyserver.ubuntu.com From 525026061404ef09baebb85631d9af3b0a4d8930 Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Thu, 27 Jun 2024 18:38:22 +0530 Subject: [PATCH 03/75] Support metalink in yum repository config (#5444) 'metalink' config can be specified instead or along with 'baseurl' in the yum repository config. Add support for specifying metalink instead of 'baseurl'. Fixes GH-5359 Signed-off-by: Ani Sinha Co-authored-by: Ben Gray --- cloudinit/config/cc_yum_add_repo.py | 24 ++++++------ doc/examples/cloud-config-yum-repo.txt | 3 +- .../unittests/config/test_cc_yum_add_repo.py | 38 +++++++++++++++++++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index cf3fd74a4ce..879e25c85c2 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -211,24 +211,22 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: n_repo_config[k] = v repo_config = n_repo_config missing_required = 0 - for req_field in ["baseurl"]: + req_fields = ["baseurl", "metalink"] + for req_field in req_fields: if req_field not in repo_config: - LOG.warning( - "Repository %s does not contain a %s" - " configuration 'required' entry", - repo_id, - req_field, - ) missing_required += 1 - if not missing_required: - repo_configs[canon_repo_id] = repo_config - repo_locations[canon_repo_id] = repo_fn_pth - else: + + if missing_required == len(req_fields): LOG.warning( - "Repository %s is missing %s required fields, skipping!", + "Repository %s should contain atleast one of the" + " following configuration entries: %s, skipping!", repo_id, - missing_required, + ", ".join(req_fields), ) + else: + repo_configs[canon_repo_id] = repo_config + repo_locations[canon_repo_id] = repo_fn_pth + for (c_repo_id, path) in repo_locations.items(): repo_blob = _format_repository_config( c_repo_id, repo_configs.get(c_repo_id) diff --git a/doc/examples/cloud-config-yum-repo.txt b/doc/examples/cloud-config-yum-repo.txt index e8f2bbb415a..6a4037e2462 100644 --- a/doc/examples/cloud-config-yum-repo.txt +++ b/doc/examples/cloud-config-yum-repo.txt @@ -11,8 +11,9 @@ yum_repos: # Any repository configuration options # See: man yum.conf # - # This one is required! + # At least one of 'baseurl' or 'metalink' is required! baseurl: http://download.fedoraproject.org/pub/epel/testing/5/$basearch + metalink: https://mirrors.fedoraproject.org/metalink?repo=epel-$releasever&arch=$basearch&infra=$infra&content=$contentdir enabled: false failovermethod: priority gpgcheck: true diff --git a/tests/unittests/config/test_cc_yum_add_repo.py b/tests/unittests/config/test_cc_yum_add_repo.py index 1707860a19f..e6a9109ee19 100644 --- a/tests/unittests/config/test_cc_yum_add_repo.py +++ b/tests/unittests/config/test_cc_yum_add_repo.py @@ -31,6 +31,7 @@ def test_bad_config(self): "yum_repos": { "epel-testing": { "name": "Extra Packages for Enterprise Linux 5 - Testing", + # At least one of baseurl or metalink must be present. # Missing this should cause the repo not to be written # 'baseurl': 'http://blah.org/pub/epel/testing/5/$barch', "enabled": False, @@ -46,6 +47,43 @@ def test_bad_config(self): IOError, util.load_text_file, "/etc/yum.repos.d/epel_testing.repo" ) + def test_metalink_config(self): + cfg = { + "yum_repos": { + "epel-testing": { + "name": "Extra Packages for Enterprise Linux 5 - Testing", + "metalink": "http://blah.org/pub/epel/testing/5/$basearch", + "enabled": False, + "gpgcheck": True, + "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL", + "failovermethod": "priority", + }, + }, + } + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + cc_yum_add_repo.handle("yum_add_repo", cfg, None, []) + contents = util.load_text_file("/etc/yum.repos.d/epel-testing.repo") + parser = configparser.ConfigParser() + parser.read_string(contents) + expected = { + "epel-testing": { + "name": "Extra Packages for Enterprise Linux 5 - Testing", + "failovermethod": "priority", + "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL", + "enabled": "0", + "metalink": "http://blah.org/pub/epel/testing/5/$basearch", + "gpgcheck": "1", + } + } + for section in expected: + self.assertTrue( + parser.has_section(section), + "Contains section {0}".format(section), + ) + for k, v in expected[section].items(): + self.assertEqual(parser.get(section, k), v) + def test_write_config(self): cfg = { "yum_repos": { From 2c09f69173d448118b02e013518bf5f1674d3c1f Mon Sep 17 00:00:00 2001 From: Alexsander de Souza <61709370+alexsander-souza@users.noreply.github.com> Date: Thu, 27 Jun 2024 12:24:50 -0300 Subject: [PATCH 04/75] fix(net): klibc ipconfig PROTO compatibility (#5437) klibc's ipconfig format [1] states that PROTO values 'none', 'off', 'static' and blank all mean no autoconfiguration, but cloud-init parser is too strict and accepts only the first. LP: #2065787 [1] https://git.kernel.org/pub/scm/libs/klibc/klibc.git/plain/usr/kinit/ipconfig/README.ipconfig --- cloudinit/net/cmdline.py | 3 +++ tests/unittests/test_net.py | 39 +++++++++++++++++++++++++++++++++++-- tools/.github-cla-signers | 1 + 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 62abe78b4bf..4455a6d8145 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -127,6 +127,9 @@ def _klibc_to_config_entry(content, mac_addrs=None): else: proto = "none" + if proto in ("static", "off"): + proto = "none" + if proto not in ("none", "dhcp", "dhcp6"): raise ValueError("Unexpected value for PROTO: %s" % proto) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 545154d5fc0..dbae4f20267 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -133,6 +133,37 @@ ], } +STATIC_CONTENT_2 = """ +DEVICE='eth1' +PROTO='static' +IPV4ADDR='10.0.0.2' +IPV4BROADCAST='10.0.0.255' +IPV4NETMASK='255.255.255.0' +IPV4GATEWAY='10.0.0.1' +IPV4DNS0='10.0.1.1' +IPV4DNS1='0.0.0.0' +HOSTNAME='foohost' +UPTIME='21' +DHCPLEASETIME='3600' +DOMAINSEARCH='foo.com' +""" + +STATIC_CONTENT_3 = """ +DEVICE='eth1' +PROTO='off' +IPV4ADDR='10.0.0.2' +IPV4BROADCAST='10.0.0.255' +IPV4NETMASK='255.255.255.0' +IPV4GATEWAY='10.0.0.1' +IPV4DNS0='10.0.1.1' +IPV4DNS1='0.0.0.0' +HOSTNAME='foohost' +UPTIME='21' +DHCPLEASETIME='3600' +DOMAINSEARCH='foo.com' +""" + + V1_NAMESERVER_ALIAS = """ config: - id: eno1 @@ -3763,8 +3794,12 @@ def test_cmdline_convert_dhcp6(self): assert found == ("eno1", DHCP6_EXPECTED_1) def test_cmdline_convert_static(self): - found = cmdline._klibc_to_config_entry(STATIC_CONTENT_1) - assert found == ("eth1", STATIC_EXPECTED_1) + found1 = cmdline._klibc_to_config_entry(STATIC_CONTENT_1) + assert found1 == ("eth1", STATIC_EXPECTED_1) + found2 = cmdline._klibc_to_config_entry(STATIC_CONTENT_2) + assert found2 == ("eth1", STATIC_EXPECTED_1) + found3 = cmdline._klibc_to_config_entry(STATIC_CONTENT_3) + assert found3 == ("eth1", STATIC_EXPECTED_1) def test_config_from_cmdline_net_cfg(self): files = [] diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d154eb725ce..29147fb0dbc 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -7,6 +7,7 @@ afbjorklund ajmyyra akutz AlexBaranowski +alexsander-souza AlexSv04047 AliyevH Aman306 From 790d2291837de9c68ccc067cb2c88b11c60e69f9 Mon Sep 17 00:00:00 2001 From: eaglegai <31752768+eaglegai@users.noreply.github.com> Date: Fri, 28 Jun 2024 01:14:01 +0800 Subject: [PATCH 05/75] fix: Add get_connection_with_tls_context() for requests 2.32.2+ (#5435) get_connection() is deprecated in requests 2.32.2+ so this will allow for the LXDSocketAdapter to avoid using a deprecated api. Fixes GH-5434 Signed-off-by: eaglegai --- cloudinit/sources/DataSourceLXD.py | 7 +++++++ tools/.github-cla-signers | 1 + 2 files changed, 8 insertions(+) diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py index 9d7d7a1ad10..a85853ec44a 100644 --- a/cloudinit/sources/DataSourceLXD.py +++ b/cloudinit/sources/DataSourceLXD.py @@ -132,6 +132,13 @@ class LXDSocketAdapter(HTTPAdapter): def get_connection(self, url, proxies=None): return SocketConnectionPool(LXD_SOCKET_PATH) + # Fix for requests 2.32.2+: + # https://github.com/psf/requests/pull/6710 + def get_connection_with_tls_context( + self, request, verify, proxies=None, cert=None + ): + return self.get_connection(request.url, proxies) + def _raw_instance_data_to_dict(metadata_type: str, metadata_value) -> dict: """Convert raw instance data from str, bytes, YAML to dict diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 29147fb0dbc..4db2e14df6d 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -55,6 +55,7 @@ dermotbradley dhalturin dhensby Dorthu +eaglegai eandersson eb3095 ederst From 6ee0079391c0c65aab6567dae769f0807b71c21e Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 27 Jun 2024 12:55:18 -0600 Subject: [PATCH 06/75] chore(cmdline): Update comments (#5458) Update based on 2c09f69 --- cloudinit/net/cmdline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 4455a6d8145..c2c1d5af45b 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -101,9 +101,9 @@ def _klibc_to_config_entry(content, mac_addrs=None): provided here. There is no good documentation on this unfortunately. DEVICE= is expected/required and PROTO should indicate if - this is 'none' (static) or 'dhcp' or 'dhcp6' (LP: #1621507). - note that IPV6PROTO is also written by newer code to address the - possibility of both ipv4 and ipv6 getting addresses. + this is 'none' (static) or 'dhcp' or 'dhcp6' (LP: #1621507) or 'static' + or 'off' (LP: 2065787). Note that IPV6PROTO is also written to address + the possibility of both ipv4 and ipv6 getting addresses. Full syntax is documented at: https://git.kernel.org/pub/scm/libs/klibc/klibc.git/plain/usr/kinit/ipconfig/README.ipconfig From 3851c5c89a7c3f0926b61d218bdd85b0bd23104d Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 27 Jun 2024 13:53:03 -0600 Subject: [PATCH 07/75] test: Add ds-identify integration test coverage (#5394) LXD instances don't have a defined datasource_list and therefore implicitly exercise ds-identify. Add coverage for all other platforms. --- tests/integration_tests/test_ds_identify.py | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/integration_tests/test_ds_identify.py diff --git a/tests/integration_tests/test_ds_identify.py b/tests/integration_tests/test_ds_identify.py new file mode 100644 index 00000000000..2375dde7909 --- /dev/null +++ b/tests/integration_tests/test_ds_identify.py @@ -0,0 +1,36 @@ +"""test that ds-identify works as expected""" + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.integration_settings import OS_IMAGE, PLATFORM +from tests.integration_tests.util import verify_clean_log, wait_for_cloud_init + +DATASOURCE_LIST_FILE = "/etc/cloud/cloud.cfg.d/90_dpkg.cfg" +MAP_PLATFORM_TO_DATASOURCE = { + "lxd_container": "lxd", + "lxd_vm": "lxd", + "qemu": "nocloud", + "ec2": "aws", + "oci": "oracle", +} + + +def test_ds_identify(client: IntegrationInstance): + """Verify that ds-identify works correctly + + Deb packaging often a defines datasource_list with a single datasource, + which bypasses ds-identify logic. This tests works by removing this file + and verifying that cloud-init doesn't experience issues. + """ + assert client.execute(f"rm {DATASOURCE_LIST_FILE}") + assert client.execute("cloud-init clean --logs") + client.restart() + wait_for_cloud_init(client) + verify_clean_log(client.execute("cat /var/log/cloud-init.log")) + assert client.execute("cloud-init status --wait") + + datasource = MAP_PLATFORM_TO_DATASOURCE.get(PLATFORM, PLATFORM) + if "lxd" == datasource and "focal" == OS_IMAGE: + datasource = "nocloud" + cloud_id = client.execute("cloud-id") + assert cloud_id.ok + assert datasource == cloud_id.stdout.rstrip() From 12f1198e8e9e884363b14eeaaf6eb69b7199c36a Mon Sep 17 00:00:00 2001 From: Curt Moore Date: Tue, 4 Jun 2024 14:37:43 -0500 Subject: [PATCH 08/75] fix(openstack): Fix bond mac_address (#5369) Fixes GH-5368 --- cloudinit/sources/helpers/openstack.py | 2 +- tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 094c889caef..4f86de1f883 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -667,7 +667,7 @@ def convert_net_json(network_json=None, known_macs=None): if link["type"] in ["bond"]: params = {} if link_mac_addr: - params["mac_address"] = link_mac_addr + cfg.update({"mac_address": link_mac_addr}) for k, v in link.items(): if k == "bond_links": continue diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 4db2e14df6d..3aea01b0023 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -86,6 +86,7 @@ izzyleung j5awry jacobsalmela jamesottinger +jcmoore3 Jehops jf jfroche From f8f9d19409fcbda32e119a5514fd5185bcd88b79 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 27 Jun 2024 11:56:58 -0600 Subject: [PATCH 09/75] test(openstack): Test bond mac address (#5369) --- .../sources/helpers/test_openstack.py | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tests/unittests/sources/helpers/test_openstack.py b/tests/unittests/sources/helpers/test_openstack.py index 4d85ec3c61f..312d66a0186 100644 --- a/tests/unittests/sources/helpers/test_openstack.py +++ b/tests/unittests/sources/helpers/test_openstack.py @@ -112,3 +112,123 @@ def test_subnet_dns(self): assert expected == openstack.convert_net_json( network_json=net_json, known_macs=macs ) + + def test_bond_mac(self): + """Verify the bond mac address is assigned correctly.""" + network_json = { + "links": [ + { + "id": "ens1f0np0", + "name": "ens1f0np0", + "type": "phy", + "ethernet_mac_address": "xx:xx:xx:xx:xx:00", + "mtu": 9000, + }, + { + "id": "ens1f1np1", + "name": "ens1f1np1", + "type": "phy", + "ethernet_mac_address": "xx:xx:xx:xx:xx:01", + "mtu": 9000, + }, + { + "id": "bond0", + "name": "bond0", + "type": "bond", + "bond_links": ["ens1f0np0", "ens1f1np1"], + "mtu": 9000, + "ethernet_mac_address": "xx:xx:xx:xx:xx:00", + "bond_mode": "802.3ad", + "bond_xmit_hash_policy": "layer3+4", + "bond_miimon": 100, + }, + { + "id": "bond0.123", + "name": "bond0.123", + "type": "vlan", + "vlan_link": "bond0", + "vlan_id": 123, + "vlan_mac_address": "xx:xx:xx:xx:xx:00", + }, + ], + "networks": [ + { + "id": "publicnet-ipv4", + "type": "ipv4", + "link": "bond0.123", + "ip_address": "x.x.x.x", + "netmask": "255.255.255.0", + "routes": [ + { + "network": "0.0.0.0", + "netmask": "0.0.0.0", + "gateway": "x.x.x.1", + } + ], + "network_id": "00000000-0000-0000-0000-000000000000", + } + ], + "services": [{"type": "dns", "address": "1.1.1.1"}], + } + expected = { + "config": [ + { + "mac_address": "xx:xx:xx:xx:xx:00", + "mtu": 9000, + "name": "ens1f0np0", + "subnets": [], + "type": "physical", + }, + { + "mac_address": "xx:xx:xx:xx:xx:01", + "mtu": 9000, + "name": "ens1f1np1", + "subnets": [], + "type": "physical", + }, + { + "bond_interfaces": ["ens1f0np0", "ens1f1np1"], + "mtu": 9000, + "name": "bond0", + "mac_address": "xx:xx:xx:xx:xx:00", + "params": { + "bond_miimon": 100, + "bond_mode": "802.3ad", + "bond_xmit_hash_policy": "layer3+4", + }, + "subnets": [], + "type": "bond", + }, + { + "mac_address": "xx:xx:xx:xx:xx:00", + "name": "bond0.123", + "subnets": [ + { + "address": "x.x.x.x", + "ipv4": True, + "netmask": "255.255.255.0", + "routes": [ + { + "gateway": "x.x.x.1", + "netmask": "0.0.0.0", + "network": "0.0.0.0", + } + ], + "type": "static", + } + ], + "type": "vlan", + "vlan_id": 123, + "vlan_link": "bond0", + }, + {"address": "1.1.1.1", "type": "nameserver"}, + ], + "version": 1, + } + macs = { + "xx:xx:xx:xx:xx:00": "ens1f0np0", + "xx:xx:xx:xx:xx:01": "ens1f1np1", + } + assert expected == openstack.convert_net_json( + network_json=network_json, known_macs=macs + ) From bcc5920a78c55af51ccfcd67425396faeae54223 Mon Sep 17 00:00:00 2001 From: Curt Moore Date: Thu, 27 Jun 2024 15:08:31 -0500 Subject: [PATCH 10/75] fix: Gracefully handle missing files (#5397) Do not raise an error when userdata files are missing in cloud-init query operations. Fixes GH-5396 --- cloudinit/cmd/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py index e1ce54ea1c0..283c6069a39 100644 --- a/cloudinit/cmd/query.py +++ b/cloudinit/cmd/query.py @@ -131,7 +131,7 @@ def load_userdata(ud_file_path): @returns: String of uncompressed userdata if possible, otherwise bytes. """ - bdata = util.load_binary_file(ud_file_path) + bdata = util.load_binary_file(ud_file_path, quiet=True) try: return bdata.decode("utf-8") except UnicodeDecodeError: From e80514b56139d6cc48e7c7d8e694fe6289eae023 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 18 Jun 2024 13:23:14 -0600 Subject: [PATCH 11/75] feat: Add deprecation boundary to logger (#5411) This will all stable distros to define whether a key is deprecated based on the version of cloud-init which deprecated the key. --- cloudinit/cmd/main.py | 21 ++++++++------------- cloudinit/features.py | 29 +++++++++++++++++++++++++++++ cloudinit/util.py | 35 +++++++++++++++++++++++++---------- 3 files changed, 62 insertions(+), 23 deletions(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index c16d2703dab..4a1c8b2e28c 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -236,17 +236,12 @@ def attempt_cmdline_url(path, network=True, cmdline=None) -> Tuple[int, str]: is_cloud_cfg = False if is_cloud_cfg: if cmdline_name == "url": - return ( - log.DEPRECATED, - str( - util.deprecate( - deprecated="The kernel command line key `url`", - deprecated_version="22.3", - extra_message=" Please use `cloud-config-url` " - "kernel command line parameter instead", - return_log=True, - ), - ), + return util.deprecate( + deprecated="The kernel command line key `url`", + deprecated_version="22.3", + extra_message=" Please use `cloud-config-url` " + "kernel command line parameter instead", + skip_log=True, ) else: if cmdline_name == "cloud-config-url": @@ -972,8 +967,8 @@ def main(sysv_args=None): deprecated="`init`", deprecated_version="24.1", extra_message="Use `cloud-init init` instead.", - return_log=True, - ) + skip_log=True, + ).message parser_mod.add_argument( "--mode", "-m", diff --git a/cloudinit/features.py b/cloudinit/features.py index d661b940b29..c3fdae18658 100644 --- a/cloudinit/features.py +++ b/cloudinit/features.py @@ -87,6 +87,35 @@ to write /etc/apt/sources.list directly. """ +DEPRECATION_INFO_BOUNDARY = "devel" +""" +DEPRECATION_INFO_BOUNDARY is used by distros to configure at which upstream +version to start logging deprecations at a level higher than INFO. + +The default value "devel" tells cloud-init to log all deprecations higher +than INFO. This value may be overriden by downstreams in order to maintain +stable behavior across releases. + +Jsonschema key deprecations and inline logger deprecations include a +deprecated_version key. When the variable below is set to a version, +cloud-init will use that version as a demarcation point. Deprecations which +are added after this version will be logged as at an INFO level. Deprecations +which predate this version will be logged at the higher DEPRECATED level. +Downstreams that want stable log behavior may set the variable below to the +first version released in their stable distro. By doing this, they can expect +that newly added deprecations will be logged at INFO level. The implication of +the different log levels is that logs at DEPRECATED level result in a return +code of 2 from `cloud-init status`. + +format: + + :: = | + ::= "devel" + ::= "." ["." ] + +where , , and are positive integers +""" + def get_features() -> Dict[str, bool]: """Return a dict of applicable features/overrides and their values.""" diff --git a/cloudinit/util.py b/cloudinit/util.py index 947da4c6337..f42e641440b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -49,6 +49,7 @@ Generator, List, Mapping, + NamedTuple, Optional, Sequence, TypeVar, @@ -61,6 +62,7 @@ from cloudinit import ( features, importer, + log, mergers, net, settings, @@ -89,6 +91,11 @@ FALSE_STRINGS = ("off", "0", "no", "false") +class DeprecationLog(NamedTuple): + log_level: int + message: str + + def kernel_version(): return tuple(map(int, os.uname().release.split(".")[:2])) @@ -3209,8 +3216,8 @@ def deprecate( deprecated_version: str, extra_message: Optional[str] = None, schedule: int = 5, - return_log: bool = False, -): + skip_log: bool = False, +) -> DeprecationLog: """Mark a "thing" as deprecated. Deduplicated deprecations are logged. @@ -3226,8 +3233,10 @@ def deprecate( @param schedule: Manually set the deprecation schedule. Defaults to 5 years. Leave a comment explaining your reason for deviation if setting this value. - @param return_log: Return log text rather than logging it. Useful for + @param skip_log: Return log text rather than logging it. Useful for running prior to logging setup. + @return: NamedTuple containing log level and log message + DeprecationLog(level: int, message: str) Note: uses keyword-only arguments to improve legibility """ @@ -3242,14 +3251,20 @@ def deprecate( f"{deprecated_version} and scheduled to be removed in " f"{version_removed}. {message}" ).rstrip() - if return_log: - return deprecate_msg - if dedup not in deprecate._log: # type: ignore + if ( + "devel" != features.DEPRECATION_INFO_BOUNDARY + and Version.from_str(features.DEPRECATION_INFO_BOUNDARY) < version + ): + LOG.info(deprecate_msg) + level = logging.INFO + elif hasattr(LOG, "deprecated"): + level = log.DEPRECATED + else: + level = logging.WARN + if not skip_log and dedup not in deprecate._log: # type: ignore deprecate._log.add(dedup) # type: ignore - if hasattr(LOG, "deprecated"): - LOG.deprecated(deprecate_msg) # type: ignore - else: - LOG.warning(deprecate_msg) + LOG.log(level, deprecate_msg) + return DeprecationLog(level, deprecate_msg) def deprecate_call( From 8906e17ef99e9dcfc6f5b6f48452dcd97226dec7 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Wed, 19 Jun 2024 15:55:26 -0600 Subject: [PATCH 12/75] feat: Add deprecation boundary support to schema validator (#5411) This will all stable distros to define whether a key is deprecated based on the version of cloud-init which deprecated the key. --- cloudinit/config/schema.py | 57 +++++++++++++++++++-------- tests/unittests/config/test_schema.py | 20 +++++----- tests/unittests/test_data.py | 2 + tests/unittests/test_features.py | 2 + 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index c641362f0cb..e552483e543 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -31,13 +31,14 @@ import yaml -from cloudinit import importer, safeyaml +from cloudinit import features, importer, safeyaml from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers import INCLUSION_TYPES_MAP, type_from_starts_with from cloudinit.helpers import Paths from cloudinit.sources import DataSourceNotFoundException from cloudinit.temp_utils import mkdtemp from cloudinit.util import ( + Version, error, get_modules_from_dir, load_text_file, @@ -137,7 +138,14 @@ class MetaSchema(TypedDict): class SchemaDeprecationError(ValidationError): - pass + def __init__( + self, + message: str, + version: str, + **kwargs, + ): + super().__init__(message, **kwargs) + self.version: str = version class SchemaProblem(NamedTuple): @@ -363,7 +371,7 @@ def _validator( msg = _add_deprecated_changed_or_new_msg( schema, annotate=True, filter_key=[filter_key] ) - yield error_type(msg) + yield error_type(msg, schema.get("deprecated_version", "devel")) _validator_deprecated = partial(_validator, filter_key="deprecated") @@ -770,6 +778,7 @@ def validate_cloudconfig_schema( errors: SchemaProblems = [] deprecations: SchemaProblems = [] + info_deprecations: SchemaProblems = [] for schema_error in sorted( validator.iter_errors(config), key=lambda e: e.path ): @@ -785,25 +794,39 @@ def validate_cloudconfig_schema( ) if prop_match: path = prop_match["name"] - problem = (SchemaProblem(path, schema_error.message),) if isinstance( schema_error, SchemaDeprecationError ): # pylint: disable=W1116 - deprecations += problem + if ( + "devel" != features.DEPRECATION_INFO_BOUNDARY + and Version.from_str(schema_error.version) + > Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + ): + info_deprecations.append( + SchemaProblem(path, schema_error.message) + ) + else: + deprecations.append(SchemaProblem(path, schema_error.message)) else: - errors += problem + errors.append(SchemaProblem(path, schema_error.message)) - if log_deprecations and deprecations: - message = _format_schema_problems( - deprecations, - prefix="Deprecated cloud-config provided:\n", - separator="\n", - ) - # This warning doesn't fit the standardized util.deprecated() utility - # format, but it is a deprecation log, so log it directly. - LOG.deprecated(message) # type: ignore - if strict and (errors or deprecations): - raise SchemaValidationError(errors, deprecations) + if log_deprecations: + if info_deprecations: + message = _format_schema_problems( + info_deprecations, + prefix="Deprecated cloud-config provided: ", + ) + LOG.info(message) + if deprecations: + message = _format_schema_problems( + deprecations, + prefix="Deprecated cloud-config provided: ", + ) + # This warning doesn't fit the standardized util.deprecated() + # utility format, but it is a deprecation log, so log it directly. + LOG.deprecated(message) # type: ignore + if strict and (errors or deprecations or info_deprecations): + raise SchemaValidationError(errors, deprecations + info_deprecations) if errors: if log_details: details = _format_schema_problems( diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index cd78faa9f14..76d89c3652e 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -629,7 +629,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"a-b": "asdf"}, - "Deprecated cloud-config provided:\na-b: " + "Deprecated cloud-config provided: a-b: " "Deprecated in version 22.1.", ), ( @@ -650,7 +650,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"x": "+5"}, - "Deprecated cloud-config provided:\nx: " + "Deprecated cloud-config provided: x: " "Deprecated in version 22.1.", ), ( @@ -671,7 +671,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"x": "5"}, - "Deprecated cloud-config provided:\nx: " + "Deprecated cloud-config provided: x: " "Deprecated in version 22.1. ", ), ( @@ -692,7 +692,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"x": "5"}, - "Deprecated cloud-config provided:\nx: " + "Deprecated cloud-config provided: x: " "Deprecated in version 22.1.", ), ( @@ -708,7 +708,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"x": "+5"}, - "Deprecated cloud-config provided:\nx: " + "Deprecated cloud-config provided: x: " "Deprecated in version 22.1.", ), ( @@ -745,7 +745,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"x": "+5"}, - "Deprecated cloud-config provided:\nx: " + "Deprecated cloud-config provided: x: " "Deprecated in version 32.3.", ), ( @@ -770,7 +770,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"x": "+5"}, - "Deprecated cloud-config provided:\nx: Deprecated in " + "Deprecated cloud-config provided: x: Deprecated in " "version 27.2.", ), ( @@ -786,7 +786,7 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"a-b": "asdf"}, - "Deprecated cloud-config provided:\na-b: " + "Deprecated cloud-config provided: a-b: " "Deprecated in version 27.2.", ), pytest.param( @@ -804,8 +804,8 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( }, }, {"a-b": "asdf"}, - "Deprecated cloud-config provided:\na-b: Deprecated " - "in version 27.2.\na-b: Changed in version 22.2. " + "Deprecated cloud-config provided: a-b: Deprecated " + "in version 27.2., a-b: Changed in version 22.2. " "Drop ballast.", id="deprecated_pattern_property_without_description", ), diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 1ee3dfa9007..14be6fa48e3 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -482,6 +482,7 @@ def test_mime_text_plain(self, init_tmp, caplog): ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES=True, EXPIRE_APPLIES_TO_HASHED_USERS=False, NETPLAN_CONFIG_ROOT_READ_ONLY=True, + DEPRECATION_INFO_BOUNDARY="devel", NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, APT_DEB822_SOURCE_LIST_FILE=True, ) @@ -513,6 +514,7 @@ def test_shellscript(self, init_tmp, tmpdir, caplog): "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES": True, "EXPIRE_APPLIES_TO_HASHED_USERS": False, "NETPLAN_CONFIG_ROOT_READ_ONLY": True, + "DEPRECATION_INFO_BOUNDARY": "devel", "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, "APT_DEB822_SOURCE_LIST_FILE": True, }, diff --git a/tests/unittests/test_features.py b/tests/unittests/test_features.py index c9eff407064..e5e81fbffa8 100644 --- a/tests/unittests/test_features.py +++ b/tests/unittests/test_features.py @@ -19,6 +19,7 @@ def test_feature_without_override(self): ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES=True, EXPIRE_APPLIES_TO_HASHED_USERS=False, NETPLAN_CONFIG_ROOT_READ_ONLY=True, + DEPRECATION_INFO_BOUNDARY="devel", NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, APT_DEB822_SOURCE_LIST_FILE=True, ): @@ -29,4 +30,5 @@ def test_feature_without_override(self): "NETPLAN_CONFIG_ROOT_READ_ONLY": True, "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, "APT_DEB822_SOURCE_LIST_FILE": True, + "DEPRECATION_INFO_BOUNDARY": "devel", } == features.get_features() From 7f98af900fe460af0606febb9501102d0ce7ee24 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 27 Jun 2024 12:11:37 -0600 Subject: [PATCH 13/75] test: Add unit tests for features.DEPRECATION_INFO_BOUNDARY (#5411) --- tests/unittests/config/test_schema.py | 38 +++++++++++++++++++++-- tests/unittests/test_log.py | 44 +++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 76d89c3652e..f8f0dcdc563 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -20,6 +20,7 @@ import pytest import yaml +from cloudinit import features from cloudinit.config.schema import ( VERSIONED_USERDATA_SCHEMA_FILE, MetaSchema, @@ -2757,10 +2758,11 @@ def test_handle_schema_unable_to_read_cfg_paths( assert expected_log in caplog.text @pytest.mark.parametrize( - "annotate, expected_output", + "annotate, deprecation_info_boundary, expected_output", [ - ( + pytest.param( True, + "devel", dedent( """\ #cloud-config @@ -2778,9 +2780,33 @@ def test_handle_schema_unable_to_read_cfg_paths( Valid schema {cfg_file} """ # noqa: E501 ), + id="test_annotated_deprecation_info_boundary_devel_shows", ), - ( + pytest.param( + True, + "22.1", + dedent( + """\ + #cloud-config + packages: + - htop + apt_update: true # D1 + apt_upgrade: true # D2 + apt_reboot_if_required: true # D3 + + # Deprecations: ------------- + # D1: Default: ``false``. Deprecated in version 22.2. Use ``package_update`` instead. + # D2: Default: ``false``. Deprecated in version 22.2. Use ``package_upgrade`` instead. + # D3: Default: ``false``. Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. + + Valid schema {cfg_file} + """ # noqa: E501 + ), + id="test_annotated_deprecation_info_boundary_below_unredacted", + ), + pytest.param( False, + "18.2", dedent( """\ Cloud config schema deprecations: \ @@ -2792,6 +2818,7 @@ def test_handle_schema_unable_to_read_cfg_paths( Valid schema {cfg_file} """ # noqa: E501 ), + id="test_deprecation_info_boundary_does_unannotated_unredacted", ), ], ) @@ -2800,11 +2827,13 @@ def test_handle_schema_args_annotate_deprecated_config( self, read_cfg_paths, annotate, + deprecation_info_boundary, expected_output, paths, caplog, capsys, tmpdir, + mocker, ): paths.get_ipath = paths.get_ipath_cur read_cfg_paths.return_value = paths @@ -2822,6 +2851,9 @@ def test_handle_schema_args_annotate_deprecated_config( """ ) ) + mocker.patch.object( + features, "DEPRECATION_INFO_BOUNDARY", deprecation_info_boundary + ) args = self.Args( config_file=str(user_data_fn), schema_type="cloud-config", diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 3a8d9683ee2..e68dcc48029 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -7,6 +7,8 @@ import logging import time +import pytest + from cloudinit import log, util from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT from tests.unittests.helpers import CiTestCase @@ -66,6 +68,48 @@ def test_deprecated_log_level(self, caplog): assert "DEPRECATED" == caplog.records[0].levelname assert "deprecated message" in caplog.text + @pytest.mark.parametrize( + "expected_log_level, deprecation_info_boundary", + ( + pytest.param( + "DEPRECATED", + "19.2", + id="test_same_deprecation_info_boundary_is_deprecated_level", + ), + pytest.param( + "INFO", + "19.1", + id="test_lower_deprecation_info_boundary_is_info_level", + ), + ), + ) + def test_deprecate_log_level_based_on_features( + self, expected_log_level, deprecation_info_boundary, caplog, mocker + ): + """Deprecation log level depends on key deprecation_version + + When DEPRECATION_INFO_BOUNDARY is set to a version number, and a key + has a deprecated_version with a version greater than the boundary + the log level is INFO instead of DEPRECATED. If + DEPRECATION_INFO_BOUNDARY is set to the default, "devel", all + deprecated keys are logged at level DEPRECATED. + """ + mocker.patch.object( + util.features, + "DEPRECATION_INFO_BOUNDARY", + deprecation_info_boundary, + ) + util.deprecate( + deprecated="some key", + deprecated_version="19.2", + extra_message="dont use it", + ) + assert expected_log_level == caplog.records[0].levelname + assert ( + "some key is deprecated in 19.2 and scheduled to be removed in" + " 24.2" in caplog.text + ) + def test_log_deduplication(self, caplog): log.define_deprecation_logger() util.deprecate( From bbcc67dda451ffce64283e2e38aca2bfb4f2acae Mon Sep 17 00:00:00 2001 From: Sally Date: Thu, 27 Jun 2024 23:39:40 +0100 Subject: [PATCH 14/75] doc(refactor): Convert module docs to new system (#5427) Convert the remaining half of the module docs to be rendered by the new template system. Includes the documentation on how to document modules under this system. Add additional developer script tools/format_yaml_doc to reformat either data.yaml or example.yaml cloud-config files for module documentation. Co-authored-by: Chad Smith --- cloudinit/config/cc_rsyslog.py | 25 +-- cloudinit/config/cc_runcmd.py | 48 +---- cloudinit/config/cc_salt_minion.py | 53 +----- cloudinit/config/cc_scripts_per_boot.py | 15 +- cloudinit/config/cc_scripts_per_instance.py | 18 +- cloudinit/config/cc_scripts_per_once.py | 19 +- cloudinit/config/cc_scripts_user.py | 19 +- cloudinit/config/cc_scripts_vendor.py | 42 +---- cloudinit/config/cc_seed_random.py | 51 +---- cloudinit/config/cc_set_hostname.py | 59 +----- cloudinit/config/cc_set_passwords.py | 64 +------ cloudinit/config/cc_snap.py | 95 +--------- cloudinit/config/cc_spacewalk.py | 30 +-- cloudinit/config/cc_ssh.py | 127 +------------ .../config/cc_ssh_authkey_fingerprints.py | 18 +- cloudinit/config/cc_ssh_import_id.py | 28 +-- cloudinit/config/cc_timezone.py | 13 +- cloudinit/config/cc_ubuntu_drivers.py | 27 +-- cloudinit/config/cc_ubuntu_pro.py | 116 +----------- cloudinit/config/cc_update_etc_hosts.py | 78 +------- cloudinit/config/cc_update_hostname.py | 71 +------ cloudinit/config/cc_users_groups.py | 164 +--------------- cloudinit/config/cc_wireguard.py | 78 +------- cloudinit/config/cc_write_files.py | 92 +-------- cloudinit/config/cc_write_files_deferred.py | 16 +- cloudinit/config/cc_yum_add_repo.py | 108 ++--------- cloudinit/config/cc_zypper_add_repo.py | 67 +------ cloudinit/config/schema.py | 11 +- doc/module-docs/cc_ansible/data.yaml | 23 ++- doc/module-docs/cc_apk_configure/data.yaml | 25 ++- doc/module-docs/cc_apt_configure/data.yaml | 49 ++--- doc/module-docs/cc_apt_pipelining/data.yaml | 31 +-- doc/module-docs/cc_bootcmd/data.yaml | 29 ++- doc/module-docs/cc_bootcmd/example1.yaml | 2 +- doc/module-docs/cc_byobu/data.yaml | 30 +-- doc/module-docs/cc_ca_certs/data.yaml | 17 +- doc/module-docs/cc_chef/data.yaml | 24 +-- doc/module-docs/cc_chef/example1.yaml | 21 +-- .../cc_disable_ec2_metadata/data.yaml | 15 +- doc/module-docs/cc_disk_setup/data.yaml | 37 ++-- doc/module-docs/cc_disk_setup/example1.yaml | 42 ++--- doc/module-docs/cc_fan/data.yaml | 18 +- doc/module-docs/cc_final_message/data.yaml | 15 +- doc/module-docs/cc_growpart/data.yaml | 83 ++++---- doc/module-docs/cc_growpart/example1.yaml | 8 +- doc/module-docs/cc_growpart/example2.yaml | 6 +- doc/module-docs/cc_grub_dpkg/data.yaml | 17 +- doc/module-docs/cc_install_hotplug/data.yaml | 27 ++- .../cc_install_hotplug/example1.yaml | 2 +- .../cc_install_hotplug/example2.yaml | 2 +- doc/module-docs/cc_keyboard/data.yaml | 18 +- doc/module-docs/cc_keys_to_console/data.yaml | 31 +-- doc/module-docs/cc_landscape/data.yaml | 55 +++--- doc/module-docs/cc_locale/data.yaml | 12 +- doc/module-docs/cc_lxd/data.yaml | 45 +++-- doc/module-docs/cc_mcollective/data.yaml | 36 ++-- doc/module-docs/cc_mounts/data.yaml | 92 ++++----- doc/module-docs/cc_ntp/data.yaml | 33 ++-- .../data.yaml | 7 +- .../example1.yaml | 9 +- doc/module-docs/cc_phone_home/data.yaml | 43 +++-- doc/module-docs/cc_phone_home/example1.yaml | 4 +- doc/module-docs/cc_phone_home/example2.yaml | 10 +- .../cc_power_state_change/data.yaml | 38 ++-- doc/module-docs/cc_puppet/data.yaml | 35 ++-- doc/module-docs/cc_resizefs/data.yaml | 33 ++-- doc/module-docs/cc_resolv_conf/data.yaml | 43 ++--- doc/module-docs/cc_resolv_conf/example1.yaml | 16 +- doc/module-docs/cc_rh_subscription/data.yaml | 26 ++- doc/module-docs/cc_rsyslog/data.yaml | 28 +++ doc/module-docs/cc_rsyslog/example1.yaml | 4 + doc/module-docs/cc_rsyslog/example2.yaml | 12 ++ doc/module-docs/cc_rsyslog/example3.yaml | 6 + doc/module-docs/cc_runcmd/data.yaml | 27 +++ doc/module-docs/cc_runcmd/example1.yaml | 7 + doc/module-docs/cc_salt_minion/data.yaml | 24 +++ doc/module-docs/cc_salt_minion/example1.yaml | 27 +++ doc/module-docs/cc_scripts_per_boot/data.yaml | 8 + .../cc_scripts_per_instance/data.yaml | 11 ++ doc/module-docs/cc_scripts_per_once/data.yaml | 10 + doc/module-docs/cc_scripts_user/data.yaml | 10 + doc/module-docs/cc_scripts_vendor/data.yaml | 19 ++ .../cc_scripts_vendor/example1.yaml | 2 + .../cc_scripts_vendor/example2.yaml | 4 + .../cc_scripts_vendor/example3.yaml | 2 + doc/module-docs/cc_seed_random/data.yaml | 27 +++ doc/module-docs/cc_seed_random/example1.yaml | 7 + doc/module-docs/cc_seed_random/example2.yaml | 5 + doc/module-docs/cc_set_hostname/data.yaml | 42 +++++ doc/module-docs/cc_set_hostname/example1.yaml | 2 + doc/module-docs/cc_set_hostname/example2.yaml | 5 + doc/module-docs/cc_set_hostname/example3.yaml | 2 + doc/module-docs/cc_set_passwords/data.yaml | 40 ++++ .../cc_set_passwords/example1.yaml | 2 + .../cc_set_passwords/example2.yaml | 8 + doc/module-docs/cc_snap/data.yaml | 42 +++++ doc/module-docs/cc_snap/example1.yaml | 11 ++ doc/module-docs/cc_snap/example2.yaml | 7 + doc/module-docs/cc_snap/example3.yaml | 7 + doc/module-docs/cc_snap/example4.yaml | 6 + doc/module-docs/cc_spacewalk/data.yaml | 16 ++ doc/module-docs/cc_spacewalk/example1.yaml | 2 + doc/module-docs/cc_ssh/data.yaml | 94 +++++++++ doc/module-docs/cc_ssh/example1.yaml | 24 +++ .../cc_ssh_authkey_fingerprints/data.yaml | 14 ++ .../cc_ssh_authkey_fingerprints/example1.yaml | 2 + .../cc_ssh_authkey_fingerprints/example2.yaml | 2 + doc/module-docs/cc_ssh_import_id/data.yaml | 13 ++ .../cc_ssh_import_id/example1.yaml | 2 + doc/module-docs/cc_timezone/data.yaml | 10 + doc/module-docs/cc_timezone/example1.yaml | 2 + doc/module-docs/cc_ubuntu_drivers/data.yaml | 10 + .../cc_ubuntu_drivers/example1.yaml | 3 + doc/module-docs/cc_ubuntu_pro/data.yaml | 52 +++++ doc/module-docs/cc_ubuntu_pro/example1.yaml | 2 + doc/module-docs/cc_ubuntu_pro/example2.yaml | 4 + doc/module-docs/cc_ubuntu_pro/example3.yaml | 5 + doc/module-docs/cc_ubuntu_pro/example4.yaml | 12 ++ doc/module-docs/cc_ubuntu_pro/example5.yaml | 4 + doc/module-docs/cc_ubuntu_pro/example6.yaml | 4 + doc/module-docs/cc_ubuntu_pro/example7.yaml | 3 + doc/module-docs/cc_update_etc_hosts/data.yaml | 55 ++++++ .../cc_update_etc_hosts/example1.yaml | 2 + .../cc_update_etc_hosts/example2.yaml | 2 + .../cc_update_etc_hosts/example3.yaml | 2 + doc/module-docs/cc_update_hostname/data.yaml | 38 ++++ .../cc_update_hostname/example1.yaml | 2 + .../cc_update_hostname/example2.yaml | 2 + .../cc_update_hostname/example3.yaml | 2 + .../cc_update_hostname/example4.yaml | 5 + .../cc_update_hostname/example5.yaml | 2 + .../cc_update_hostname/example6.yaml | 2 + doc/module-docs/cc_users_groups/data.yaml | 107 +++++++++++ doc/module-docs/cc_users_groups/example1.yaml | 2 + doc/module-docs/cc_users_groups/example2.yaml | 4 + doc/module-docs/cc_users_groups/example3.yaml | 9 + doc/module-docs/cc_users_groups/example4.yaml | 8 + doc/module-docs/cc_users_groups/example5.yaml | 4 + doc/module-docs/cc_users_groups/example6.yaml | 4 + doc/module-docs/cc_users_groups/example7.yaml | 3 + doc/module-docs/cc_users_groups/example8.yaml | 2 + doc/module-docs/cc_wireguard/data.yaml | 43 +++++ doc/module-docs/cc_wireguard/example1.yaml | 27 +++ doc/module-docs/cc_write_files/data.yaml | 40 ++++ doc/module-docs/cc_write_files/example1.yaml | 7 + doc/module-docs/cc_write_files/example2.yaml | 6 + doc/module-docs/cc_write_files/example3.yaml | 7 + doc/module-docs/cc_write_files/example4.yaml | 3 + doc/module-docs/cc_write_files/example5.yaml | 15 ++ doc/module-docs/cc_yum_add_repo/data.yaml | 27 +++ doc/module-docs/cc_yum_add_repo/example1.yaml | 5 + doc/module-docs/cc_yum_add_repo/example2.yaml | 10 + doc/module-docs/cc_yum_add_repo/example3.yaml | 10 + doc/module-docs/cc_yum_add_repo/example4.yaml | 8 + doc/module-docs/cc_zypper_add_repo/data.yaml | 23 +++ .../cc_zypper_add_repo/example1.yaml | 8 + doc/rtd/conf.py | 26 ++- doc/rtd/development/docs_layout.rst | 42 ++++- doc/rtd/development/logging.rst | 2 +- doc/rtd/development/module_creation.rst | 178 +++++++++++------- doc/rtd/reference/base_config_reference.rst | 6 +- doc/rtd/reference/datasources/azure.rst | 2 +- doc/rtd/reference/datasources/nocloud.rst | 2 +- doc/rtd/reference/datasources/wsl.rst | 5 +- doc/rtd/reference/modules.rst | 120 ++++++------ doc/rtd/templates/modules.tmpl | 22 ++- doc/rtd/tutorial/lxd.rst | 8 +- doc/rtd/tutorial/qemu.rst | 4 +- doc/rtd/tutorial/wsl.rst | 4 +- tests/unittests/config/test_schema.py | 20 +- tests/unittests/test_cli.py | 2 +- tools/format_yaml_doc | 31 +++ 172 files changed, 2163 insertions(+), 2162 deletions(-) create mode 100644 doc/module-docs/cc_rsyslog/data.yaml create mode 100644 doc/module-docs/cc_rsyslog/example1.yaml create mode 100644 doc/module-docs/cc_rsyslog/example2.yaml create mode 100644 doc/module-docs/cc_rsyslog/example3.yaml create mode 100644 doc/module-docs/cc_runcmd/data.yaml create mode 100644 doc/module-docs/cc_runcmd/example1.yaml create mode 100644 doc/module-docs/cc_salt_minion/data.yaml create mode 100644 doc/module-docs/cc_salt_minion/example1.yaml create mode 100644 doc/module-docs/cc_scripts_per_boot/data.yaml create mode 100644 doc/module-docs/cc_scripts_per_instance/data.yaml create mode 100644 doc/module-docs/cc_scripts_per_once/data.yaml create mode 100644 doc/module-docs/cc_scripts_user/data.yaml create mode 100644 doc/module-docs/cc_scripts_vendor/data.yaml create mode 100644 doc/module-docs/cc_scripts_vendor/example1.yaml create mode 100644 doc/module-docs/cc_scripts_vendor/example2.yaml create mode 100644 doc/module-docs/cc_scripts_vendor/example3.yaml create mode 100644 doc/module-docs/cc_seed_random/data.yaml create mode 100644 doc/module-docs/cc_seed_random/example1.yaml create mode 100644 doc/module-docs/cc_seed_random/example2.yaml create mode 100644 doc/module-docs/cc_set_hostname/data.yaml create mode 100644 doc/module-docs/cc_set_hostname/example1.yaml create mode 100644 doc/module-docs/cc_set_hostname/example2.yaml create mode 100644 doc/module-docs/cc_set_hostname/example3.yaml create mode 100644 doc/module-docs/cc_set_passwords/data.yaml create mode 100644 doc/module-docs/cc_set_passwords/example1.yaml create mode 100644 doc/module-docs/cc_set_passwords/example2.yaml create mode 100644 doc/module-docs/cc_snap/data.yaml create mode 100644 doc/module-docs/cc_snap/example1.yaml create mode 100644 doc/module-docs/cc_snap/example2.yaml create mode 100644 doc/module-docs/cc_snap/example3.yaml create mode 100644 doc/module-docs/cc_snap/example4.yaml create mode 100644 doc/module-docs/cc_spacewalk/data.yaml create mode 100644 doc/module-docs/cc_spacewalk/example1.yaml create mode 100644 doc/module-docs/cc_ssh/data.yaml create mode 100644 doc/module-docs/cc_ssh/example1.yaml create mode 100644 doc/module-docs/cc_ssh_authkey_fingerprints/data.yaml create mode 100644 doc/module-docs/cc_ssh_authkey_fingerprints/example1.yaml create mode 100644 doc/module-docs/cc_ssh_authkey_fingerprints/example2.yaml create mode 100644 doc/module-docs/cc_ssh_import_id/data.yaml create mode 100644 doc/module-docs/cc_ssh_import_id/example1.yaml create mode 100644 doc/module-docs/cc_timezone/data.yaml create mode 100644 doc/module-docs/cc_timezone/example1.yaml create mode 100644 doc/module-docs/cc_ubuntu_drivers/data.yaml create mode 100644 doc/module-docs/cc_ubuntu_drivers/example1.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/data.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/example1.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/example2.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/example3.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/example4.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/example5.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/example6.yaml create mode 100644 doc/module-docs/cc_ubuntu_pro/example7.yaml create mode 100644 doc/module-docs/cc_update_etc_hosts/data.yaml create mode 100644 doc/module-docs/cc_update_etc_hosts/example1.yaml create mode 100644 doc/module-docs/cc_update_etc_hosts/example2.yaml create mode 100644 doc/module-docs/cc_update_etc_hosts/example3.yaml create mode 100644 doc/module-docs/cc_update_hostname/data.yaml create mode 100644 doc/module-docs/cc_update_hostname/example1.yaml create mode 100644 doc/module-docs/cc_update_hostname/example2.yaml create mode 100644 doc/module-docs/cc_update_hostname/example3.yaml create mode 100644 doc/module-docs/cc_update_hostname/example4.yaml create mode 100644 doc/module-docs/cc_update_hostname/example5.yaml create mode 100644 doc/module-docs/cc_update_hostname/example6.yaml create mode 100644 doc/module-docs/cc_users_groups/data.yaml create mode 100644 doc/module-docs/cc_users_groups/example1.yaml create mode 100644 doc/module-docs/cc_users_groups/example2.yaml create mode 100644 doc/module-docs/cc_users_groups/example3.yaml create mode 100644 doc/module-docs/cc_users_groups/example4.yaml create mode 100644 doc/module-docs/cc_users_groups/example5.yaml create mode 100644 doc/module-docs/cc_users_groups/example6.yaml create mode 100644 doc/module-docs/cc_users_groups/example7.yaml create mode 100644 doc/module-docs/cc_users_groups/example8.yaml create mode 100644 doc/module-docs/cc_wireguard/data.yaml create mode 100644 doc/module-docs/cc_wireguard/example1.yaml create mode 100644 doc/module-docs/cc_write_files/data.yaml create mode 100644 doc/module-docs/cc_write_files/example1.yaml create mode 100644 doc/module-docs/cc_write_files/example2.yaml create mode 100644 doc/module-docs/cc_write_files/example3.yaml create mode 100644 doc/module-docs/cc_write_files/example4.yaml create mode 100644 doc/module-docs/cc_write_files/example5.yaml create mode 100644 doc/module-docs/cc_yum_add_repo/data.yaml create mode 100644 doc/module-docs/cc_yum_add_repo/example1.yaml create mode 100644 doc/module-docs/cc_yum_add_repo/example2.yaml create mode 100644 doc/module-docs/cc_yum_add_repo/example3.yaml create mode 100644 doc/module-docs/cc_yum_add_repo/example4.yaml create mode 100644 doc/module-docs/cc_zypper_add_repo/data.yaml create mode 100644 doc/module-docs/cc_zypper_add_repo/example1.yaml create mode 100755 tools/format_yaml_doc diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index a04595bb90a..3edf9972bf9 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -20,31 +20,12 @@ from cloudinit import log, subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS, Distro from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module configures remote system logging using rsyslog. - -Configuration for remote servers can be specified in ``configs``, but for -convenience it can be specified as key value pairs in ``remotes``. - -This module can install rsyslog if not already present on the system using the -``install_rsyslog``, ``packages``, and ``check_exe`` options. Installation -may not work on systems where this module runs before networking is up. - -.. note:: - On BSD cloud-init will attempt to disable and stop the base system syslogd. - This may fail on a first run. - We recommend creating images with ``service syslogd disable``. -""" - meta: MetaSchema = { "id": "cc_rsyslog", - "name": "Rsyslog", - "title": "Configure system logging via rsyslog", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "examples": [ @@ -86,9 +67,7 @@ ), ], "activate_by_schema_keys": ["rsyslog"], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore RSYSLOG_CONFIG = { "config_dir": "/etc/rsyslog.d", diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 3ea1e71b479..2212fd36a97 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -10,12 +10,11 @@ import logging import os -from textwrap import dedent from cloudinit import util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE @@ -25,55 +24,12 @@ # configuration options before actually attempting to deploy with said # configuration. - -MODULE_DESCRIPTION = """\ -Run arbitrary commands at a rc.local like time-frame with output to the -console. Each item can be either a list or a string. The item type affects -how it is executed: - - -* If the item is a string, it will be interpreted by ``sh``. -* If the item is a list, the items will be executed as if passed to execve(3) - (with the first arg as the command). - -Note that the ``runcmd`` module only writes the script to be run -later. The module that actually runs the script is ``scripts_user`` -in the :ref:`Final` boot stage. - -.. note:: - - all commands must be proper yaml, so you have to quote any characters - yaml would eat (':' can be problematic) - -.. note:: - - when writing files, do not use /tmp dir as it races with - systemd-tmpfiles-clean LP: #1707222. Use /run/somedir instead. -""" - meta: MetaSchema = { "id": "cc_runcmd", - "name": "Runcmd", - "title": "Run arbitrary commands", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - runcmd: - - [ ls, -l, / ] - - [ sh, -xc, "echo $(date) ': hello world!'" ] - - [ sh, -c, echo "=========hello world'=========" ] - - ls -l /root - - [ wget, "http://example.org", -O, /tmp/index.html ] - """ - ) - ], "activate_by_schema_keys": ["runcmd"], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 5515a68cc4a..2bfb94f7358 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -6,70 +6,21 @@ import logging import os -from textwrap import dedent from cloudinit import safeyaml, subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module installs, configures and starts salt minion. If the ``salt_minion`` -key is present in the config parts, then salt minion will be installed and -started. Configuration for salt minion can be specified in the ``conf`` key -under ``salt_minion``. Any conf values present there will be assigned in -``/etc/salt/minion``. The public and private keys to use for salt minion can be -specified with ``public_key`` and ``private_key`` respectively. Optionally if -you have a custom package name, service name or config directory you can -specify them with ``pkg_name``, ``service_name`` and ``config_dir``. - -Salt keys can be manually generated by: ``salt-key --gen-keys=GEN_KEYS``, -where ``GEN_KEYS`` is the name of the keypair, e.g. 'minion'. The keypair -will be copied to ``/etc/salt/pki`` on the minion instance. -""" - meta: MetaSchema = { "id": "cc_salt_minion", - "name": "Salt Minion", - "title": "Setup and run salt minion", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - salt_minion: - pkg_name: salt-minion - service_name: salt-minion - config_dir: /etc/salt - conf: - file_client: local - fileserver_backend: - - gitfs - gitfs_remotes: - - https://github.com/_user_/_repo_.git - master: salt.example.com - grains: - role: - - web - public_key: | - ------BEGIN PUBLIC KEY------- - - ------END PUBLIC KEY------- - private_key: | - ------BEGIN PRIVATE KEY------ - - ------END PRIVATE KEY------- - pki_dir: /etc/salt/pki/minion - """ - ) - ], "activate_by_schema_keys": ["salt_minion"], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) # Note: see https://docs.saltstack.com/en/latest/topics/installation/ diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 73ddc231e2b..fa78b8a48d8 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -13,30 +13,19 @@ from cloudinit import subp from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -MODULE_DESCRIPTION = """\ -Any scripts in the ``scripts/per-boot`` directory on the datasource will be run -every time the system boots. Scripts will be run in alphabetical order. This -module does not accept any config keys. -""" - meta: MetaSchema = { "id": "cc_scripts_per_boot", - "name": "Scripts Per Boot", - "title": "Run per boot scripts", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": frequency, - "examples": [], "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) SCRIPT_SUBDIR = "per-boot" diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index c4e10dc6633..c5c52b97cf2 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -13,31 +13,17 @@ from cloudinit import subp from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -Any scripts in the ``scripts/per-instance`` directory on the datasource will -be run when a new instance is first booted. Scripts will be run in alphabetical -order. This module does not accept any config keys. - -Some cloud platforms change instance-id if a significant change was made to -the system. As a result per-instance scripts will run again. -""" - meta: MetaSchema = { "id": "cc_scripts_per_instance", - "name": "Scripts Per Instance", - "title": "Run per instance scripts", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [], "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) SCRIPT_SUBDIR = "per-instance" diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index 82c6f9c1d09..4d9ac583c49 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -13,30 +13,17 @@ from cloudinit import subp from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_ONCE -frequency = PER_ONCE -MODULE_DESCRIPTION = """\ -Any scripts in the ``scripts/per-once`` directory on the datasource will be run -only once. Changes to the instance will not force a re-run. The only way to -re-run these scripts is to run the clean subcommand and reboot. Scripts will -be run in alphabetical order. This module does not accept any config keys. -""" - meta: MetaSchema = { "id": "cc_scripts_per_once", - "name": "Scripts Per Once", - "title": "Run one time scripts", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], - "frequency": frequency, - "examples": [], + "frequency": PER_ONCE, "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) SCRIPT_SUBDIR = "per-once" diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 9c39b75b28c..eb63f7566fc 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -13,34 +13,19 @@ from cloudinit import subp from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module runs all user scripts. User scripts are not specified in the -``scripts`` directory in the datasource, but rather are present in the -``scripts`` dir in the instance configuration. Any cloud-config parts with a -``#!`` will be treated as a script and run. Scripts specified as cloud-config -parts will be run in the order they are specified in the configuration. -This module does not accept any config keys. -""" - meta: MetaSchema = { "id": "cc_scripts_user", - "name": "Scripts User", - "title": "Run user scripts", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [], "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) - SCRIPT_SUBDIR = "scripts" diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index fb8dddad4d3..a102c794ab5 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -7,61 +7,23 @@ import logging import os -from textwrap import dedent from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -On select Datasources, vendors have a channel for the consumption -of all supported user data types via a special channel called -vendor data. Any scripts in the ``scripts/vendor`` directory in the datasource -will be run when a new instance is first booted. Scripts will be run in -alphabetical order. This module allows control over the execution of -vendor data. -""" - meta: MetaSchema = { "id": "cc_scripts_vendor", - "name": "Scripts Vendor", - "title": "Run vendor scripts", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - vendor_data: - enabled: true - prefix: /usr/bin/ltrace - """ - ), - dedent( - """\ - vendor_data: - enabled: true - prefix: [timeout, 30] - """ - ), - dedent( - """\ - # Vendor data will not be processed - vendor_data: - enabled: false - """ - ), - ], "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) - SCRIPT_SUBDIR = "vendor" diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 3e8b8267bb5..edac6dbb9f3 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -11,69 +11,22 @@ import base64 import logging from io import BytesIO -from textwrap import dedent from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) -MODULE_DESCRIPTION = """\ -All cloud instances started from the same image will produce very similar -data when they are first booted as they are all starting with the same seed -for the kernel's entropy keyring. To avoid this, random seed data can be -provided to the instance either as a string or by specifying a command to run -to generate the data. - -Configuration for this module is under the ``random_seed`` config key. If -the cloud provides its own random seed data, it will be appended to ``data`` -before it is written to ``file``. - -If the ``command`` key is specified, the given command will be executed. This -will happen after ``file`` has been populated. That command's environment will -contain the value of the ``file`` key as ``RANDOM_SEED_FILE``. If a command is -specified that cannot be run, no error will be reported unless -``command_required`` is set to true. -""" - meta: MetaSchema = { "id": "cc_seed_random", - "name": "Seed Random", - "title": "Provide random seed data", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - random_seed: - file: /dev/urandom - data: my random string - encoding: raw - command: ['sh', '-c', 'dd if=/dev/urandom of=$RANDOM_SEED_FILE'] - command_required: true - """ - ), - dedent( - """\ - # To use 'pollinate' to gather data from a remote entropy - # server and write it to '/dev/urandom', the following - # could be used: - random_seed: - file: /dev/urandom - command: ["pollinate", "--server=http://local.polinate.server"] - command_required: true - """ - ), - ], "activate_by_schema_keys": [], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore def _decode(data, encoding=None): diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index db39a1a4342..d2c73322fbd 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -9,75 +9,22 @@ import logging import os -from textwrap import dedent from cloudinit import util from cloudinit.atomic_helper import write_json from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -frequency = PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module handles setting the system hostname and fully qualified domain -name (FQDN). If ``preserve_hostname`` is set, then the hostname will not be -altered. - -A hostname and FQDN can be provided by specifying a full domain name under the -``FQDN`` key. Alternatively, a hostname can be specified using the ``hostname`` -key, and the FQDN of the cloud will be used. If a FQDN specified with the -``hostname`` key, it will be handled properly, although it is better to use -the ``fqdn`` config key. If both ``fqdn`` and ``hostname`` are set, -the ``prefer_fqdn_over_hostname`` will force the use of FQDN in all distros -when true, and when false it will force the short hostname. Otherwise, the -hostname to use is distro-dependent. - -.. note:: - cloud-init performs no hostname input validation before sending the - hostname to distro-specific tools, and most tools will not accept a - trailing dot on the FQDN. - -This module will run in the init-local stage before networking is configured -if the hostname is set by metadata or user data on the local system. - -This will occur on datasources like nocloud and ovf where metadata and user -data are available locally. This ensures that the desired hostname is applied -before any DHCP requests are performed on these platforms where dynamic DNS is -based on initial hostname. -""" - meta: MetaSchema = { "id": "cc_set_hostname", - "name": "Set Hostname", - "title": "Set hostname and FQDN", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], - "frequency": frequency, - "examples": [ - "preserve_hostname: true", - dedent( - """\ - hostname: myhost - create_hostname_file: true - fqdn: myhost.example.com - prefer_fqdn_over_hostname: true - """ - ), - dedent( - """\ - # On a machine without an ``/etc/hostname`` file, don't create it - # In most clouds, this will result in a DHCP-configured hostname - # provided by the cloud - create_hostname_file: false - """ - ), - ], + "frequency": PER_INSTANCE, "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 24d8267ad23..21408105c74 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -10,82 +10,22 @@ import logging import re from string import ascii_letters, digits -from textwrap import dedent from typing import List from cloudinit import features, subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS, Distro, ug_util from cloudinit.settings import PER_INSTANCE from cloudinit.ssh_util import update_ssh_config -MODULE_DESCRIPTION = """\ -This module consumes three top-level config keys: ``ssh_pwauth``, ``chpasswd`` -and ``password``. - -The ``ssh_pwauth`` config key determines whether or not sshd will be configured -to accept password authentication. - -The ``chpasswd`` config key accepts a dictionary containing either or both of -``users`` and ``expire``. The ``users`` key is used to assign a password to a -corresponding pre-existing user. The ``expire`` key is used to set -whether to expire all user passwords specified by this module, -such that a password will need to be reset on the user's next login. - -.. note:: - Prior to cloud-init 22.3, the ``expire`` key only applies to plain text - (including ``RANDOM``) passwords. Post 22.3, the ``expire`` key applies to - both plain text and hashed passwords. - -``password`` config key is used to set the default user's password. It is -ignored if the ``chpasswd`` ``users`` is used. Note: the ``list`` keyword is -deprecated in favor of ``users``. -""" - meta: MetaSchema = { "id": "cc_set_passwords", - "name": "Set Passwords", - "title": "Set user passwords and enable/disable SSH password auth", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - # Set a default password that would need to be changed - # at first login - ssh_pwauth: true - password: password1 - """ - ), - dedent( - """\ - # Disable ssh password authentication - # Don't require users to change their passwords on next login - # Set the password for user1 to be 'password1' (OS does hashing) - # Set the password for user2 to a pre-hashed password - # Set the password for user3 to be a randomly generated password, - # which will be written to the system console - ssh_pwauth: false - chpasswd: - expire: false - users: - - name: user1 - password: password1 - type: text - - name: user2 - password: $6$rounds=4096$5DJ8a9WMTEzIo5J4$Yms6imfeBvf3Yfu84mQBerh18l7OR1Wm1BJXZqFSpJ6BVas0AYJqIjP7czkOaAZHZi1kxQ5Y1IhgWN8K9NgxR1 - - name: user3 - type: RANDOM - """ # noqa - ), - ], "activate_by_schema_keys": [], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index bcf16ed3d4f..b2844035c86 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -6,111 +6,22 @@ import logging import os -from textwrap import dedent from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE from cloudinit.subp import prepend_base_command -distros = ["ubuntu"] -frequency = PER_INSTANCE - LOG = logging.getLogger(__name__) meta: MetaSchema = { "id": "cc_snap", - "name": "Snap", - "title": "Install, configure and manage snapd and snap packages", - "description": dedent( - """\ - This module provides a simple configuration namespace in cloud-init to - both setup snapd and install snaps. - - .. note:: - Both ``assertions`` and ``commands`` values can be either a - dictionary or a list. If these configs are provided as a - dictionary, the keys are only used to order the execution of the - assertions or commands and the dictionary is merged with any - vendor-data snap configuration provided. If a list is provided by - the user instead of a dict, any vendor-data snap configuration is - ignored. - - The ``assertions`` configuration option is a dictionary or list of - properly-signed snap assertions which will run before any snap - ``commands``. They will be added to snapd's assertion database by - invoking ``snap ack ``. - - Snap ``commands`` is a dictionary or list of individual snap - commands to run on the target system. These commands can be used to - create snap users, install snaps and provide snap configuration. - - .. note:: - If 'side-loading' private/unpublished snaps on an instance, it is - best to create a snap seed directory and seed.yaml manifest in - **/var/lib/snapd/seed/** which snapd automatically installs on - startup. - """ - ), - "distros": distros, - "examples": [ - dedent( - """\ - snap: - assertions: - 00: | - signed_assertion_blob_here - 02: | - signed_assertion_blob_here - commands: - 00: snap create-user --sudoer --known @mydomain.com - 01: snap install canonical-livepatch - 02: canonical-livepatch enable - """ - ), - dedent( - """\ - # Convenience: the snap command can be omitted when specifying commands - # as a list and 'snap' will automatically be prepended. - # The following commands are equivalent: - snap: - commands: - 00: ['install', 'vlc'] - 01: ['snap', 'install', 'vlc'] - 02: snap install vlc - 03: 'snap install vlc' - """ - ), - dedent( - """\ - # You can use a list of commands - snap: - commands: - - ['install', 'vlc'] - - ['snap', 'install', 'vlc'] - - snap install vlc - - 'snap install vlc' - """ - ), - dedent( - """\ - # You can use a list of assertions - snap: - assertions: - - signed_assertion_blob_here - - | - signed_assertion_blob_here - """ - ), - ], + "distros": ["ubuntu"], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["snap"], -} - - -__doc__ = get_meta_doc(meta) +} # type: ignore SNAP_CMD = "snap" diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py index 08514f2528e..6b364aa938b 100644 --- a/cloudinit/config/cc_spacewalk.py +++ b/cloudinit/config/cc_spacewalk.py @@ -2,48 +2,22 @@ """Spacewalk: Install and configure spacewalk""" import logging -from textwrap import dedent from cloudinit import subp from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module installs spacewalk and applies basic configuration. If the -``spacewalk`` config key is present spacewalk will be installed. The server to -connect to after installation must be provided in the ``server`` in spacewalk -configuration. A proxy to connect through and a activation key may optionally -be specified. - -For more information about spacewalk see: https://fedorahosted.org/spacewalk/ -""" - meta: MetaSchema = { "id": "cc_spacewalk", - "name": "Spacewalk", - "title": "Install and configure spacewalk", - "description": MODULE_DESCRIPTION, "distros": ["rhel", "fedora", "openeuler"], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - spacewalk: - server: - proxy: - activation_key: - """ - ) - ], "activate_by_schema_keys": ["spacewalk"], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) - distros = ["redhat", "fedora"] required_packages = ["rhn-setup"] def_ca_cert_path = "/usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT" diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index d44e1302581..00687cf867d 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -12,150 +12,27 @@ import os import re import sys -from textwrap import dedent from typing import List, Optional, Sequence from cloudinit import ssh_util, subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS, ug_util from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module handles most configuration for SSH and both host and authorized SSH -keys. - -**Authorized keys** - -Authorized keys are a list of public SSH keys that are allowed to connect to -a user account on a system. They are stored in `.ssh/authorized_keys` in that -account's home directory. Authorized keys for the default user defined in -``users`` can be specified using ``ssh_authorized_keys``. Keys -should be specified as a list of public keys. - -.. note:: - See the ``cc_set_passwords`` module documentation to enable/disable SSH - password authentication. - -Root login can be enabled/disabled using the ``disable_root`` config key. Root -login options can be manually specified with ``disable_root_opts``. - -Supported public key types for the ``ssh_authorized_keys`` are: - - - rsa - - ecdsa - - ed25519 - - ecdsa-sha2-nistp256-cert-v01@openssh.com - - ecdsa-sha2-nistp256 - - ecdsa-sha2-nistp384-cert-v01@openssh.com - - ecdsa-sha2-nistp384 - - ecdsa-sha2-nistp521-cert-v01@openssh.com - - ecdsa-sha2-nistp521 - - sk-ecdsa-sha2-nistp256-cert-v01@openssh.com - - sk-ecdsa-sha2-nistp256@openssh.com - - sk-ssh-ed25519-cert-v01@openssh.com - - sk-ssh-ed25519@openssh.com - - ssh-ed25519-cert-v01@openssh.com - - ssh-ed25519 - - ssh-rsa-cert-v01@openssh.com - - ssh-rsa - - ssh-xmss-cert-v01@openssh.com - - ssh-xmss@openssh.com - -.. note:: - this list has been filtered out from the supported keytypes of - `OpenSSH`_ source, where the sigonly keys are removed. Please see - ``ssh_util`` for more information. - - ``rsa``, ``ecdsa`` and ``ed25519`` are added for legacy, - as they are valid public keys in some old distros. They can possibly - be removed in the future when support for the older distros are dropped - -.. _OpenSSH: https://github.com/openssh/openssh-portable/blob/master/sshkey.c - -**Host keys** - -Host keys are for authenticating a specific instance. Many images have default -host SSH keys, which can be removed using ``ssh_deletekeys``. - -Host keys can be added using the ``ssh_keys`` configuration key. - -When host keys are generated the output of the ssh-keygen command(s) can be -displayed on the console using the ``ssh_quiet_keygen`` configuration key. - -.. note:: - When specifying private host keys in cloud-config, care should be taken to - ensure that the communication between the data source and the instance is - secure. - - -If no host keys are specified using ``ssh_keys``, then keys will be generated -using ``ssh-keygen``. By default one public/private pair of each supported -host key type will be generated. The key types to generate can be specified -using the ``ssh_genkeytypes`` config flag, which accepts a list of host key -types to use. For each host key type for which this module has been instructed -to create a keypair, if a key of the same type is already present on the -system (i.e. if ``ssh_deletekeys`` was false), no key will be generated. - -Supported host key types for the ``ssh_keys`` and the ``ssh_genkeytypes`` -config flags are: - - - ecdsa - - ed25519 - - rsa - -Unsupported host key types for the ``ssh_keys`` and the ``ssh_genkeytypes`` -config flags are: - - - ecdsa-sk - - ed25519-sk -""" - # Note: We do not support *-sk key types because: # 1) In the autogeneration case user interaction with the device is needed # which does not fit with a cloud-context. # 2) This type of keys are user-based, not hostkeys. - meta: MetaSchema = { "id": "cc_ssh", - "name": "SSH", - "title": "Configure SSH and SSH keys", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - ssh_keys: - rsa_private: | - -----BEGIN RSA PRIVATE KEY----- - MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qco - ... - -----END RSA PRIVATE KEY----- - rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ... - rsa_certificate: | - ssh-rsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt ... - ssh_authorized_keys: - - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEA3FSyQwBI6Z+nCSjUU ... - - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZ ... - ssh_deletekeys: true - ssh_genkeytypes: [rsa, ecdsa, ed25519] - disable_root: true - disable_root_opts: no-port-forwarding,no-agent-forwarding,no-X11-forwarding - allow_public_ssh_keys: true - ssh_quiet_keygen: true - ssh_publish_hostkeys: - enabled: true - blacklist: [rsa] - """ # noqa: E501 - ) - ], "activate_by_schema_keys": [], -} +} # type:ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) GENERATE_KEY_NAMES = ["rsa", "ecdsa", "ed25519"] diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 31c2717f9b7..106b3cbd0c3 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -12,32 +12,18 @@ from cloudinit import ssh_util, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS, ug_util from cloudinit.settings import PER_INSTANCE from cloudinit.simpletable import SimpleTable -MODULE_DESCRIPTION = """\ -Write fingerprints of authorized keys for each user to log. This is enabled by -default, but can be disabled using ``no_ssh_fingerprints``. The hash type for -the keys can be specified, but defaults to ``sha256``. -""" - meta: MetaSchema = { "id": "cc_ssh_authkey_fingerprints", - "name": "SSH AuthKey Fingerprints", - "title": "Log fingerprints of user SSH keys", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [ - "no_ssh_fingerprints: true", - "authkey_hash: sha512", - ], "activate_by_schema_keys": [], -} +} # type:ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index a4ca1b981f7..8abf3914fc6 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -9,47 +9,25 @@ import logging import pwd -from textwrap import dedent from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ug_util from cloudinit.settings import PER_INSTANCE # https://launchpad.net/ssh-import-id -distros = ["alpine", "cos", "debian", "ubuntu"] SSH_IMPORT_ID_BINARY = "ssh-import-id" -MODULE_DESCRIPTION = """\ -This module imports SSH keys from either a public keyserver, usually launchpad -or github using ``ssh-import-id``. Keys are referenced by the username they are -associated with on the keyserver. The keyserver can be specified by prepending -either ``lp:`` for launchpad or ``gh:`` for github to the username. -""" meta: MetaSchema = { "id": "cc_ssh_import_id", - "name": "SSH Import ID", - "title": "Import SSH id", - "description": MODULE_DESCRIPTION, - "distros": distros, + "distros": ["alpine", "cos", "debian", "ubuntu"], "frequency": PER_INSTANCE, - "examples": [ - dedent( - """\ - ssh_import_id: - - user - - gh:user - - lp:user - """ - ) - ], "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index 5d3c04d3067..db9d168b1e2 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -12,29 +12,20 @@ from cloudinit import util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -Sets the system `timezone `_ based on the -value provided. -""" - meta: MetaSchema = { "id": "cc_timezone", - "name": "Timezone", - "title": "Set the system timezone", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "examples": [ "timezone: US/Eastern", ], "activate_by_schema_keys": ["timezone"], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py index faa82a4d751..b4c5e4e1a4d 100644 --- a/cloudinit/config/cc_ubuntu_drivers.py +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -4,7 +4,6 @@ import logging import os -from textwrap import dedent from cloudinit.cloud import Cloud from cloudinit.distros import Distro @@ -20,37 +19,17 @@ from cloudinit import subp, temp_utils, type_utils, util from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) -distros = ["ubuntu"] - meta: MetaSchema = { "id": "cc_ubuntu_drivers", - "name": "Ubuntu Drivers", - "title": "Interact with third party drivers in Ubuntu.", - "description": dedent( - """\ - This module interacts with the 'ubuntu-drivers' command to install - third party driver packages.""" - ), - "distros": distros, - "examples": [ - dedent( - """\ - drivers: - nvidia: - license-accepted: true - """ - ) - ], + "distros": ["ubuntu"], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["drivers"], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = ( "ubuntu-drivers: error: argument : invalid choice: 'install'" diff --git a/cloudinit/config/cc_ubuntu_pro.py b/cloudinit/config/cc_ubuntu_pro.py index de372d635b3..a82c1b8d538 100644 --- a/cloudinit/config/cc_ubuntu_pro.py +++ b/cloudinit/config/cc_ubuntu_pro.py @@ -5,134 +5,24 @@ import json import logging import re -from textwrap import dedent from typing import Any, List from urllib.parse import urlparse from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE PRO_URL = "https://ubuntu.com/pro" - -distros = ["ubuntu"] - DEPRECATED_KEYS = set(["ubuntu-advantage", "ubuntu_advantage"]) meta: MetaSchema = { "id": "cc_ubuntu_pro", - "name": "Ubuntu Pro", - "title": "Configure Ubuntu Pro support services", - "description": dedent( - """\ - Attach machine to an existing Ubuntu Pro support contract and - enable or disable support services such as Livepatch, ESM, - FIPS and FIPS Updates. When attaching a machine to Ubuntu Pro, - one can also specify services to enable. When the 'enable' - list is present, only named services will be activated. Whereas - if the 'enable' list is not present, the contract's default - services will be enabled. - - On Pro instances, when ``ubuntu_pro`` config is provided to - cloud-init, Pro's auto-attach feature will be disabled and cloud-init - will perform the Pro auto-attach ignoring the ``token`` key. - The ``enable`` and ``enable_beta`` values will strictly determine what - services will be enabled, ignoring contract defaults. - - Note that when enabling FIPS or FIPS updates you will need to schedule - a reboot to ensure the machine is running the FIPS-compliant kernel. - See `Power State Change`_ for information on how to configure - cloud-init to perform this reboot. - """ - ), - "distros": distros, - "examples": [ - dedent( - """\ - # Attach the machine to an Ubuntu Pro support contract with a - # Pro contract token obtained from %s. - ubuntu_pro: - token: - """ - % PRO_URL - ), - dedent( - """\ - # Attach the machine to an Ubuntu Pro support contract enabling - # only fips and esm services. Services will only be enabled if - # the environment supports said service. Otherwise warnings will - # be logged for incompatible services specified. - ubuntu_pro: - token: - enable: - - fips - - esm - """ - ), - dedent( - """\ - # Attach the machine to an Ubuntu Pro support contract and enable - # the FIPS service. Perform a reboot once cloud-init has - # completed. - power_state: - mode: reboot - ubuntu_pro: - token: - enable: - - fips - """ - ), - dedent( - """\ - # Set a http(s) proxy before attaching the machine to an - # Ubuntu Pro support contract and enabling the FIPS service. - ubuntu_pro: - token: - config: - http_proxy: 'http://some-proxy:8088' - https_proxy: 'https://some-proxy:8088' - global_apt_https_proxy: 'https://some-global-apt-proxy:8088/' - global_apt_http_proxy: 'http://some-global-apt-proxy:8088/' - ua_apt_http_proxy: 'http://10.0.10.10:3128' - ua_apt_https_proxy: 'https://10.0.10.10:3128' - enable: - - fips - """ - ), - dedent( - """\ - # On Ubuntu PRO instances, auto-attach but enable no PRO services. - ubuntu_pro: - enable: [] - enable_beta: [] - """ - ), - dedent( - """\ - # Enable esm and beta realtime-kernel services in Ubuntu Pro instances. - ubuntu_pro: - enable: - - esm - enable_beta: - - realtime-kernel - """ - ), - dedent( - """\ - # Disable auto-attach in Ubuntu Pro instances. - ubuntu_pro: - features: - disable_auto_attach: True - """ - ), - ], + "distros": ["ubuntu"], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["ubuntu_pro"] + list(DEPRECATED_KEYS), -} - -__doc__ = get_meta_doc(meta) +} # type: ignore LOG = logging.getLogger(__name__) REDACTED = "REDACTED" diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 7ac0d955ea2..45bb2df7d4b 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -9,92 +9,20 @@ """Update Etc Hosts: Update the hosts file (usually ``/etc/hosts``)""" import logging -from textwrap import dedent from cloudinit import templater, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_ALWAYS -MODULE_DESCRIPTION = """\ -This module will update the contents of the local hosts database (hosts file; -usually ``/etc/hosts``) based on the hostname/fqdn specified in config. -Management of the hosts file is controlled using ``manage_etc_hosts``. If this -is set to false, cloud-init will not manage the hosts file at all. This is the -default behavior. - -If set to ``true``, cloud-init will generate the hosts file -using the template located in ``/etc/cloud/templates/hosts.tmpl``. In the -``/etc/cloud/templates/hosts.tmpl`` template, the strings ``$hostname`` and -``$fqdn`` will be replaced with the hostname and fqdn respectively. - -If ``manage_etc_hosts`` is set to ``localhost``, then cloud-init will not -rewrite the hosts file entirely, but rather will ensure that a entry for the -fqdn with a distribution dependent ip is present (i.e. ``ping `` will -ping ``127.0.0.1`` or ``127.0.1.1`` or other ip). - -.. note:: - if ``manage_etc_hosts`` is set ``true``, the contents - of the hosts file will be updated every boot. To make any changes to - the hosts file persistent they must be made in - ``/etc/cloud/templates/hosts.tmpl`` - -.. note:: - for instructions on specifying hostname and fqdn, see documentation for - ``cc_set_hostname`` -""" - -distros = ["all"] - meta: MetaSchema = { "id": "cc_update_etc_hosts", - "name": "Update Etc Hosts", - "title": "Update the hosts file (usually ``/etc/hosts``)", - "description": MODULE_DESCRIPTION, - "distros": distros, - "examples": [ - dedent( - """\ - # Do not update or manage /etc/hosts at all. This is the default behavior. - # - # Whatever is present at instance boot time will be present after boot. - # User changes will not be overwritten. - manage_etc_hosts: false - """ - ), - dedent( - """\ - # Manage /etc/hosts with cloud-init. - # On every boot, /etc/hosts will be re-written from - # ``/etc/cloud/templates/hosts.tmpl``. - # - # The strings '$hostname' and '$fqdn' are replaced in the template - # with the appropriate values either from the config-config ``fqdn`` or - # ``hostname`` if provided. When absent, the cloud metadata will be - # checked for ``local-hostname` which can be split into .. - # - # To make modifications persistent across a reboot, you must modify - # ``/etc/cloud/templates/hosts.tmpl``. - manage_etc_hosts: true - """ - ), - dedent( - """\ - # Update /etc/hosts every boot providing a "localhost" 127.0.1.1 entry - # with the latest hostname and fqdn as provided by either IMDS or - # cloud-config. - # All other entries will be left as is. - # 'ping `hostname`' will ping 127.0.1.1 - manage_etc_hosts: localhost - """ - ), - ], + "distros": ["all"], "frequency": PER_ALWAYS, "activate_by_schema_keys": ["manage_etc_hosts"], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 01b570825c7..7fb3587ec70 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -10,85 +10,20 @@ import logging import os -from textwrap import dedent from cloudinit import util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_ALWAYS -MODULE_DESCRIPTION = """\ -This module will update the system hostname and fqdn. If ``preserve_hostname`` -is set ``true``, then the hostname will not be altered. - -.. note:: - for instructions on specifying hostname and fqdn, see documentation for - ``cc_set_hostname`` -""" - -distros = ["all"] - meta: MetaSchema = { "id": "cc_update_hostname", - "name": "Update Hostname", - "title": "Update hostname and fqdn", - "description": MODULE_DESCRIPTION, - "distros": distros, - "examples": [ - dedent( - """\ - # By default: when ``preserve_hostname`` is not specified cloud-init - # updates ``/etc/hostname`` per-boot based on the cloud provided - # ``local-hostname`` setting. If you manually change ``/etc/hostname`` - # after boot cloud-init will no longer modify it. - # - # This default cloud-init behavior is equivalent to this cloud-config: - preserve_hostname: false - """ - ), - dedent( - """\ - # Prevent cloud-init from updating the system hostname. - preserve_hostname: true - """ - ), - dedent( - """\ - # Prevent cloud-init from updating ``/etc/hostname`` - preserve_hostname: true - """ - ), - dedent( - """\ - # Set hostname to "external.fqdn.me" instead of "myhost" - fqdn: external.fqdn.me - hostname: myhost - prefer_fqdn_over_hostname: true - create_hostname_file: true - """ - ), - dedent( - """\ - # Set hostname to "external" instead of "external.fqdn.me" when - # cloud metadata provides the ``local-hostname``: "external.fqdn.me". - prefer_fqdn_over_hostname: false - """ - ), - dedent( - """\ - # On a machine without an ``/etc/hostname`` file, don't create it - # In most clouds, this will result in a DHCP-configured hostname - # provided by the cloud - create_hostname_file: false - """ - ), - ], + "distros": ["all"], "frequency": PER_ALWAYS, "activate_by_schema_keys": [], -} +} # type: ignore -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 2a87341ebf8..ace17733c3a 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -7,7 +7,6 @@ """Users and Groups: Configure users and groups""" import logging -from textwrap import dedent from cloudinit.cloud import Cloud @@ -15,175 +14,16 @@ # since the module attribute 'distros' # is a list of distros that are supported, not a sub-module from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.distros import ug_util from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module configures users and groups. For more detailed information on user -options, see the :ref:`Including users and groups` config -example. - -Groups to add to the system can be specified under the ``groups`` key as -a string of comma-separated groups to create, or a list. Each item in -the list should either contain a string of a single group to create, -or a dictionary with the group name as the key and string of a single user as -a member of that group or a list of users who should be members of the group. - -.. note:: - Groups are added before users, so any users in a group list must - already exist on the system. - -Users to add can be specified as a string or list under the ``users`` key. -Each entry in the list should either be a string or a dictionary. If a string -is specified, that string can be comma-separated usernames to create or the -reserved string ``default`` which represents the primary admin user used to -access the system. The ``default`` user varies per distribution and is -generally configured in ``/etc/cloud/cloud.cfg`` by the ``default_user`` key. - -Each ``users`` dictionary item must contain either a ``name`` or ``snapuser`` -key, otherwise it will be ignored. Omission of ``default`` as the first item -in the ``users`` list skips creation the default user. If no ``users`` key is -provided the default behavior is to create the default user via this config:: - - users: - - default - -.. note:: - Specifying a hash of a user's password with ``passwd`` is a security risk - if the cloud-config can be intercepted. SSH authentication is preferred. - -.. note:: - If specifying a doas rule for a user, ensure that the syntax for the rule - is valid, as the only checking performed by cloud-init is to ensure that - the user referenced in the rule is the correct user. - -.. note:: - If specifying a sudo rule for a user, ensure that the syntax for the rule - is valid, as it is not checked by cloud-init. - -.. note:: - Most of these configuration options will not be honored if the user - already exists. The following options are the exceptions; they are applied - to already-existing users: ``plain_text_passwd``, ``doas``, - ``hashed_passwd``, ``lock_passwd``, ``sudo``, ``ssh_authorized_keys``, - ``ssh_redirect_user``. - -The ``user`` key can be used to override the ``default_user`` configuration -defined in ``/etc/cloud/cloud.cfg``. The ``user`` value should be a dictionary -which supports the same config keys as the ``users`` dictionary items. -""" - meta: MetaSchema = { "id": "cc_users_groups", - "name": "Users and Groups", - "title": "Configure users and groups", - "description": MODULE_DESCRIPTION, "distros": ["all"], - "examples": [ - dedent( - """\ - # Add the ``default_user`` from /etc/cloud/cloud.cfg. - # This is also the default behavior of cloud-init when no `users` key - # is provided. - users: - - default - """ - ), - dedent( - """\ - # Add the 'admingroup' with members 'root' and 'sys' and an empty - # group cloud-users. - groups: - - admingroup: [root,sys] - - cloud-users - """ - ), - dedent( - """\ - # Skip creation of the user and only create newsuper. - # Password-based login is rejected, but the github user TheRealFalcon - # and the launchpad user falcojr can SSH as newsuper. The default - # shell for newsuper is bash instead of system default. - users: - - name: newsuper - gecos: Big Stuff - groups: users, admin - sudo: ALL=(ALL) NOPASSWD:ALL - shell: /bin/bash - lock_passwd: true - ssh_import_id: - - lp:falcojr - - gh:TheRealFalcon - """ - ), - dedent( - """\ - # Skip creation of the user and only create newsuper. - # Password-based login is rejected, but the github user TheRealFalcon - # and the launchpad user falcojr can SSH as newsuper. doas/opendoas - # is configured to permit this user to run commands as other users - # (without being prompted for a password) except not as root. - users: - - name: newsuper - gecos: Big Stuff - groups: users, admin - doas: - - permit nopass newsuper - - deny newsuper as root - lock_passwd: true - ssh_import_id: - - lp:falcojr - - gh:TheRealFalcon - """ - ), - dedent( - """\ - # On a system with SELinux enabled, add youruser and set the - # SELinux user to 'staff_u'. When omitted on SELinux, the system will - # select the configured default SELinux user. - users: - - default - - name: youruser - selinux_user: staff_u - """ - ), - dedent( - """\ - # To redirect a legacy username to the user for a - # distribution, ssh_redirect_user will accept an SSH connection and - # emit a message telling the client to ssh as the user. - # SSH clients will get the message: - users: - - default - - name: nosshlogins - ssh_redirect_user: true - """ - ), - dedent( - """\ - # Override any ``default_user`` config in /etc/cloud/cloud.cfg with - # supplemental config options. - # This config will make the default user to mynewdefault and change - # the user to not have sudo rights. - ssh_import_id: [chad.smith] - user: - name: mynewdefault - sudo: null - """ - ), - dedent( - """\ - # Avoid creating any ``default_user``. - users: [] - """ - ), - ], "frequency": PER_INSTANCE, "activate_by_schema_keys": [], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_wireguard.py b/cloudinit/config/cc_wireguard.py index e78c3e77785..0cacd1338e0 100644 --- a/cloudinit/config/cc_wireguard.py +++ b/cloudinit/config/cc_wireguard.py @@ -5,93 +5,19 @@ import logging import re -from textwrap import dedent from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = dedent( - """\ -Wireguard module provides a dynamic interface for configuring -Wireguard (as a peer or server) in an easy way. - -This module takes care of: - - writing interface configuration files - - enabling and starting interfaces - - installing wireguard-tools package - - loading wireguard kernel module - - executing readiness probes - -What's a readiness probe?\n -The idea behind readiness probes is to ensure Wireguard connectivity -before continuing the cloud-init process. This could be useful if you -need access to specific services like an internal APT Repository Server -(e.g Landscape) to install/update packages. - -Example:\n -An edge device can't access the internet but uses cloud-init modules which -will install packages (e.g landscape, packages, ubuntu_advantage). Those -modules will fail due to missing internet connection. The "wireguard" module -fixes that problem as it waits until all readinessprobes (which can be -arbitrary commands - e.g. checking if a proxy server is reachable over -Wireguard network) are finished before continuing the cloud-init -"config" stage. - -.. note:: - In order to use DNS with Wireguard you have to install ``resolvconf`` - package or symlink it to systemd's ``resolvectl``, otherwise ``wg-quick`` - commands will throw an error message that executable ``resolvconf`` is - missing which leads wireguard module to fail. -""" -) - meta: MetaSchema = { "id": "cc_wireguard", - "name": "Wireguard", - "title": "Module to configure Wireguard tunnel", - "description": MODULE_DESCRIPTION, "distros": ["ubuntu"], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["wireguard"], - "examples": [ - dedent( - """\ - # Configure one or more WG interfaces and provide optional readinessprobes - wireguard: - interfaces: - - name: wg0 - config_path: /etc/wireguard/wg0.conf - content: | - [Interface] - PrivateKey = - Address =
- [Peer] - PublicKey = - Endpoint = : - AllowedIPs = , , ... - - name: wg1 - config_path: /etc/wireguard/wg1.conf - content: | - [Interface] - PrivateKey = - Address =
- [Peer] - PublicKey = - Endpoint = : - AllowedIPs = - readinessprobe: - - 'systemctl restart service' - - 'curl https://webhook.endpoint/example' - - 'nc -zv some-service-fqdn 443' - """ - ), - ], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index bcd51c62db5..c97d1225afc 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -9,12 +9,11 @@ import base64 import logging import os -from textwrap import dedent from cloudinit import util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE DEFAULT_PERMS = 0o644 @@ -25,97 +24,10 @@ meta: MetaSchema = { "id": "cc_write_files", - "name": "Write Files", - "title": "write arbitrary files", - "description": dedent( - """\ - Write out arbitrary content to files, optionally setting permissions. - Parent folders in the path are created if absent. - Content can be specified in plain text or binary. Data encoded with - either base64 or binary gzip data can be specified and will be decoded - before being written. For empty file creation, content can be omitted. - - .. note:: - If multiline data is provided, care should be taken to ensure that it - follows yaml formatting standards. To specify binary data, use the yaml - option ``!!binary`` - - .. note:: - Do not write files under /tmp during boot because of a race with - systemd-tmpfiles-clean that can cause temp files to get cleaned during - the early boot process. Use /run/somedir instead to avoid race - LP:1707222. - - .. warning:: - Existing files will be overridden.""" - ), "distros": ["all"], - "examples": [ - dedent( - """\ - # Write out base64 encoded content to /etc/sysconfig/selinux - write_files: - - encoding: b64 - content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4... - owner: root:root - path: /etc/sysconfig/selinux - permissions: '0644' - """ - ), - dedent( - """\ - # Appending content to an existing file - write_files: - - content: | - 15 * * * * root ship_logs - path: /etc/crontab - append: true - """ - ), - dedent( - """\ - # Provide gzipped binary content - write_files: - - encoding: gzip - content: !!binary | - H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= - path: /usr/bin/hello - permissions: '0755' - """ - ), - dedent( - """\ - # Create an empty file on the system - write_files: - - path: /root/CLOUD_INIT_WAS_HERE - """ - ), - dedent( - """\ - # Defer writing the file until after the package (Nginx) is - # installed and its user is created alongside - write_files: - - path: /etc/nginx/conf.d/example.com.conf - content: | - server { - server_name example.com; - listen 80; - root /var/www; - location / { - try_files $uri $uri/ $uri.html =404; - } - } - owner: 'nginx:nginx' - permissions: '0640' - defer: true - """ - ), - ], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["write_files"], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: diff --git a/cloudinit/config/cc_write_files_deferred.py b/cloudinit/config/cc_write_files_deferred.py index d18fee7e5a5..0dc0662e1e7 100644 --- a/cloudinit/config/cc_write_files_deferred.py +++ b/cloudinit/config/cc_write_files_deferred.py @@ -14,28 +14,14 @@ from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -This module is based on `'Write Files' `__, and -will handle all files from the write_files list, that have been -marked as deferred and thus are not being processed by the -write_files module. - -*Please note that his module is not exposed to the user through -its own dedicated top-level directive.* -""" meta: MetaSchema = { "id": "cc_write_files_deferred", - "name": "Write Files Deferred", - "title": "Defer writing certain files", - "description": __doc__, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, - "examples": [], "activate_by_schema_keys": ["write_files"], -} +} # type: ignore # This module is undocumented in our schema docs -__doc__ = "" LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 879e25c85c2..9a717af3d1a 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -10,39 +10,13 @@ import logging import os from configparser import ConfigParser -from textwrap import dedent from cloudinit import util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_INSTANCE -MODULE_DESCRIPTION = """\ -Add yum repository configuration to ``/etc/yum.repos.d``. Configuration files -are named based on the opaque dictionary key under the ``yum_repos`` they are -specified with. If a config file already exists with the same name as a config -entry, the config entry will be skipped. -""" - -distros = [ - "almalinux", - "azurelinux", - "centos", - "cloudlinux", - "eurolinux", - "fedora", - "mariner", - "openeuler", - "OpenCloudOS", - "openmandriva", - "photon", - "rhel", - "rocky", - "TencentOS", - "virtuozzo", -] - COPR_BASEURL = ( "https://download.copr.fedorainfracloud.org/results/@cloud-init/" "cloud-init-dev/epel-8-$basearch/" @@ -58,72 +32,28 @@ meta: MetaSchema = { "id": "cc_yum_add_repo", - "name": "Yum Add Repo", - "title": "Add yum repository configuration to the system", - "description": MODULE_DESCRIPTION, - "distros": distros, - "examples": [ - dedent( - """\ - yum_repos: - my_repo: - baseurl: http://blah.org/pub/epel/testing/5/$basearch/ - yum_repo_dir: /store/custom/yum.repos.d - """ - ), - dedent( - f"""\ - # Enable cloud-init upstream's daily testing repo for EPEL 8 to - # install latest cloud-init from tip of `main` for testing. - yum_repos: - cloud-init-daily: - name: Copr repo for cloud-init-dev owned by @cloud-init - baseurl: {COPR_BASEURL} - type: rpm-md - skip_if_unavailable: true - gpgcheck: true - gpgkey: {COPR_GPG_URL} - enabled_metadata: 1 - """ - ), - dedent( - f"""\ - # Add the file /etc/yum.repos.d/epel_testing.repo which can then - # subsequently be used by yum for later operations. - yum_repos: - # The name of the repository - epel-testing: - baseurl: {EPEL_TESTING_BASEURL} - enabled: false - failovermethod: priority - gpgcheck: true - gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL - name: Extra Packages for Enterprise Linux 5 - Testing - """ - ), - dedent( - """\ - # Any yum repo configuration can be passed directly into - # the repository file created. See: man yum.conf for supported - # config keys. - # - # Write /etc/yum.conf.d/my-package-stream.repo with gpgkey checks - # on the repo data of the repository enabled. - yum_repos: - my package stream: - baseurl: http://blah.org/pub/epel/testing/5/$basearch/ - mirrorlist: http://some-url-to-list-of-baseurls - repo_gpgcheck: 1 - enable_gpgcheck: true - gpgkey: https://url.to.ascii-armored-gpg-key - """ - ), + "distros": [ + "almalinux", + "azurelinux", + "centos", + "cloudlinux", + "eurolinux", + "fedora", + "mariner", + "openeuler", + "OpenCloudOS", + "openmandriva", + "photon", + "rhel", + "rocky", + "TencentOS", + "virtuozzo", ], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["yum_repos"], -} +} # type: ignore + -__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_zypper_add_repo.py b/cloudinit/config/cc_zypper_add_repo.py index c82a1b4858e..02b372dcc70 100644 --- a/cloudinit/config/cc_zypper_add_repo.py +++ b/cloudinit/config/cc_zypper_add_repo.py @@ -7,76 +7,29 @@ import logging import os -from textwrap import dedent import configobj from cloudinit import util from cloudinit.cloud import Cloud from cloudinit.config import Config -from cloudinit.config.schema import MetaSchema, get_meta_doc +from cloudinit.config.schema import MetaSchema from cloudinit.settings import PER_ALWAYS -distros = [ - "opensuse", - "opensuse-microos", - "opensuse-tumbleweed", - "opensuse-leap", - "sle_hpc", - "sle-micro", - "sles", -] - -MODULE_DESCRIPTION = """\ -Zypper behavior can be configured using the ``config`` key, which will modify -``/etc/zypp/zypp.conf``. The configuration writer will only append the -provided configuration options to the configuration file. Any duplicate -options will be resolved by the way the zypp.conf INI file is parsed. - -.. note:: - Setting ``configdir`` is not supported and will be skipped. - -The ``repos`` key may be used to add repositories to the system. Beyond the -required ``id`` and ``baseurl`` attributions, no validation is performed -on the ``repos`` entries. It is assumed the user is familiar with the -zypper repository file format. This configuration is also applicable for -systems with transactional-updates. -""" meta: MetaSchema = { "id": "cc_zypper_add_repo", - "name": "Zypper Add Repo", - "title": "Configure zypper behavior and add zypper repositories", - "description": MODULE_DESCRIPTION, - "distros": distros, - "examples": [ - dedent( - """\ - zypper: - repos: - - id: opensuse-oss - name: os-oss - baseurl: http://dl.opensuse.org/dist/leap/v/repo/oss/ - enabled: 1 - autorefresh: 1 - - id: opensuse-oss-update - name: os-oss-up - baseurl: http://dl.opensuse.org/dist/leap/v/update - # any setting per - # https://en.opensuse.org/openSUSE:Standards_RepoInfo - # enable and autorefresh are on by default - config: - reposdir: /etc/zypp/repos.dir - servicesdir: /etc/zypp/services.d - download.use_deltarpm: true - # any setting in /etc/zypp/zypp.conf - """ - ) + "distros": [ + "opensuse", + "opensuse-microos", + "opensuse-tumbleweed", + "opensuse-leap", + "sle_hpc", + "sle-micro", + "sles", ], "frequency": PER_ALWAYS, "activate_by_schema_keys": ["zypper"], -} - -__doc__ = get_meta_doc(meta) +} # type: ignore LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index e552483e543..cf7fd10763e 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -107,8 +107,6 @@ "{prefix}* Each object in **{prop_name}** list supports " "the following keys:" ) -SCHEMA_EXAMPLES_HEADER = "" -SCHEMA_EXAMPLES_SPACER_TEMPLATE = "\n # --- Example{example_count} ---\n\n" DEPRECATED_KEY = "deprecated" # user-data files typically must begin with a leading '#' @@ -124,8 +122,8 @@ from typing_extensions import NotRequired, TypedDict class MetaSchema(TypedDict): - name: str id: str + name: str title: str description: str distros: typing.List[str] @@ -1488,11 +1486,8 @@ def _get_examples(meta: MetaSchema) -> str: examples = meta.get("examples") if not examples: return "" - rst_content = SCHEMA_EXAMPLES_HEADER - for count, example in enumerate(examples, 1): - rst_content += SCHEMA_EXAMPLES_SPACER_TEMPLATE.format( - example_count=count - ) + rst_content: str = "" + for example in examples: # FIXME(drop conditional when all mods have rtd/module-doc/*/data.yaml) if isinstance(example, dict): if example["comment"]: diff --git a/doc/module-docs/cc_ansible/data.yaml b/doc/module-docs/cc_ansible/data.yaml index bceff056ef3..ca5ac3bfbaf 100644 --- a/doc/module-docs/cc_ansible/data.yaml +++ b/doc/module-docs/cc_ansible/data.yaml @@ -1,17 +1,16 @@ cc_ansible: - name: Ansible - title: Configure ansible for instance - description: - This module provides ``ansible`` integration for - augmenting cloud-init's configuration of the local - node. - + description: > + This module provides Ansible integration for augmenting cloud-init's + configuration of the local node. - This module installs ansible during boot and - then uses ``ansible-pull`` to run the playbook - repository at the remote URL. + This module installs ``ansible`` during boot and then uses ``ansible-pull`` + to run the playbook repository at the remote URL. examples: - - comment: "" + - comment: | + Example 1: file: cc_ansible/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_ansible/example2.yaml + name: Ansible + title: Configure Ansible for instance diff --git a/doc/module-docs/cc_apk_configure/data.yaml b/doc/module-docs/cc_apk_configure/data.yaml index 76c8a88d369..89385616656 100644 --- a/doc/module-docs/cc_apk_configure/data.yaml +++ b/doc/module-docs/cc_apk_configure/data.yaml @@ -1,16 +1,23 @@ cc_apk_configure: - name: APK Configure - title: Configure apk repositories file description: | - This module handles configuration of the /etc/apk/repositories file. + This module handles configuration of the Alpine Package Keeper (APK) + ``/etc/apk/repositories`` file. - .. note:: - To ensure that apk configuration is valid yaml, any strings - containing special characters, especially ``:`` should be quoted. + .. note:: + To ensure that APK configuration is valid YAML, any strings + containing special characters, especially colons, should be quoted + (":"). examples: - - comment: Keep the existing /etc/apk/repositories file unaltered. + - comment: > + Example 1: Keep the existing ``/etc/apk/repositories`` file unaltered. file: cc_apk_configure/example1.yaml - - comment: Create repositories file for Alpine v3.12 main and community using default mirror site. + - comment: > + Example 2: Create repositories file for Alpine v3.12 main and community + using default mirror site. file: cc_apk_configure/example2.yaml - - comment: Create repositories file for Alpine Edge main, community, and testing using a specified mirror site and also a local repo. + - comment: > + Example 3: Create repositories file for Alpine Edge main, community, and + testing using a specified mirror site and also a local repo. file: cc_apk_configure/example3.yaml + name: APK Configure + title: Configure APK repositories file diff --git a/doc/module-docs/cc_apt_configure/data.yaml b/doc/module-docs/cc_apt_configure/data.yaml index 3e28e3f7b64..9d5509d1f2a 100644 --- a/doc/module-docs/cc_apt_configure/data.yaml +++ b/doc/module-docs/cc_apt_configure/data.yaml @@ -1,31 +1,34 @@ cc_apt_configure: - name: Apt Configure - title: Configure apt for the user description: | - This module handles both configuration of apt options and adding - source lists. There are configuration options such as - ``apt_get_wrapper`` and ``apt_get_command`` that control how - cloud-init invokes apt-get. These configuration options are - handled on a per-distro basis, so consult documentation for - cloud-init's distro support for instructions on using - these config options. - - By default, cloud-init will generate default - apt sources information in deb822 format at - :file:`/etc/apt/sources.list.d/.sources`. When the value - of `sources_list` does not appear to be deb822 format, or stable - distribution releases disable deb822 format, + This module handles configuration of advanced package tool (APT) options + and adding source lists. There are configuration options such as + `apt_get_wrapper`` and ``apt_get_command`` that control how cloud-init + invokes ``apt-get``. These configuration options are handled on a + per-distro basis, so consult documentation for cloud-init's distro support + for instructions on using these config options. + + By default, cloud-init will generate default APT sources information in + ``deb822`` format at :file:`/etc/apt/sources.list.d/.sources`. When + the value of ``sources_list`` does not appear to be ``deb822`` format, or + stable distribution releases disable ``deb822`` format, :file:`/etc/apt/sources.list` will be written instead. - + .. note:: - To ensure that apt configuration is valid yaml, any strings - containing special characters, especially ``:`` should be quoted. - + To ensure that APT configuration is valid YAML, any strings containing + special characters, especially colons, should be quoted (":"). + .. note:: - For more information about apt configuration, see the - ``Additional apt configuration`` example. + For more information about APT configuration, see the "Additional APT + configuration" example. examples: - - comment: "" + - comment: | + Example 1: file: cc_apt_configure/example1.yaml - - comment: "cloud-init version 23.4 will generate a deb822 formatted sources file at /etc/apt/sources.list.d/.sources instead of /etc/apt/sources.list when `sources_list` content is deb822 format." + - comment: > + Example 2: Cloud-init version 23.4 will generate a ``deb822``-formatted + ``sources`` file at ``/etc/apt/sources.list.d/.sources`` instead + of ``/etc/apt/sources.list`` when ``sources_list`` content is in + ``deb822`` format. file: cc_apt_configure/example2.yaml + name: Apt Configure + title: Configure APT for the user diff --git a/doc/module-docs/cc_apt_pipelining/data.yaml b/doc/module-docs/cc_apt_pipelining/data.yaml index 07077de1812..fb0d95c5fff 100644 --- a/doc/module-docs/cc_apt_pipelining/data.yaml +++ b/doc/module-docs/cc_apt_pipelining/data.yaml @@ -1,21 +1,24 @@ cc_apt_pipelining: - name: Apt Pipelining - title: Configure apt pipelining description: | - This module configures apt's ``Acquite::http::Pipeline-Depth`` option, - which controls how apt handles HTTP pipelining. It may be useful for - pipelining to be disabled, because some web servers, such as S3 do not - pipeline properly (LP: #948461). - - Value configuration options for this module are: + This module configures APT's ``Acquire::http::Pipeline-Depth`` option, + which controls how APT handles HTTP pipelining. It may be useful for + pipelining to be disabled, because some web servers (such as S3) do not + pipeline properly (LP: #948461). - * ``os``: (Default) use distro default - * ``false`` disable pipelining altogether - * ````: Manually specify pipeline depth. This is not recommended. + Value configuration options for this module are: + + - ``os``: (Default) use distro default + - ``false``: Disable pipelining altogether + - ````: Manually specify pipeline depth. This is not recommended. examples: - - comment: "" + - comment: | + Example 1: file: cc_apt_pipelining/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_apt_pipelining/example2.yaml - - comment: "" + - comment: | + Example 3: file: cc_apt_pipelining/example3.yaml + name: Apt Pipelining + title: Configure APT pipelining diff --git a/doc/module-docs/cc_bootcmd/data.yaml b/doc/module-docs/cc_bootcmd/data.yaml index cc5e8987eda..552e16887c9 100644 --- a/doc/module-docs/cc_bootcmd/data.yaml +++ b/doc/module-docs/cc_bootcmd/data.yaml @@ -1,23 +1,22 @@ cc_bootcmd: - name: Bootcmd - title: Run arbitrary commands early in the boot process description: | - This module runs arbitrary commands very early in the boot process, - only slightly after a boothook would run. This is very similar to a - boothook, but more user friendly. The environment variable - ``INSTANCE_ID`` will be set to the current instance id for all run - commands. Commands can be specified either as lists or strings. For - invocation details, see ``runcmd``. + This module runs arbitrary commands very early in the boot process, only + slightly after a boothook would run. This is very similar to a boothook, + but more user friendly. The environment variable ``INSTANCE_ID`` will be + set to the current instance ID for all run commands. Commands can be + specified either as lists or strings. For invocation details, see + ``runcmd``. .. note:: - - bootcmd should only be used for things that could not be done later - in the boot process. + ``bootcmd`` should only be used for things that could not be done later + in the boot process. .. note:: - - when writing files, do not use /tmp dir as it races with - systemd-tmpfiles-clean LP: #1707222. Use /run/somedir instead. + When writing files, do not use ``/tmp`` dir as it races with + ``systemd-tmpfiles-clean`` (LP: #1707222). Use ``/run/somedir`` instead. examples: - - comment: "" + - comment: | + Example 1: file: cc_bootcmd/example1.yaml + name: Bootcmd + title: Run arbitrary commands early in the boot process diff --git a/doc/module-docs/cc_bootcmd/example1.yaml b/doc/module-docs/cc_bootcmd/example1.yaml index 4aa6912f994..92106c74a0a 100644 --- a/doc/module-docs/cc_bootcmd/example1.yaml +++ b/doc/module-docs/cc_bootcmd/example1.yaml @@ -1,4 +1,4 @@ #cloud-config bootcmd: - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts -- [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] +- [cloud-init-per, once, mymkfs, mkfs, /dev/vdb] diff --git a/doc/module-docs/cc_byobu/data.yaml b/doc/module-docs/cc_byobu/data.yaml index 5d119cd5ea6..c9a0fa1dade 100644 --- a/doc/module-docs/cc_byobu/data.yaml +++ b/doc/module-docs/cc_byobu/data.yaml @@ -1,24 +1,26 @@ cc_byobu: - name: Byobu - title: Enable/disable byobu system wide and for default user description: | - This module controls whether byobu is enabled or disabled system wide and - for the default system user. If byobu is to be enabled, this module wil - ensure it is installed. Likewise, if it is to be disabled, it will be - removed if installed. + This module controls whether Byobu is enabled or disabled system-wide and + for the default system user. If Byobu is to be enabled, this module will + ensure it is installed. Likewise, if Byobu is to be disabled, it will be + removed (if installed). Valid configuration options for this module are: - - ``enable-system``: enable byobu system wide - - ``enable-user``: enable byobu for the default user - - ``disable-system``: disable byobu system wide - - ``disable-user``: disable byobu for the default user - - ``enable``: enable byobu both system wide and for default user - - ``disable``: disable byobu for all users + - ``enable-system``: enable Byobu system-wide + - ``enable-user``: enable Byobu for the default user + - ``disable-system``: disable Byobu system-wide + - ``disable-user``: disable Byobu for the default user + - ``enable``: enable Byobu both system-wide and for the default user + - ``disable``: disable Byobu for all users - ``user``: alias for ``enable-user`` - ``system``: alias for ``enable-system`` examples: - - comment: "" + - comment: | + Example 1: file: cc_byobu/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_byobu/example2.yaml + name: Byobu + title: Enable/disable Byobu system-wide and for the default user diff --git a/doc/module-docs/cc_ca_certs/data.yaml b/doc/module-docs/cc_ca_certs/data.yaml index abe7c63e96e..d18451a2956 100644 --- a/doc/module-docs/cc_ca_certs/data.yaml +++ b/doc/module-docs/cc_ca_certs/data.yaml @@ -1,6 +1,4 @@ cc_ca_certs: - name: CA Certificates - title: Add ca certificates description: | This module adds CA certificates to the system's CA store and updates any related files using the appropriate OS-specific utility. The default CA @@ -8,12 +6,15 @@ cc_ca_certs: configuration option ``remove_defaults``. .. note:: - certificates must be specified using valid yaml. in order to specify a - multiline certificate, the yaml multiline list syntax must be used - + Certificates must be specified using valid YAML. To specify a + multi-line certificate, the YAML multi-line list syntax must be used. + .. note:: - Alpine Linux requires the ca-certificates package to be installed in - order to provide the ``update-ca-certificates`` command. + Alpine Linux requires the ``ca-certificates`` package to be installed + in order to provide the ``update-ca-certificates`` command. examples: - - comment: "" + - comment: | + Example 1: file: cc_ca_certs/example1.yaml + name: CA Certificates + title: Add CA certificates diff --git a/doc/module-docs/cc_chef/data.yaml b/doc/module-docs/cc_chef/data.yaml index d249b85e524..f78158f185e 100644 --- a/doc/module-docs/cc_chef/data.yaml +++ b/doc/module-docs/cc_chef/data.yaml @@ -1,15 +1,17 @@ cc_chef: - name: Chef - title: module that configures, starts and installs chef description: | - This module enables chef to be installed (from packages, - gems, or from omnibus). Before this occurs, chef configuration is - written to disk (validation.pem, client.pem, firstboot.json, - client.rb), and required directories are created (/etc/chef and - /var/log/chef and so-on). If configured, chef will be - installed and started in either daemon or non-daemon mode. - If run in non-daemon mode, post run actions are executed to do - finishing activities such as removing validation.pem. + This module enables Chef to be installed (from packages, gems, or from + omnibus). Before this occurs, Chef configuration is written to disk + (``validation.pem``, ``client.pem``, ``firstboot.json``, ``client.rb``), + and required directories are created (``/etc/chef`` and ``/var/log/chef`` + and so on). + + If configured, Chef will be installed and started in either daemon or + non-daemon mode. If run in non-daemon mode, post-run actions are executed + to do finishing activities such as removing ``validation.pem``. examples: - - comment: "" + - comment: | + Example 1: file: cc_chef/example1.yaml + name: Chef + title: Module that installs, configures, and starts Chef diff --git a/doc/module-docs/cc_chef/example1.yaml b/doc/module-docs/cc_chef/example1.yaml index 75cec8f7a63..d71ff3dfae2 100644 --- a/doc/module-docs/cc_chef/example1.yaml +++ b/doc/module-docs/cc_chef/example1.yaml @@ -1,22 +1,17 @@ #cloud-config chef: - directories: - - /etc/chef - - /var/log/chef - validation_cert: system - install_type: omnibus - initial_attributes: - apache: - prefork: - maxclients: 100 - keepalive: off - run_list: - - recipe[apache2] - - role[db] + directories: [/etc/chef, /var/log/chef] encrypted_data_bag_secret: /etc/chef/encrypted_data_bag_secret environment: _default + initial_attributes: + apache: + keepalive: false + prefork: {maxclients: 100} + install_type: omnibus log_level: :auto omnibus_url_retries: 2 + run_list: ['recipe[apache2]', 'role[db]'] server_url: https://chef.yourorg.com:4000 ssl_verify_mode: :verify_peer + validation_cert: system validation_name: yourorg-validator diff --git a/doc/module-docs/cc_disable_ec2_metadata/data.yaml b/doc/module-docs/cc_disable_ec2_metadata/data.yaml index f2134961097..ce47a30bdc0 100644 --- a/doc/module-docs/cc_disable_ec2_metadata/data.yaml +++ b/doc/module-docs/cc_disable_ec2_metadata/data.yaml @@ -1,10 +1,11 @@ cc_disable_ec2_metadata: - name: Disable EC2 Metadata - title: Disable AWS EC2 Metadata - description: - This module can disable the ec2 datasource by rejecting the route to - ``169.254.169.254``, the usual route to the datasource. This module - is disabled by default. + description: | + This module can disable the EC2 datasource by rejecting the route to + ``169.254.169.254``, the usual route to the datasource. This module is + disabled by default. examples: - - comment: "" + - comment: | + Example 1: file: cc_disable_ec2_metadata/example1.yaml + name: Disable EC2 Metadata + title: Disable AWS EC2 Metadata diff --git a/doc/module-docs/cc_disk_setup/data.yaml b/doc/module-docs/cc_disk_setup/data.yaml index 7affc517071..abc56e64306 100644 --- a/doc/module-docs/cc_disk_setup/data.yaml +++ b/doc/module-docs/cc_disk_setup/data.yaml @@ -1,30 +1,31 @@ cc_disk_setup: - name: Disk Setup - title: Configure partitions and filesystems description: | - This module is able to configure simple partition tables and filesystems. - + This module configures simple partition tables and filesystems. + .. note:: - for more detail about configuration options for disk setup, see the disk - setup example + For more detail about configuration options for disk setup, see the + disk setup example. .. note:: - if a swap partition is being created via ``disk_setup`` then a - ``fs_entry`` entry is also needed in order for mkswap to be run, + If a swap partition is being created via ``disk_setup``, then an + ``fs_entry`` entry is also needed in order for ``mkswap`` to be run, otherwise when swap activation is later attempted it will fail. - + For convenience, aliases can be specified for disks using the - ``device_aliases`` config key, which takes a dictionary of alias: path - mappings. There are automatic aliases for ``swap`` and ``ephemeral``, where - ``swap`` will always refer to the active swap partition and ``ephemeral`` - will refer to the block device of the ephemeral image. + ``device_aliases`` config key, which takes a dictionary of ``alias: path`` + mappings. There are automatic aliases for ``swap`` and ``ephemeral``, + where ``swap`` will always refer to the active swap partition and + ``ephemeral`` will refer to the block device of the ephemeral image. Disk partitioning is done using the ``disk_setup`` directive. This config - directive accepts a dictionary where each key is either a path to a block - device or an alias specified in ``device_aliases``, and each value is the - configuration options for the device. File system configuration is done using - the ``fs_setup`` directive. This config directive accepts a list of + directive accepts a dictionary where each key is either a path to a + block device or an alias specified in ``device_aliases``, and each value is + the configuration options for the device. File system configuration is done + using the ``fs_setup`` directive. This config directive accepts a list of filesystem configs. examples: - - comment: "" + - comment: | + Example 1: file: cc_disk_setup/example1.yaml + name: Disk Setup + title: Configure partitions and filesystems diff --git a/doc/module-docs/cc_disk_setup/example1.yaml b/doc/module-docs/cc_disk_setup/example1.yaml index 0abc704fe46..fe2fe6e0239 100644 --- a/doc/module-docs/cc_disk_setup/example1.yaml +++ b/doc/module-docs/cc_disk_setup/example1.yaml @@ -1,36 +1,24 @@ #cloud-config -device_aliases: - my_alias: /dev/sdb - swap_disk: /dev/sdc +device_aliases: {my_alias: /dev/sdb, swap_disk: /dev/sdc} disk_setup: + /dev/sdd: {layout: true, overwrite: true, table_type: mbr} my_alias: - table_type: gpt layout: [50, 50] overwrite: true - swap_disk: table_type: gpt - layout: [[100, 82]] - overwrite: true - /dev/sdd: - table_type: mbr - layout: true + swap_disk: + layout: + - [100, 82] overwrite: true + table_type: gpt fs_setup: -- label: fs1 - filesystem: ext4 - device: my_alias.1 - cmd: mkfs -t %(filesystem)s -L %(label)s %(device)s -- label: fs2 - device: my_alias.2 - filesystem: ext4 -- label: swap - device: swap_disk.1 - filesystem: swap -- label: fs3 - device: /dev/sdd1 - filesystem: ext4 +- {cmd: mkfs -t %(filesystem)s -L %(label)s %(device)s, device: my_alias.1, filesystem: ext4, + label: fs1} +- {device: my_alias.2, filesystem: ext4, label: fs2} +- {device: swap_disk.1, filesystem: swap, label: swap} +- {device: /dev/sdd1, filesystem: ext4, label: fs3} mounts: -- ["my_alias.1", "/mnt1"] -- ["my_alias.2", "/mnt2"] -- ["swap_disk.1", "none", "swap", "sw", "0", "0"] -- ["/dev/sdd1", "/mnt3"] +- [my_alias.1, /mnt1] +- [my_alias.2, /mnt2] +- [swap_disk.1, none, swap, sw, '0', '0'] +- [/dev/sdd1, /mnt3] diff --git a/doc/module-docs/cc_fan/data.yaml b/doc/module-docs/cc_fan/data.yaml index 1b477cf0aed..eba08fcb236 100644 --- a/doc/module-docs/cc_fan/data.yaml +++ b/doc/module-docs/cc_fan/data.yaml @@ -1,19 +1,19 @@ cc_fan: - name: Fan - title: Configure ubuntu fan networking description: | - This module installs, configures and starts the ubuntu fan network system. - For more information about Ubuntu Fan, see: - ``https://wiki.ubuntu.com/FanNetworking``. + This module installs, configures and starts the Ubuntu fan network + system (`Read more about Ubuntu Fan `_). If cloud-init sees a ``fan`` entry in cloud-config it will: - - write ``config_path`` with the contents of the ``config`` key - - install the package ``ubuntu-fan`` if it is not installed - - ensure the service is started (or restarted if was previously running) + - Write ``config_path`` with the contents of the ``config`` key + - Install the package ``ubuntu-fan`` if it is not installed + - Ensure the service is started (or restarted if was previously running) Additionally, the ``ubuntu-fan`` package will be automatically installed if not present. examples: - - comment: "" + - comment: | + Example 1: file: cc_fan/example1.yaml + name: Fan + title: Configure Ubuntu fan networking diff --git a/doc/module-docs/cc_final_message/data.yaml b/doc/module-docs/cc_final_message/data.yaml index 74ad35aba2a..1759909a565 100644 --- a/doc/module-docs/cc_final_message/data.yaml +++ b/doc/module-docs/cc_final_message/data.yaml @@ -1,20 +1,23 @@ cc_final_message: - name: Final Message - title: Output final message when cloud-init has finished description: | - This module configures the final message that cloud-init writes. The message is specified as a jinja template with the following variables set: + This module configures the final message that cloud-init writes. The + message is specified as a Jinja template with the following variables set: - ``version``: cloud-init version - ``timestamp``: time at cloud-init finish - ``datasource``: cloud-init data source - ``uptime``: system uptime - This message is written to the cloud-init log (usually /var/log/cloud-init.log) - as well as stderr (which usually redirects to /var/log/cloud-init-output.log). + This message is written to the cloud-init log (usually + ``/var/log/cloud-init.log``) as well as stderr (which usually redirects to + ``/var/log/cloud-init-output.log``). Upon exit, this module writes the system uptime, timestamp, and cloud-init version to ``/var/lib/cloud/instance/boot-finished`` independent of any user data specified for this module. examples: - - comment: "" + - comment: | + Example 1: file: cc_final_message/example1.yaml + name: Final Message + title: Output final message when cloud-init has finished diff --git a/doc/module-docs/cc_growpart/data.yaml b/doc/module-docs/cc_growpart/data.yaml index b87e0c595a2..a6fc6377131 100644 --- a/doc/module-docs/cc_growpart/data.yaml +++ b/doc/module-docs/cc_growpart/data.yaml @@ -1,45 +1,56 @@ cc_growpart: - name: Growpart - title: Grow partitions description: | - Growpart resizes partitions to fill the available disk space. - This is useful for cloud instances with a larger amount of disk space available - than the pristine image uses, as it allows the instance to automatically make - use of the extra space. Note that this only works if the partition to be - resized is the last one on a disk with classic partitioning scheme (MBR, BSD, - GPT). LVM, Btrfs and ZFS have no such restrictions. + Growpart resizes partitions to fill the available disk space. This is + useful for cloud instances with a larger amount of disk space available + than the pristine image uses, as it allows the instance to automatically + make use of the extra space. + + Note that this only works if the partition to be resized is the last one + on a disk with classic partitioning scheme (MBR, BSD, GPT). LVM, Btrfs and + ZFS have no such restrictions. The devices on which to run growpart are specified as a list under the ``devices`` key. - There is some functionality overlap between this module and the ``growroot`` - functionality of ``cloud-initramfs-tools``. However, there are some situations - where one tool is able to function and the other is not. The default - configuration for both should work for most cloud instances. To explicitly - prevent ``cloud-initramfs-tools`` from running ``growroot``, the file - ``/etc/growroot-disabled`` can be created. By default, both ``growroot`` and - ``cc_growpart`` will check for the existence of this file and will not run if - it is present. However, this file can be ignored for ``cc_growpart`` by setting - ``ignore_growroot_disabled`` to ``true``. For more information on - ``cloud-initramfs-tools`` see: https://launchpad.net/cloud-initramfs-tools - - On FreeBSD, there is also the ``growfs`` service, which has a lot of overlap - with ``cc_growpart`` and ``cc_resizefs``, but only works on the root partition. - In that configuration, we use it, otherwise, we fall back to ``gpart``. - - Note however, that ``growfs`` may insert a swap partition, if none is present, - unless instructed not to via ``growfs_swap_size=0`` in either ``kenv(1)``, or - ``rc.conf(5)``. - - Growpart is enabled by default on the root partition. The default config for - growpart is:: - - growpart: - mode: auto - devices: ["/"] - ignore_growroot_disabled: false + There is some functionality overlap between this module and the + ``growroot`` functionality of ``cloud-initramfs-tools``. However, there + are some situations where one tool is able to function and the other is + not. The default configuration for both should work for most cloud + instances. To explicitly prevent ``cloud-initramfs-tools`` from running + ``growroot``, the file ``/etc/growroot-disabled`` can be created. + + By default, both ``growroot`` and ``cc_growpart`` will check for the + existence of this file and will not run if it is present. However, this + file can be ignored for ``cc_growpart`` by setting + ``ignore_growroot_disabled`` to ``true``. + `Read more about `_ + ``cloud-initramfs-tools``. + + On FreeBSD, there is also the ``growfs`` service, which has a lot of + overlap with ``cc_growpart`` and ``cc_resizefs``, but only works on the + root partition. In that configuration, we use it, otherwise, we fall back + to ``gpart``. + + .. note:: + ``growfs`` may insert a swap partition, if none is present, unless + instructed not to via ``growfs_swap_size=0`` in either ``kenv(1)``, or + ``rc.conf(5)``. + + Growpart is enabled by default on the root partition. The default config + for growpart is: + + .. code-block:: yaml + + growpart: + mode: auto + devices: [\"/\"] + ignore_growroot_disabled: false examples: - - comment: "" + - comment: | + Example 1: file: cc_growpart/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_growpart/example2.yaml + name: Growpart + title: Grow partitions diff --git a/doc/module-docs/cc_growpart/example1.yaml b/doc/module-docs/cc_growpart/example1.yaml index 20fb7c9f386..0d09125a0fa 100644 --- a/doc/module-docs/cc_growpart/example1.yaml +++ b/doc/module-docs/cc_growpart/example1.yaml @@ -1,5 +1,5 @@ #cloud-config - growpart: - mode: auto - devices: ["/"] - ignore_growroot_disabled: false +growpart: + devices: [/] + ignore_growroot_disabled: false + mode: auto diff --git a/doc/module-docs/cc_growpart/example2.yaml b/doc/module-docs/cc_growpart/example2.yaml index 5767e4f8cde..045a7846398 100644 --- a/doc/module-docs/cc_growpart/example2.yaml +++ b/doc/module-docs/cc_growpart/example2.yaml @@ -1,7 +1,5 @@ #cloud-config growpart: - mode: growpart - devices: - - "/" - - "/dev/vdb1" + devices: [/, /dev/vdb1] ignore_growroot_disabled: true + mode: growpart diff --git a/doc/module-docs/cc_grub_dpkg/data.yaml b/doc/module-docs/cc_grub_dpkg/data.yaml index 5d8ac3cf3a1..9530024689a 100644 --- a/doc/module-docs/cc_grub_dpkg/data.yaml +++ b/doc/module-docs/cc_grub_dpkg/data.yaml @@ -1,19 +1,20 @@ cc_grub_dpkg: - name: Grub Dpkg - title: Configure grub debconf installation device description: | - Configure which device is used as the target for grub installation. This + Configure which device is used as the target for GRUB installation. This module can be enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config dict. This module automatically selects a disk using ``grub-probe`` if no installation device is specified. - The value which is placed into the debconf database is in the format which - the grub postinstall script expects. Normally, this is a /dev/disk/by-id/ - value, but we do fallback to the plain disk name if a by-id name is not + The value placed into the debconf database is in the format expected by the + GRUB post-install script expects. Normally, this is a ``/dev/disk/by-id/`` + value, but we do fallback to the plain disk name if a ``by-id`` name is not present. If this module is executed inside a container, then the debconf database is - seeded with empty values, and install_devices_empty is set to true. + seeded with empty values, and ``install_devices_empty`` is set to ``true``. examples: - - comment: "" + - comment: | + Example 1: file: cc_grub_dpkg/example1.yaml + name: GRUB dpkg + title: Configure GRUB debconf installation device diff --git a/doc/module-docs/cc_install_hotplug/data.yaml b/doc/module-docs/cc_install_hotplug/data.yaml index cde12d7f530..32369719b8c 100644 --- a/doc/module-docs/cc_install_hotplug/data.yaml +++ b/doc/module-docs/cc_install_hotplug/data.yaml @@ -1,22 +1,21 @@ cc_install_hotplug: - name: Install Hotplug - title: Install hotplug udev rules if supported and enabled description: | - This module will install the udev rules to enable hotplug if - supported by the datasource and enabled in the userdata. The udev - rules will be installed as - ``/etc/udev/rules.d/90-cloud-init-hook-hotplug.rules``. + This module will install the udev rules to enable hotplug if supported by + the datasource and enabled in the userdata. The udev rules will be + installed as ``/etc/udev/rules.d/90-cloud-init-hook-hotplug.rules``. - When hotplug is enabled, newly added network devices will be added - to the system by cloud-init. After udev detects the event, - cloud-init will refresh the instance metadata from the datasource, - detect the device in the updated metadata, then apply the updated - network configuration. + When hotplug is enabled, newly added network devices will be added to the + system by cloud-init. After udev detects the event, cloud-init will + refresh the instance metadata from the datasource, detect the device in + the updated metadata, then apply the updated network configuration. Currently supported datasources: Openstack, EC2 examples: - - comment: "Enable hotplug of network devices" + - comment: | + Example 1: Enable hotplug of network devices file: cc_install_hotplug/example1.yaml - - comment: "Enable network hotplug alongside boot event" + - comment: | + Example 2: Enable network hotplug alongside boot event file: cc_install_hotplug/example2.yaml - + name: Install Hotplug + title: Install hotplug udev rules if supported and enabled diff --git a/doc/module-docs/cc_install_hotplug/example1.yaml b/doc/module-docs/cc_install_hotplug/example1.yaml index e1e6432e43c..9e49a6acdff 100644 --- a/doc/module-docs/cc_install_hotplug/example1.yaml +++ b/doc/module-docs/cc_install_hotplug/example1.yaml @@ -1,4 +1,4 @@ #cloud-config updates: network: - when: ["hotplug"] + when: [hotplug] diff --git a/doc/module-docs/cc_install_hotplug/example2.yaml b/doc/module-docs/cc_install_hotplug/example2.yaml index 6ba3c0b59ee..dba1ca0f2a4 100644 --- a/doc/module-docs/cc_install_hotplug/example2.yaml +++ b/doc/module-docs/cc_install_hotplug/example2.yaml @@ -1,4 +1,4 @@ #cloud-config updates: network: - when: ["boot", "hotplug"] + when: [boot, hotplug] diff --git a/doc/module-docs/cc_keyboard/data.yaml b/doc/module-docs/cc_keyboard/data.yaml index bbd24334261..4e1e5da792e 100644 --- a/doc/module-docs/cc_keyboard/data.yaml +++ b/doc/module-docs/cc_keyboard/data.yaml @@ -1,14 +1,16 @@ cc_keyboard: - name: Keyboard - title: Set keyboard layout - description: Handle keyboard configuration + description: | + Handle keyboard configuration. examples: - - comment: Set keyboard layout to "us" + - comment: | + Example 1: Set keyboard layout to "us" file: cc_keyboard/example1.yaml - - comment: Set specific keyboard layout, model, variant, options + - comment: | + Example 2: Set specific keyboard layout, model, variant, options file: cc_keyboard/example2.yaml - comment: > - For Alpine Linux set specific keyboard layout and variant, as used by - setup-keymap. Model and options are ignored. + Example 3: For Alpine Linux, set specific keyboard layout and variant, + as used by ``setup-keymap``. Model and options are ignored. file: cc_keyboard/example3.yaml - + name: Keyboard + title: Set keyboard layout diff --git a/doc/module-docs/cc_keys_to_console/data.yaml b/doc/module-docs/cc_keys_to_console/data.yaml index 58a55009f59..8e1b9a87986 100644 --- a/doc/module-docs/cc_keys_to_console/data.yaml +++ b/doc/module-docs/cc_keys_to_console/data.yaml @@ -1,21 +1,26 @@ cc_keys_to_console: - name: Keys to Console - title: Control which SSH host keys may be written to console description: | - For security reasons it may be desirable not to write SSH host keys - and their fingerprints to the console. To avoid either being written - to the console the ``emit_keys_to_console`` config key under the - main ``ssh`` config key can be used. To avoid the fingerprint of - types of SSH host keys being written to console the - ``ssh_fp_console_blacklist`` config key can be used. By default, - all types of keys will have their fingerprints written to console. + For security reasons it may be desirable not to write SSH host keys and + their fingerprints to the console. To avoid either of them being written + to the console, the ``emit_keys_to_console`` config key under the main + ``ssh`` config key can be used. + + To avoid the fingerprint of types of SSH host keys being written to + console the ``ssh_fp_console_blacklist`` config key can be used. By + default, all types of keys will have their fingerprints written to console. + To avoid host keys of a key type being written to console the - ``ssh_key_console_blacklist`` config key can be used. By default + ``ssh_key_console_blacklist`` config key can be used. By default, all supported host keys are written to console. examples: - - comment: Do not print any SSH keys to system console + - comment: | + Example 1: Do not print any SSH keys to system console file: cc_keys_to_console/example1.yaml - - comment: Do not print certain ssh key types to console + - comment: | + Example 2: Do not print certain SSH key types to console file: cc_keys_to_console/example2.yaml - - comment: Do not print specific ssh key fingerprints to console + - comment: | + Example 3: Do not print specific SSH key fingerprints to console file: cc_keys_to_console/example3.yaml + name: Keys to Console + title: Control which SSH host keys may be written to console diff --git a/doc/module-docs/cc_landscape/data.yaml b/doc/module-docs/cc_landscape/data.yaml index 189946a9171..bec53109b4c 100644 --- a/doc/module-docs/cc_landscape/data.yaml +++ b/doc/module-docs/cc_landscape/data.yaml @@ -1,35 +1,46 @@ cc_landscape: - name: Landscape - title: Install and configure landscape client description: | - This module installs and configures ``landscape-client``. The landscape + This module installs and configures ``landscape-client``. The Landscape client will only be installed if the key ``landscape`` is present in - config. Landscape client configuration is given under the ``client`` key - under the main ``landscape`` config key. The config parameters are not - interpreted by cloud-init, but rather are converted into a ConfigObj - formatted file and written out to the `[client]` section in - ``/etc/landscape/client.conf``. + config. - The following default client config is provided, but can be overridden:: + Landscape client configuration is given under the ``client`` key under the + main ``landscape`` config key. The config parameters are not interpreted by + cloud-init, but rather are converted into a ``ConfigObj``-formatted file + and written out to the ``[client]`` section in + ``/etc/landscape/client.conf``. The following default client config is + provided, but can be overridden + + .. code-block:: yaml - landscape: - client: - log_level: "info" - url: "https://landscape.canonical.com/message-system" - ping_url: "http://landscape.canoncial.com/ping" - data_path: "/var/lib/landscape/client" + landscape: + client: + log_level: "info" + url: "https://landscape.canonical.com/message-system" + ping_url: "http://landscape.canoncial.com/ping" + data_path: "/var/lib/landscape/client" .. note:: - see landscape documentation for client config keys + See `Landscape documentation `_ for + client config keys. .. note:: - if ``tags`` is defined, its contents should be a string delimited with - ``,`` rather than a list + If ``tags`` is defined, its contents should be a string delimited with + a comma (",") rather than a list. examples: - - comment: - To discover additional supported client keys, run man landscape-config + - comment: > + To discover additional supported client keys, run + ``man landscape-config``. + + Example 1: file: cc_landscape/example1.yaml - - comment: Minimum viable config requires account_name and computer_title + - comment: > + Example 2: Minimum viable config requires ``account_name`` and + ``computer_title``. file: cc_landscape/example2.yaml - - comment: To install landscape-client from a PPA, specify apt.sources + - comment: > + Example 3: To install ``landscape-client`` from a PPA, specify + ``apt.sources``. file: cc_landscape/example3.yaml + name: Landscape + title: Install and configure Landscape client diff --git a/doc/module-docs/cc_locale/data.yaml b/doc/module-docs/cc_locale/data.yaml index 5805b58aab7..d5c0961ffba 100644 --- a/doc/module-docs/cc_locale/data.yaml +++ b/doc/module-docs/cc_locale/data.yaml @@ -1,11 +1,13 @@ cc_locale: - name: Locale - title: Set system locale description: | - Configure the system locale and apply it system wide. By default use the + Configure the system locale and apply it system-wide. By default, use the locale specified by the datasource. examples: - - comment: Set the locale to ar_AE + - comment: | + Example 1: Set the locale to ``"ar_AE"`` file: cc_locale/example1.yaml - - comment: Set the locale to fr_CA in /etc/alternate_path/locale + - comment: | + Example 2: Set the locale to ``"fr_CA"`` in ``/etc/alternate_path/locale`` file: cc_locale/example2.yaml + name: Locale + title: Set system locale diff --git a/doc/module-docs/cc_lxd/data.yaml b/doc/module-docs/cc_lxd/data.yaml index 2588b65ba44..5ca35c57561 100644 --- a/doc/module-docs/cc_lxd/data.yaml +++ b/doc/module-docs/cc_lxd/data.yaml @@ -1,25 +1,34 @@ cc_lxd: - name: LXD - title: Configure LXD with ``lxd init`` and optionally lxd-bridge description: | - This module configures lxd with user specified options using ``lxd init``. - If lxd is not present on the system but lxd configuration is provided, then - lxd will be installed. If the selected storage backend userspace utility is - not installed, it will be installed. If network bridge configuration is - provided, then lxd-bridge will be configured accordingly. + This module configures LXD with user-specified options using ``lxd init``. + + - If ``lxd`` is not present on the system but LXD configuration is provided, + then ``lxd`` will be installed. + - If the selected storage backend userspace utility is not installed, it + will be installed. + - If network bridge configuration is provided, then ``lxd-bridge`` will be + configured accordingly. examples: - - comment: Simplest working directory backed LXD configuration + - comment: | + Example 1: Simplest working directory-backed LXD configuration. file: cc_lxd/example1.yaml - - comment: LXD init showcasing cloud-init's LXD config options + - comment: | + Example 2: ``lxd-init`` showcasing cloud-init's LXD config options. file: cc_lxd/example2.yaml - comment: > - For more complex non-interactive LXD configuration of networks, - storage_pools, profiles, projects, clusters and core config, - `lxd:preseed` config will be passed as stdin to the command: - lxd init --preseed - See https://documentation.ubuntu.com/lxd/en/latest/howto/initialize/#non-interactive-configuration or run: lxd init --dump to see viable preseed YAML allowed. - Preseed settings configuring the LXD daemon for HTTPS connections - on 192.168.1.1 port 9999, a nested profile which allows for - LXD nesting on containers and a limited project allowing for - RBAC approach when defining behavior for sub projects. + Example 3: For more complex non-interactive LXD configuration of + networks, storage pools, profiles, projects, clusters and core config, + ``lxd:preseed`` config will be passed as stdin to the command: + ``lxd init --preseed``. + + See the + `LXD non-interactive configuration `_ + or run ``lxd init --dump`` to see viable preseed YAML allowed. + + Preseed settings configuring the LXD daemon for HTTPS connections on + 192.168.1.1 port 9999, a nested profile which allows for LXD nesting on + containers and a limited project allowing for RBAC approach when defining + behavior for sub-projects. file: cc_lxd/example3.yaml + name: LXD + title: Configure LXD with ``lxd init`` and (optionally) ``lxd-bridge`` diff --git a/doc/module-docs/cc_mcollective/data.yaml b/doc/module-docs/cc_mcollective/data.yaml index cbb4b0e6aec..cec844310dd 100644 --- a/doc/module-docs/cc_mcollective/data.yaml +++ b/doc/module-docs/cc_mcollective/data.yaml @@ -1,32 +1,26 @@ cc_mcollective: - name: Mcollective - title: Install, configure and start mcollective description: | - This module installs, configures and starts mcollective. If the - ``mcollective`` key is present in config, then mcollective will be + This module installs, configures and starts MCollective. If the + ``mcollective`` key is present in config, then MCollective will be installed and started. - + Configuration for ``mcollective`` can be specified in the ``conf`` key - under ``mcollective``. Each config value consists of a key value pair and + under ``mcollective``. Each config value consists of a key-value pair and will be written to ``/etc/mcollective/server.cfg``. The ``public-cert`` and ``private-cert`` keys, if present in conf may be used to specify the - public and private certificates for mcollective. Their values will be + public and private certificates for MCollective. Their values will be written to ``/etc/mcollective/ssl/server-public.pem`` and ``/etc/mcollective/ssl/server-private.pem``. - .. note:: - The ec2 metadata service is readable by non-root users. - If security is a concern, use include-once and ssl urls. + .. warning:: + The EC2 metadata service is a network service and thus is readable by + non-root users on the system (i.e., ``ec2metadata --user-data``). If + security is a concern, use ``include-once`` and SSL URLS. examples: - - comment: | - Provide server private and public key and provide the following - config settings in /etc/mcollective/server.cfg: - loglevel: debug - plugin.stomp.host: dbhost - - .. warning:: - The ec2 metadata service is a network service, and thus is - readable by non-root users on the system - (ie: 'ec2metadata --user-data') - If you want security for this, please use include-once + SSL urls + - comment: > + Example 1: Provide server private and public key, and provide the + ``loglevel: debug`` and ``plugin.stomp.host: dbhost`` config settings in + ``/etc/mcollective/server.cfg:`` file: cc_mcollective/example1.yaml + name: MCollective + title: Install, configure and start MCollective diff --git a/doc/module-docs/cc_mounts/data.yaml b/doc/module-docs/cc_mounts/data.yaml index 0209264951e..751b301d501 100644 --- a/doc/module-docs/cc_mounts/data.yaml +++ b/doc/module-docs/cc_mounts/data.yaml @@ -1,61 +1,69 @@ cc_mounts: - name: Mounts - title: Configure mount points and swap files description: | - This module can add or remove mountpoints from ``/etc/fstab`` as well as - configure swap. The ``mounts`` config key takes a list of fstab entries to - add. Each entry is specified as a list of ``[ fs_spec, fs_file, fs_vfstype, - fs_mntops, fs-freq, fs_passno ]``. For more information on these options, - consult the manual for ``/etc/fstab``. When specifying the ``fs_spec``, if - the device name starts with one of ``xvd``, ``sd``, ``hd``, or ``vd``, the - leading ``/dev`` may be omitted. - - Any mounts that do not appear to either an attached block device or network - resource will be skipped with a log like "Ignoring nonexistent mount ...". - + This module can add or remove mount points from ``/etc/fstab`` as well as + configure swap. The ``mounts`` config key takes a list of ``fstab`` entries + to add. Each entry is specified as a list of ``[ fs_spec, fs_file, + fs_vfstype, fs_mntops, fs-freq, fs_passno ]``. + + For more information on these options, consult the manual for + ``/etc/fstab``. When specifying the ``fs_spec``, if the device name starts + with one of ``xvd``, ``sd``, ``hd``, or ``vd``, the leading ``/dev`` may be + omitted. Any mounts that do not appear to either an attached block device + or network resource will be skipped with a log like "Ignoring nonexistent + mount ...". + Cloud-init will attempt to add the following mount directives if available - and unconfigured in `/etc/fstab`:: + and unconfigured in ``/etc/fstab``: - mounts: - - ["ephemeral0", "/mnt", "auto", "defaults,nofail,x-systemd.after=cloud-init.service", "0", "2"] - - ["swap", "none", "swap", "sw", "0", "0"] + .. code-block:: yaml - In order to remove a previously listed mount, an entry can be added to - the `mounts` list containing ``fs_spec`` for the device to be removed but - no mountpoint (i.e. ``[ swap ]`` or ``[ swap, null ]``). + mounts: + - ["ephemeral0", "/mnt", "auto", "defaults,nofail,x-systemd.after=cloud-init.service", "0", "2"] + - ["swap", "none", "swap", "sw", "0", "0"] + In order to remove a previously-listed mount, an entry can be added to the + ``mounts`` list containing ``fs_spec`` for the device to be removed but no + mount point (i.e. ``[ swap ]`` or ``[ swap, null ]``). + The ``mount_default_fields`` config key allows default values to be specified for the fields in a ``mounts`` entry that are not specified, aside from the ``fs_spec`` and the ``fs_file`` fields. If specified, this - must be a list containing 6 values. It defaults to:: + must be a list containing 6 values. It defaults to: + + .. code-block:: yaml - mount_default_fields: [none, none, "auto", "defaults,nofail,x-systemd.after=cloud-init.service", "0", "2"] + mount_default_fields: [none, none, "auto", "defaults,nofail,x-systemd.after=cloud-init.service", "0", "2"] Non-systemd init systems will vary in ``mount_default_fields``. Swap files can be configured by setting the path to the swap file to create - with ``filename``, the size of the swap file with ``size`` maximum size of - the swap file if using an ``size: auto`` with ``maxsize``. By default no - swap file is created. + with ``filename``, the size of the swap file with ``size``, maximum size + of the swap file if using an ``size: auto`` with ``maxsize``. By default, + no swap file is created. .. note:: - If multiple mounts are specified where a subsequent mount's mountpoint - is inside of a previously declared mount's mountpoint (i.e. the 1st - mount has a mountpoint of ``/abc`` and the 2nd mount has a mountpoint - of ``/abc/def``) then this will not work as expected - ``cc_mounts`` - first creates the directories for all the mountpoints **before** it - starts to perform any mounts and so the sub-mountpoint directory will - not be created correctly inside the parent mountpoint. - - For systems using util-linux's ``mount`` program this issue can be - worked around by specifying ``X-mount.mkdir`` as part of a - ``fs_mntops`` value for the subsequent mount entry. + If multiple mounts are specified, where a subsequent mount's mount point + is inside of a previously-declared mount's mount point, (i.e. the 1st + mount has a mount point of ``/abc`` and the 2nd mount has a mount point + of ``/abc/def``) then this will not work as expected -- ``cc_mounts`` + first creates the directories for all the mount points **before** it + starts to perform any mounts and so the sub-mount point directory will + not be created correctly inside the parent mount point. + + For systems using ``util-linux``'s ``mount`` program, this issue can be + worked around by specifying ``X-mount.mkdir`` as part of a ``fs_mntops`` + value for the subsequent mount entry. examples: - - comment: | - Mount ephemeral0 with "noexec" flag, /dev/sdc with mount_default_fields, - and /dev/xvdh with custom fs_passno "0" to avoid fsck on the mount. - Also provide an automatically sized swap with a max size of 10485760 - bytes. + - comment: > + Example 1: Mount ``ephemeral0`` with ``noexec`` flag, ``/dev/sdc`` with + ``mount_default_fields``, and ``/dev/xvdh`` with custom ``fs_passno "0"`` + to avoid ``fsck`` on the mount. + + Also provide an automatically-sized swap with a max size of 10485760 bytes. file: cc_mounts/example1.yaml - - comment: Create a 2 GB swap file at /swapfile using human-readable values + - comment: > + Example 2: Create a 2 GB swap file at ``/swapfile`` using human-readable + values. file: cc_mounts/example2.yaml + name: Mounts + title: Configure mount points and swap files diff --git a/doc/module-docs/cc_ntp/data.yaml b/doc/module-docs/cc_ntp/data.yaml index 9766b9f23e4..0ce46148279 100644 --- a/doc/module-docs/cc_ntp/data.yaml +++ b/doc/module-docs/cc_ntp/data.yaml @@ -1,17 +1,26 @@ cc_ntp: - name: NTP - title: enable and configure ntp description: | - Handle ntp configuration. If ntp is not installed on the system and - ntp configuration is specified, ntp will be installed. If there is a - default ntp config file in the image or one is present in the - distro's ntp package, it will be copied to a file with ``.dist`` - appended to the filename before any changes are made. A list of ntp - pools and ntp servers can be provided under the ``ntp`` config key. - If no ntp ``servers`` or ``pools`` are provided, 4 pools will be used - in the format ``{0-3}.{distro}.pool.ntp.org``. + Handle Network Time Protocol (NTP) configuration. If ``ntp`` is not + installed on the system and NTP configuration is specified, ``ntp`` will + be installed. + + If there is a default NTP config file in the image or one is present in the + distro's ``ntp`` package, it will be copied to a file with ``.dist`` + appended to the filename before any changes are made. + + A list of NTP pools and NTP servers can be provided under the ``ntp`` + config key. + + If no NTP ``servers`` or ``pools`` are provided, 4 pools will be used + in the format: + + ``{0-3}.{distro}.pool.ntp.org`` examples: - - comment: Override ntp with chrony configuration on Ubuntu + - comment: | + Example 1: Override NTP with chrony configuration on Ubuntu. file: cc_ntp/example1.yaml - - comment: Provide a custom ntp client configuration + - comment: | + Example 2: Provide a custom NTP client configuration. file: cc_ntp/example2.yaml + name: NTP + title: Enable and configure NTP diff --git a/doc/module-docs/cc_package_update_upgrade_install/data.yaml b/doc/module-docs/cc_package_update_upgrade_install/data.yaml index 013305641a3..121720ab1bd 100644 --- a/doc/module-docs/cc_package_update_upgrade_install/data.yaml +++ b/doc/module-docs/cc_package_update_upgrade_install/data.yaml @@ -1,6 +1,4 @@ cc_package_update_upgrade_install: - name: Package Update Upgrade Install - title: Update, upgrade, and install packages description: | This module allows packages to be updated, upgraded or installed during boot. If any packages are to be installed or an upgrade is to be performed @@ -8,5 +6,8 @@ cc_package_update_upgrade_install: upgrade requires a reboot, then a reboot can be performed if ``package_reboot_if_required`` is specified. examples: - - comment: "" + - comment: | + Example 1: file: cc_package_update_upgrade_install/example1.yaml + name: Package Update Upgrade Install + title: Update, upgrade, and install packages diff --git a/doc/module-docs/cc_package_update_upgrade_install/example1.yaml b/doc/module-docs/cc_package_update_upgrade_install/example1.yaml index 2677e7f7984..9f6adba917d 100644 --- a/doc/module-docs/cc_package_update_upgrade_install/example1.yaml +++ b/doc/module-docs/cc_package_update_upgrade_install/example1.yaml @@ -1,4 +1,7 @@ #cloud-config +package_reboot_if_required: true +package_update: true +package_upgrade: true packages: - pwgen - pastebinit @@ -7,8 +10,4 @@ packages: - certbot - [juju, --edge] - [lxd, --channel=5.15/stable] -- apt: - - mg -package_update: true -package_upgrade: true -package_reboot_if_required: true +- apt: [mg] diff --git a/doc/module-docs/cc_phone_home/data.yaml b/doc/module-docs/cc_phone_home/data.yaml index 10721362eda..f5af3f6bde3 100644 --- a/doc/module-docs/cc_phone_home/data.yaml +++ b/doc/module-docs/cc_phone_home/data.yaml @@ -1,35 +1,40 @@ cc_phone_home: - name: Phone Home - title: Post data to url description: | This module can be used to post data to a remote host after boot is - complete. If the post url contains the string ``$INSTANCE_ID`` it will be - replaced with the id of the current instance. Either all data can be posted - or a list of keys to post. Available keys are: + complete. If the post URL contains the string ``$INSTANCE_ID`` it will be + replaced with the ID of the current instance. + + Either all data can be posted, or a list of keys to post. + Available keys are: + - ``pub_key_rsa`` - ``pub_key_ecdsa`` - ``pub_key_ed25519`` - ``instance_id`` - ``hostname`` - ``fdqn`` - + Data is sent as ``x-www-form-urlencoded`` arguments. - + **Example HTTP POST**: - + .. code-block:: http - - POST / HTTP/1.1 - Content-Length: 1337 - User-Agent: Cloud-Init/21.4 - Accept-Encoding: gzip, deflate - Accept: */* - Content-Type: application/x-www-form-urlencoded - - pub_key_rsa=rsa_contents&pub_key_ecdsa=ecdsa_contents&pub_key_ed25519=ed25519_contents&instance_id=i-87018aed&hostname=myhost&fqdn=myhost.internal + + POST / HTTP/1.1 + Content-Length: 1337 + User-Agent: Cloud-Init/21.4 + Accept-Encoding: gzip, deflate + Accept: */* + Content-Type: application/x-www-form-urlencoded + + pub_key_rsa=rsa_contents&pub_key_ecdsa=ecdsa_contents&pub_key_ed25519=ed25519_contents&instance_id=i-87018aed&hostname=myhost&fqdn=myhost.internal examples: - - comment: "" + - comment: | + Example 1: file: cc_phone_home/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_phone_home/example2.yaml + name: Phone Home + title: Post data to URL diff --git a/doc/module-docs/cc_phone_home/example1.yaml b/doc/module-docs/cc_phone_home/example1.yaml index e9fc281ad40..1278f497eb5 100644 --- a/doc/module-docs/cc_phone_home/example1.yaml +++ b/doc/module-docs/cc_phone_home/example1.yaml @@ -1,4 +1,2 @@ #cloud-config -phone_home: - url: http://example.com/$INSTANCE_ID/ - post: all +phone_home: {post: all, url: 'http://example.com/$INSTANCE_ID/'} diff --git a/doc/module-docs/cc_phone_home/example2.yaml b/doc/module-docs/cc_phone_home/example2.yaml index 94c4c69da43..fe9ec638f3b 100644 --- a/doc/module-docs/cc_phone_home/example2.yaml +++ b/doc/module-docs/cc_phone_home/example2.yaml @@ -1,11 +1,5 @@ #cloud-config phone_home: - url: http://example.com/$INSTANCE_ID/ - post: - - pub_key_rsa - - pub_key_ecdsa - - pub_key_ed25519 - - instance_id - - hostname - - fqdn + post: [pub_key_rsa, pub_key_ecdsa, pub_key_ed25519, instance_id, hostname, fqdn] tries: 5 + url: http://example.com/$INSTANCE_ID/ diff --git a/doc/module-docs/cc_power_state_change/data.yaml b/doc/module-docs/cc_power_state_change/data.yaml index b49e0da8641..2aed5081b2c 100644 --- a/doc/module-docs/cc_power_state_change/data.yaml +++ b/doc/module-docs/cc_power_state_change/data.yaml @@ -1,28 +1,28 @@ cc_power_state_change: - name: Power State Change - title: Change power state description: | - This module handles shutdown/reboot after all config modules have been run. By - default it will take no action, and the system will keep running unless a - package installation/upgrade requires a system reboot (e.g. installing a new - kernel) and ``package_reboot_if_required`` is true. - - Using this module ensures that cloud-init is entirely finished with - modules that would be executed. + This module handles shutdown/reboot after all config modules have been + run. By default it will take no action, and the system will keep running + unless a package installation/upgrade requires a system reboot (e.g. + installing a new kernel) and ``package_reboot_if_required`` is ``true``. - An example to distinguish delay from timeout: + Using this module ensures that cloud-init is entirely finished with modules + that would be executed. An example to distinguish delay from timeout: - If you delay 5 (5 minutes) and have a timeout of - 120 (2 minutes), then the max time until shutdown will be 7 minutes, though - it could be as soon as 5 minutes. Cloud-init will invoke 'shutdown +5' after - the process finishes, or when 'timeout' seconds have elapsed. + If you delay 5 (5 minutes) and have a timeout of 120 (2 minutes), the max + time until shutdown will be 7 minutes, though it could be as soon as 5 + minutes. Cloud-init will invoke 'shutdown +5' after the process finishes, + or when 'timeout' seconds have elapsed. .. note:: - With Alpine Linux any message value specified is ignored as Alpine's halt, - poweroff, and reboot commands do not support broadcasting a message. - + With Alpine Linux any message value specified is ignored as Alpine's + ``halt``, ``poweroff``, and ``reboot`` commands do not support + broadcasting a message. examples: - - comment: "" + - comment: | + Example 1: file: cc_power_state_change/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_power_state_change/example2.yaml + name: Power State Change + title: Change power state diff --git a/doc/module-docs/cc_puppet/data.yaml b/doc/module-docs/cc_puppet/data.yaml index 0d96f4c845a..47c5bb9ebbe 100644 --- a/doc/module-docs/cc_puppet/data.yaml +++ b/doc/module-docs/cc_puppet/data.yaml @@ -1,23 +1,28 @@ cc_puppet: - name: Puppet - title: Install, configure and start puppet description: | - This module handles puppet installation and configuration. If the + This module handles Puppet installation and configuration. If the ``puppet`` key does not exist in global configuration, no action will be - taken. If a config entry for ``puppet`` is present, then by default the - latest version of puppet will be installed. If the ``puppet`` config key - exists in the config archive, this module will attempt to start puppet even - if no installation was performed. + taken. + + If a config entry for ``puppet`` is present, then by default the latest + version of Puppet will be installed. If the ``puppet`` config key exists in + the config archive, this module will attempt to start puppet even if no + installation was performed. + + The module also provides keys for configuring the new Puppet 4 paths and + installing the ``puppet`` package from the + `puppetlabs repositories `_. - The module also provides keys for configuring the new puppet 4 paths and - installing the puppet package from the puppetlabs repositories: - https://docs.puppet.com/puppet/4.2/reference/whered_it_go.html The keys are ``package_name``, ``conf_file``, ``ssl_dir`` and - ``csr_attributes_path``. If unset, their values will default to - ones that work with puppet 3.x and with distributions that ship modified - puppet 4.x that uses the old paths. + ``csr_attributes_path``. If unset, their values will default to ones that + work with Puppet 3.X, and with distributions that ship modified Puppet 4.X, + that use the old paths. examples: - - comment: "" + - comment: | + Example 1: file: cc_puppet/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_puppet/example2.yaml + name: Puppet + title: Install, configure and start Puppet diff --git a/doc/module-docs/cc_resizefs/data.yaml b/doc/module-docs/cc_resizefs/data.yaml index f846f3df837..909778c9a92 100644 --- a/doc/module-docs/cc_resizefs/data.yaml +++ b/doc/module-docs/cc_resizefs/data.yaml @@ -1,18 +1,25 @@ cc_resizefs: - name: Resizefs - title: Resize filesystem description: | - Resize a filesystem to use all available space on partition. This - module is useful along with ``cc_growpart`` and will ensure that if the - root partition has been resized the root filesystem will be resized - along with it. By default, ``cc_resizefs`` will resize the root - partition and will block the boot process while the resize command is - running. Optionally, the resize operation can be performed in the - background while cloud-init continues running modules. This can be - enabled by setting ``resize_rootfs`` to ``noblock``. This module can be - disabled altogether by setting ``resize_rootfs`` to ``false``. + Resize a filesystem to use all available space on partition. This module is + useful along with ``cc_growpart`` and will ensure that if the root + partition has been resized, the root filesystem will be resized along with + it. + + By default, ``cc_resizefs`` will resize the root partition and will block + the boot process while the ``resize`` command is running. + + Optionally, the resize operation can be performed in the background + while cloud-init continues running modules. This can be enabled by setting + ``resize_rootfs`` to ``noblock``. + + This module can be disabled altogether by setting ``resize_rootfs`` to + ``false``. examples: - - comment: Disable root filesystem resize operation + - comment: | + Example 1: Disable root filesystem resize operation. file: cc_resizefs/example1.yaml - - comment: Runs resize operation in the background + - comment: | + Example 2: Runs resize operation in the background. file: cc_resizefs/example2.yaml + name: Resizefs + title: Resize filesystem diff --git a/doc/module-docs/cc_resolv_conf/data.yaml b/doc/module-docs/cc_resolv_conf/data.yaml index cce8f0b4545..f775c77eda3 100644 --- a/doc/module-docs/cc_resolv_conf/data.yaml +++ b/doc/module-docs/cc_resolv_conf/data.yaml @@ -1,32 +1,33 @@ cc_resolv_conf: - name: Resolv Conf - title: Configure resolv.conf description: | - Unless manually editing :file:`/etc/resolv.conf` is the correct way to - manage nameserver information on your operating system, you do not want to - use this module. Many distros have moved away from manually editing - ``resolv.conf`` so please verify that this is the preferred nameserver - management method for your distro before using this module. - - Note that using :ref:`network_config` is preferred, rather than using this - module, when possible. + You should not use this module unless manually editing + :file:`/etc/resolv.conf` is the correct way to manage nameserver + information on your operating system. + + Many distros have moved away from manually editing ``resolv.conf`` so + please verify that this is the preferred nameserver management method for + your distro before using this module. Note that using :ref:`network_config` + is preferred, rather than using this module, when possible. - This module is intended to manage resolv.conf in environments where early - configuration of resolv.conf is necessary for further bootstrapping and/or - where configuration management such as puppet or chef own DNS - configuration. + This module is intended to manage ``resolv.conf`` in environments where + early configuration of ``resolv.conf`` is necessary for further + bootstrapping and/or where configuration management such as Puppet or Chef + own DNS configuration. When using a :ref:`datasource_config_drive` and a RHEL-like system, - resolv.conf will also be managed automatically due to the available + ``resolv.conf`` will also be managed automatically due to the available information provided for DNS servers in the :ref:`network_config_v2` - format. For those that wish to have different settings, use this module. + format. For those who wish to have different settings, use this module. - In order for the ``resolv_conf`` section to be applied, - ``manage_resolv_conf`` must be set ``true``. + For the ``resolv_conf`` section to be applied, ``manage_resolv_conf`` must + be set ``true``. .. note:: - For Red Hat with sysconfig, be sure to set PEERDNS=no for all DHCP - enabled NICs. + For Red Hat with ``sysconfig``, be sure to set ``PEERDNS=no`` for all + DHCP-enabled NICs. examples: - - comment: "" + - comment: | + Example 1: file: cc_resolv_conf/example1.yaml + name: Resolv Conf + title: Configure ``resolv.conf`` diff --git a/doc/module-docs/cc_resolv_conf/example1.yaml b/doc/module-docs/cc_resolv_conf/example1.yaml index 3f97dbb3c5a..3c990cd11ff 100644 --- a/doc/module-docs/cc_resolv_conf/example1.yaml +++ b/doc/module-docs/cc_resolv_conf/example1.yaml @@ -1,16 +1,8 @@ #cloud-config manage_resolv_conf: true resolv_conf: - nameservers: - - 8.8.8.8 - - 8.8.4.4 - searchdomains: - - foo.example.com - - bar.example.com domain: example.com - sortlist: - - 10.0.0.1/255 - - 10.0.0.2 - options: - rotate: true - timeout: 1 + nameservers: [8.8.8.8, 8.8.4.4] + options: {rotate: true, timeout: 1} + searchdomains: [foo.example.com, bar.example.com] + sortlist: [10.0.0.1/255, 10.0.0.2] diff --git a/doc/module-docs/cc_rh_subscription/data.yaml b/doc/module-docs/cc_rh_subscription/data.yaml index 8632222c822..e87a95afbb2 100644 --- a/doc/module-docs/cc_rh_subscription/data.yaml +++ b/doc/module-docs/cc_rh_subscription/data.yaml @@ -1,19 +1,25 @@ cc_rh_subscription: - name: Red Hat Subscription - title: Register Red Hat Enterprise Linux based system description: | - Register a Red Hat system either by username and password *or* activation - and org. Following a successful registration, you can: + Register a Red Hat system, either by username and password **or** by + activation and org. + + Following a successful registration, you can: - auto-attach subscriptions - set the service level - - add subscriptions based on pool id - - enable/disable yum repositories based on repo id - - alter the rhsm_baseurl and server-hostname in ``/etc/rhsm/rhs.conf``. + - add subscriptions based on pool ID + - enable/disable yum repositories based on repo ID + - alter the ``rhsm_baseurl`` and ``server-hostname`` in + ``/etc/rhsm/rhs.conf``. examples: - - comment: "" + - comment: | + Example 1: file: cc_rh_subscription/example1.yaml - - comment: "" + - comment: | + Example 2: file: cc_rh_subscription/example2.yaml - - comment: "" + - comment: | + Example 3: file: cc_rh_subscription/example3.yaml + name: Red Hat Subscription + title: Register Red Hat Enterprise Linux-based system diff --git a/doc/module-docs/cc_rsyslog/data.yaml b/doc/module-docs/cc_rsyslog/data.yaml new file mode 100644 index 00000000000..2c045499d2f --- /dev/null +++ b/doc/module-docs/cc_rsyslog/data.yaml @@ -0,0 +1,28 @@ +cc_rsyslog: + description: | + This module configures remote system logging using rsyslog. + + Configuration for remote servers can be specified in ``configs``, but for + convenience it can be specified as key-value pairs in ``remotes``. + + This module can install rsyslog if not already present on the system using + the ``install_rsyslog``, ``packages``, and ``check_exe`` options. + Installation may not work on systems where this module runs before + networking is up. + + .. note:: + On BSD, cloud-init will attempt to disable and stop the base system + syslogd. This may fail on a first run. We recommend creating images + with ``service syslogd disable``. + examples: + - comment: | + Example 1: + file: cc_rsyslog/example1.yaml + - comment: | + Example 2: + file: cc_rsyslog/example2.yaml + - comment: | + Example 3: Default (no) configuration with package installation on FreeBSD. + file: cc_rsyslog/example3.yaml + name: Rsyslog + title: Configure system logging via rsyslog diff --git a/doc/module-docs/cc_rsyslog/example1.yaml b/doc/module-docs/cc_rsyslog/example1.yaml new file mode 100644 index 00000000000..8abbc1eeb21 --- /dev/null +++ b/doc/module-docs/cc_rsyslog/example1.yaml @@ -0,0 +1,4 @@ +#cloud-config +rsyslog: + remotes: {juju: 10.0.4.1, maas: 192.168.1.1} + service_reload_command: auto diff --git a/doc/module-docs/cc_rsyslog/example2.yaml b/doc/module-docs/cc_rsyslog/example2.yaml new file mode 100644 index 00000000000..a33c85486a2 --- /dev/null +++ b/doc/module-docs/cc_rsyslog/example2.yaml @@ -0,0 +1,12 @@ +#cloud-config +rsyslog: + config_dir: /opt/etc/rsyslog.d + config_filename: 99-late-cloud-config.conf + configs: + - '*.* @@192.158.1.1' + - {content: '*.* @@192.0.2.1:10514', filename: 01-example.conf} + - {content: '*.* @@syslogd.example.com + + '} + remotes: {juju: 10.0.4.1, maas: 192.168.1.1} + service_reload_command: [your, syslog, restart, command] diff --git a/doc/module-docs/cc_rsyslog/example3.yaml b/doc/module-docs/cc_rsyslog/example3.yaml new file mode 100644 index 00000000000..bf21f8bd04c --- /dev/null +++ b/doc/module-docs/cc_rsyslog/example3.yaml @@ -0,0 +1,6 @@ +#cloud-config +rsyslog: + check_exe: rsyslogd + config_dir: /usr/local/etc/rsyslog.d + install_rsyslog: true + packages: [rsyslogd] diff --git a/doc/module-docs/cc_runcmd/data.yaml b/doc/module-docs/cc_runcmd/data.yaml new file mode 100644 index 00000000000..54915fd7308 --- /dev/null +++ b/doc/module-docs/cc_runcmd/data.yaml @@ -0,0 +1,27 @@ +cc_runcmd: + description: | + Run arbitrary commands at a ``rc.local``-like time-frame with output to the + console. Each item can be either a list or a string. The item type affects + how it is executed: + + - If the item is a string, it will be interpreted by ``sh``. + - If the item is a list, the items will be executed as if passed to + ``execve(3)`` (with the first argument as the command). + + The ``runcmd`` module only writes the script to be run later. The module + that actually runs the script is ``scripts_user`` in the + :ref:`Final boot stage `. + + .. note:: + All commands must be proper YAML, so you must quote any characters YAML + would eat (":" can be problematic). + + .. note:: + When writing files, do not use ``/tmp`` dir as it races with + ``systemd-tmpfiles-clean`` (LP: #1707222). Use ``/run/somedir`` instead. + examples: + - comment: | + Example 1: + file: cc_runcmd/example1.yaml + name: Runcmd + title: Run arbitrary commands diff --git a/doc/module-docs/cc_runcmd/example1.yaml b/doc/module-docs/cc_runcmd/example1.yaml new file mode 100644 index 00000000000..03812f926e9 --- /dev/null +++ b/doc/module-docs/cc_runcmd/example1.yaml @@ -0,0 +1,7 @@ +#cloud-config +runcmd: +- [ls, -l, /] +- [sh, -xc, 'echo $(date) '': hello world!'''] +- [sh, -c, echo "=========hello world'========="] +- ls -l /root +- [wget, 'http://example.org', -O, /tmp/index.html] diff --git a/doc/module-docs/cc_salt_minion/data.yaml b/doc/module-docs/cc_salt_minion/data.yaml new file mode 100644 index 00000000000..d0003c2f7ca --- /dev/null +++ b/doc/module-docs/cc_salt_minion/data.yaml @@ -0,0 +1,24 @@ +cc_salt_minion: + description: | + This module installs, configures and starts Salt Minion. If the + ``salt_minion`` key is present in the config parts, then Salt Minion will + be installed and started. + + Configuration for Salt Minion can be specified in the ``conf`` key under + ``salt_minion``. Any config values present there will be assigned in + ``/etc/salt/minion``. The public and private keys to use for Salt Minion + can be specified with ``public_key`` and ``private_key`` respectively. + + If you have a custom package name, service name, or config directory, you + can specify them with ``pkg_name``, ``service_name``, and ``config_dir`` + respectively. + + Salt keys can be manually generated by ``salt-key --gen-keys=GEN_KEYS``, + where ``GEN_KEYS`` is the name of the keypair, e.g. ''minion''. The keypair + will be copied to ``/etc/salt/pki`` on the Minion instance. + examples: + - comment: | + Example 1: + file: cc_salt_minion/example1.yaml + name: Salt Minion + title: Set up and run Salt Minion diff --git a/doc/module-docs/cc_salt_minion/example1.yaml b/doc/module-docs/cc_salt_minion/example1.yaml new file mode 100644 index 00000000000..f4bf6c961d6 --- /dev/null +++ b/doc/module-docs/cc_salt_minion/example1.yaml @@ -0,0 +1,27 @@ +#cloud-config +salt_minion: + conf: + file_client: local + fileserver_backend: [gitfs] + gitfs_remotes: ['https://github.com/_user_/_repo_.git'] + master: salt.example.com + config_dir: /etc/salt + grains: + role: [web] + pkg_name: salt-minion + pki_dir: /etc/salt/pki/minion + private_key: '------BEGIN PRIVATE KEY------ + + + + ------END PRIVATE KEY------- + + ' + public_key: '------BEGIN PUBLIC KEY------- + + + + ------END PUBLIC KEY------- + + ' + service_name: salt-minion diff --git a/doc/module-docs/cc_scripts_per_boot/data.yaml b/doc/module-docs/cc_scripts_per_boot/data.yaml new file mode 100644 index 00000000000..16ed96fbcfb --- /dev/null +++ b/doc/module-docs/cc_scripts_per_boot/data.yaml @@ -0,0 +1,8 @@ +cc_scripts_per_boot: + description: | + Any scripts in the ``scripts/per-boot`` directory on the datasource will + be run every time the system boots. Scripts will be run in alphabetical + order. This module does not accept any config keys. + examples: [] + name: Scripts Per Boot + title: Run per-boot scripts diff --git a/doc/module-docs/cc_scripts_per_instance/data.yaml b/doc/module-docs/cc_scripts_per_instance/data.yaml new file mode 100644 index 00000000000..8d3de066300 --- /dev/null +++ b/doc/module-docs/cc_scripts_per_instance/data.yaml @@ -0,0 +1,11 @@ +cc_scripts_per_instance: + description: | + Any scripts in the ``scripts/per-instance`` directory on the datasource + will be run when a new instance is first booted. Scripts will be run in + alphabetical order. This module does not accept any config keys. + + Some cloud platforms change ``instance-id`` if a significant change was + made to the system. As a result, per-instance scripts will run again. + examples: [] + name: Scripts Per Instance + title: Run per-instance scripts diff --git a/doc/module-docs/cc_scripts_per_once/data.yaml b/doc/module-docs/cc_scripts_per_once/data.yaml new file mode 100644 index 00000000000..6f9f2ad57c9 --- /dev/null +++ b/doc/module-docs/cc_scripts_per_once/data.yaml @@ -0,0 +1,10 @@ +cc_scripts_per_once: + description: | + Any scripts in the ``scripts/per-once`` directory on the datasource will + be run only once. Changes to the instance will not force a re-run. The + only way to re-run these scripts is to run the ``clean`` subcommand and + reboot. Scripts will be run in alphabetical order. This module does not + accept any config keys. + examples: [] + name: Scripts Per Once + title: Run one-time scripts diff --git a/doc/module-docs/cc_scripts_user/data.yaml b/doc/module-docs/cc_scripts_user/data.yaml new file mode 100644 index 00000000000..120cc2e0704 --- /dev/null +++ b/doc/module-docs/cc_scripts_user/data.yaml @@ -0,0 +1,10 @@ +cc_scripts_user: + description: | + This module runs all user scripts present in the ``scripts`` directory in + the instance configuration. Any cloud-config parts with a ``#!`` will be + treated as a script and run. Scripts specified as cloud-config parts will + be run in the order they are specified in the configuration. This module + does not accept any config keys. + examples: [] + name: Scripts User + title: Run user scripts diff --git a/doc/module-docs/cc_scripts_vendor/data.yaml b/doc/module-docs/cc_scripts_vendor/data.yaml new file mode 100644 index 00000000000..51ab8337c3d --- /dev/null +++ b/doc/module-docs/cc_scripts_vendor/data.yaml @@ -0,0 +1,19 @@ +cc_scripts_vendor: + description: | + On select Datasources, vendors have a channel for the consumption of all + supported user data types via a special channel called vendor data. Any + scripts in the ``scripts/vendor`` directory in the datasource will be run + when a new instance is first booted. Scripts will be run in alphabetical + order. This module allows control over the execution of vendor data. + examples: + - comment: | + Example 1: + file: cc_scripts_vendor/example1.yaml + - comment: | + Example 2: + file: cc_scripts_vendor/example2.yaml + - comment: | + Example 3: Vendor data will not be processed. + file: cc_scripts_vendor/example3.yaml + name: Scripts Vendor + title: Run vendor scripts diff --git a/doc/module-docs/cc_scripts_vendor/example1.yaml b/doc/module-docs/cc_scripts_vendor/example1.yaml new file mode 100644 index 00000000000..a6e0d7265cb --- /dev/null +++ b/doc/module-docs/cc_scripts_vendor/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +vendor_data: {enabled: true, prefix: /usr/bin/ltrace} diff --git a/doc/module-docs/cc_scripts_vendor/example2.yaml b/doc/module-docs/cc_scripts_vendor/example2.yaml new file mode 100644 index 00000000000..3427396027f --- /dev/null +++ b/doc/module-docs/cc_scripts_vendor/example2.yaml @@ -0,0 +1,4 @@ +#cloud-config +vendor_data: + enabled: true + prefix: [timeout, 30] diff --git a/doc/module-docs/cc_scripts_vendor/example3.yaml b/doc/module-docs/cc_scripts_vendor/example3.yaml new file mode 100644 index 00000000000..97651375b75 --- /dev/null +++ b/doc/module-docs/cc_scripts_vendor/example3.yaml @@ -0,0 +1,2 @@ +#cloud-config +vendor_data: {enabled: false} diff --git a/doc/module-docs/cc_seed_random/data.yaml b/doc/module-docs/cc_seed_random/data.yaml new file mode 100644 index 00000000000..f16383ce98a --- /dev/null +++ b/doc/module-docs/cc_seed_random/data.yaml @@ -0,0 +1,27 @@ +cc_seed_random: + description: | + All cloud instances started from the same image will produce similar data + when they are first booted as they are all starting with the same + seed for the kernel's entropy keyring. To avoid this, random seed data can + be provided to the instance, either as a string or by specifying a command + to run to generate the data. + + Configuration for this module is under the ``random_seed`` config key. If + the cloud provides its own random seed data, it will be appended to + ``data`` before it is written to ``file``. + + If the ``command`` key is specified, the given command will be executed. + This will happen after ``file`` has been populated. That command's + environment will contain the value of the ``file`` key as + ``RANDOM_SEED_FILE``. If a command is specified that cannot be run, no + error will be reported unless ``command_required`` is set to ``true``. + examples: + - comment: | + Example 1: + file: cc_seed_random/example1.yaml + - comment: > + Example 2: Use ``pollinate`` to gather data from a remote entropy + server and write it to ``/dev/urandom``: + file: cc_seed_random/example2.yaml + name: Seed Random + title: Provide random seed data diff --git a/doc/module-docs/cc_seed_random/example1.yaml b/doc/module-docs/cc_seed_random/example1.yaml new file mode 100644 index 00000000000..41c54e5a02b --- /dev/null +++ b/doc/module-docs/cc_seed_random/example1.yaml @@ -0,0 +1,7 @@ +#cloud-config +random_seed: + command: [sh, -c, dd if=/dev/urandom of=$RANDOM_SEED_FILE] + command_required: true + data: my random string + encoding: raw + file: /dev/urandom diff --git a/doc/module-docs/cc_seed_random/example2.yaml b/doc/module-docs/cc_seed_random/example2.yaml new file mode 100644 index 00000000000..9211c690d08 --- /dev/null +++ b/doc/module-docs/cc_seed_random/example2.yaml @@ -0,0 +1,5 @@ +#cloud-config +random_seed: + command: [pollinate, '--server=http://local.pollinate.server'] + command_required: true + file: /dev/urandom diff --git a/doc/module-docs/cc_set_hostname/data.yaml b/doc/module-docs/cc_set_hostname/data.yaml new file mode 100644 index 00000000000..f0e74b400b5 --- /dev/null +++ b/doc/module-docs/cc_set_hostname/data.yaml @@ -0,0 +1,42 @@ +cc_set_hostname: + description: | + This module handles setting the system hostname and fully qualified domain + name (FQDN). If ``preserve_hostname`` is set, then the hostname will not be + altered. + + A hostname and FQDN can be provided by specifying a full domain name under + the ``fqdn`` key. Alternatively, a hostname can be specified using the + ``hostname`` key, and the FQDN of the cloud will be used. If a FQDN is + specified with the ``hostname`` key, it will be handled properly, although + it is better to use the ``fqdn`` config key. If both ``fqdn`` and + ``hostname`` are set, then ``prefer_fqdn_over_hostname`` will force use of + FQDN in all distros when true, and when false it will force the short + hostname. Otherwise, the hostname to use is distro-dependent. + + .. note:: + Cloud-init performs no hostname input validation before sending the + hostname to distro-specific tools, and most tools will not accept a + trailing dot on the FQDN. + + This module will run in the init-local stage before networking is + configured if the hostname is set by metadata or user data on the local + system. + + This will occur on datasources like NoCloud and OVF where metadata and user + data are available locally. This ensures that the desired hostname is + applied before any DHCP requests are performed on these platforms where + dynamic DNS is based on initial hostname. + examples: + - comment: | + Example 1: + file: cc_set_hostname/example1.yaml + - comment: | + Example 2: + file: cc_set_hostname/example2.yaml + - comment: > + Example 3: On a machine without an ``/etc/hostname`` file, don't create + it. In most clouds, this will result in a DHCP-configured hostname + provided by the cloud. + file: cc_set_hostname/example3.yaml + name: Set Hostname + title: Set hostname and FQDN diff --git a/doc/module-docs/cc_set_hostname/example1.yaml b/doc/module-docs/cc_set_hostname/example1.yaml new file mode 100644 index 00000000000..6edec76adf6 --- /dev/null +++ b/doc/module-docs/cc_set_hostname/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +preserve_hostname: true diff --git a/doc/module-docs/cc_set_hostname/example2.yaml b/doc/module-docs/cc_set_hostname/example2.yaml new file mode 100644 index 00000000000..1bc1e2e7e00 --- /dev/null +++ b/doc/module-docs/cc_set_hostname/example2.yaml @@ -0,0 +1,5 @@ +#cloud-config +hostname: myhost +create_hostname_file: true +fqdn: myhost.example.com +prefer_fqdn_over_hostname: true diff --git a/doc/module-docs/cc_set_hostname/example3.yaml b/doc/module-docs/cc_set_hostname/example3.yaml new file mode 100644 index 00000000000..785e6167d89 --- /dev/null +++ b/doc/module-docs/cc_set_hostname/example3.yaml @@ -0,0 +1,2 @@ +#cloud-config +create_hostname_file: false diff --git a/doc/module-docs/cc_set_passwords/data.yaml b/doc/module-docs/cc_set_passwords/data.yaml new file mode 100644 index 00000000000..dae5b5d59d5 --- /dev/null +++ b/doc/module-docs/cc_set_passwords/data.yaml @@ -0,0 +1,40 @@ +cc_set_passwords: + description: | + This module consumes three top-level config keys: ``ssh_pwauth``, + ``chpasswd`` and ``password``. + + The ``ssh_pwauth`` config key determines whether or not sshd will be + configured to accept password authentication. + + The ``chpasswd`` config key accepts a dictionary containing either (or + both) of ``users`` and ``expire``. + + - The ``users`` key is used to assign a password to a corresponding + pre-existing user. + - The ``expire`` key is used to set whether to expire all user passwords + specified by this module, such that a password will need to be reset on + the user's next login. + + .. note:: + Prior to cloud-init 22.3, the ``expire`` key only applies to plain text + (including ``RANDOM``) passwords. Post-22.3, the ``expire`` key applies + to both plain text and hashed passwords. + + The ``password`` config key is used to set the default user's password. It + is ignored if the ``chpasswd`` ``users`` is used. Note that the ``list`` + keyword is deprecated in favor of ``users``. + examples: + - comment: | + Example 1: Set a default password, to be changed at first login. + file: cc_set_passwords/example1.yaml + - comment: | + Example 2: + - Disable SSH password authentication. + - Don't require users to change their passwords on next login. + - Set the password for user1 to be 'password1' (OS does hashing). + - Set the password for user2 to a pre-hashed password. + - Set the password for user3 to be a randomly generated password, which + will be written to the system console. + file: cc_set_passwords/example2.yaml + name: Set Passwords + title: Set user passwords and enable/disable SSH password auth diff --git a/doc/module-docs/cc_set_passwords/example1.yaml b/doc/module-docs/cc_set_passwords/example1.yaml new file mode 100644 index 00000000000..48742419110 --- /dev/null +++ b/doc/module-docs/cc_set_passwords/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +{password: password1, ssh_pwauth: true} diff --git a/doc/module-docs/cc_set_passwords/example2.yaml b/doc/module-docs/cc_set_passwords/example2.yaml new file mode 100644 index 00000000000..98ce8b5e8e2 --- /dev/null +++ b/doc/module-docs/cc_set_passwords/example2.yaml @@ -0,0 +1,8 @@ +#cloud-config +chpasswd: + expire: false + users: + - {name: user1, password: password1, type: text} + - {name: user2, password: $6$rounds=4096$5DJ8a9WMTEzIo5J4$Yms6imfeBvf3Yfu84mQBerh18l7OR1Wm1BJXZqFSpJ6BVas0AYJqIjP7czkOaAZHZi1kxQ5Y1IhgWN8K9NgxR1} + - {name: user3, type: RANDOM} +ssh_pwauth: false diff --git a/doc/module-docs/cc_snap/data.yaml b/doc/module-docs/cc_snap/data.yaml new file mode 100644 index 00000000000..c3ae845c572 --- /dev/null +++ b/doc/module-docs/cc_snap/data.yaml @@ -0,0 +1,42 @@ +cc_snap: + description: | + This module provides a simple configuration namespace in cloud-init for + setting up snapd and installing snaps. + + Both ``assertions`` and ``commands`` values can be either a dictionary or a + list. If these configs are provided as a dictionary, the keys are only used + to order the execution of the assertions or commands and the dictionary is + merged with any vendor data the snap configuration provided. If a list is + provided by the user instead of a dict, any vendor data snap configuration + is ignored. + + The ``assertions`` configuration option is a dictionary or list of + properly-signed snap assertions, which will run before any snap commands. + They will be added to snapd's ``assertion`` database by invoking + ``snap ack ``. + + Snap ``commands`` is a dictionary or list of individual snap commands to + run on the target system. These commands can be used to create snap users, + install snaps, and provide snap configuration. + + .. note:: + If 'side-loading' private/unpublished snaps on an instance, it is best + to create a snap seed directory and ``seed.yaml`` manifest in + ``/var/lib/snapd/seed/`` which snapd automatically installs on startup. + examples: + - comment: | + Example 1: + file: cc_snap/example1.yaml + - comment: > + Example 2: For convenience, the ``snap`` command can be omitted when + specifying commands as a list - ``snap`` will be automatically prepended. + The following commands are all equivalent: + file: cc_snap/example2.yaml + - comment: | + Example 3: You can use a list of commands. + file: cc_snap/example4.yaml + - comment: | + Example 4: You can also use a list of assertions. + file: cc_snap/example4.yaml + name: Snap + title: Install, configure and manage snapd and snap packages diff --git a/doc/module-docs/cc_snap/example1.yaml b/doc/module-docs/cc_snap/example1.yaml new file mode 100644 index 00000000000..a65072fdbf1 --- /dev/null +++ b/doc/module-docs/cc_snap/example1.yaml @@ -0,0 +1,11 @@ +#cloud-config +snap: + assertions: + 00: | + signed_assertion_blob_here + 02: | + signed_assertion_blob_here + commands: + 00: snap create-user --sudoer --known @mydomain.com + 01: snap install canonical-livepatch + 02: canonical-livepatch enable diff --git a/doc/module-docs/cc_snap/example2.yaml b/doc/module-docs/cc_snap/example2.yaml new file mode 100644 index 00000000000..615c7c741a7 --- /dev/null +++ b/doc/module-docs/cc_snap/example2.yaml @@ -0,0 +1,7 @@ +#cloud-config +snap: + commands: + 0: [install, vlc] + 1: [snap, install, vlc] + 2: snap install vlc + 3: snap install vlc diff --git a/doc/module-docs/cc_snap/example3.yaml b/doc/module-docs/cc_snap/example3.yaml new file mode 100644 index 00000000000..0aeb0f6ae23 --- /dev/null +++ b/doc/module-docs/cc_snap/example3.yaml @@ -0,0 +1,7 @@ +#cloud-config +snap: + commands: + - [install, vlc] + - [snap, install, vlc] + - snap install vlc + - snap install vlc diff --git a/doc/module-docs/cc_snap/example4.yaml b/doc/module-docs/cc_snap/example4.yaml new file mode 100644 index 00000000000..5f7ed529953 --- /dev/null +++ b/doc/module-docs/cc_snap/example4.yaml @@ -0,0 +1,6 @@ +#cloud-config +snap: + assertions: + - signed_assertion_blob_here + - | + signed_assertion_blob_here diff --git a/doc/module-docs/cc_spacewalk/data.yaml b/doc/module-docs/cc_spacewalk/data.yaml new file mode 100644 index 00000000000..b7e556f411c --- /dev/null +++ b/doc/module-docs/cc_spacewalk/data.yaml @@ -0,0 +1,16 @@ +cc_spacewalk: + description: | + This module installs Spacewalk and applies basic configuration. If the + Spacewalk config key is present, Spacewalk will be installed. The server + to connect to after installation must be provided in the ``server`` in + Spacewalk configuration. A proxy to connect through and an activation key + may optionally be specified. + + For more details about spacewalk see the + `Fedora documentation `_. + examples: + - comment: | + Example 1: + file: cc_spacewalk/example1.yaml + name: Spacewalk + title: Install and configure spacewalk diff --git a/doc/module-docs/cc_spacewalk/example1.yaml b/doc/module-docs/cc_spacewalk/example1.yaml new file mode 100644 index 00000000000..810c9fdbfff --- /dev/null +++ b/doc/module-docs/cc_spacewalk/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +spacewalk: {activation_key: , proxy: , server: } diff --git a/doc/module-docs/cc_ssh/data.yaml b/doc/module-docs/cc_ssh/data.yaml new file mode 100644 index 00000000000..ccb21375bc0 --- /dev/null +++ b/doc/module-docs/cc_ssh/data.yaml @@ -0,0 +1,94 @@ +cc_ssh: + description: | + This module handles most configuration for SSH, and for both host and + authorized SSH keys. + + **Authorized keys** + + Authorized keys are a list of public SSH keys that are allowed to connect + to a user account on a system. They are stored in ``.ssh/authorized_keys`` + in that account's home directory. Authorized keys for the default user + defined in ``users`` can be specified using ``ssh_authorized_keys``. Keys + should be specified as a list of public keys. + + .. note:: + See the ``cc_set_passwords`` module documentation to enable/disable SSH + password authentication. + + Root login can be enabled/disabled using the ``disable_root`` config key. + Root login options can be manually specified with ``disable_root_opts``. + + Supported public key types for the ssh_authorized_keys are: + + - rsa + - ecdsa + - ed25519 + - ecdsa-sha2-nistp256-cert-v01@openssh.com + - ecdsa-sha2-nistp256 + - ecdsa-sha2-nistp384-cert-v01@openssh.com + - ecdsa-sha2-nistp384 + - ecdsa-sha2-nistp521-cert-v01@openssh.com + - ecdsa-sha2-nistp521 + - sk-ecdsa-sha2-nistp256-cert-v01@openssh.com + - sk-ecdsa-sha2-nistp256@openssh.com + - sk-ssh-ed25519-cert-v01@openssh.com + - sk-ssh-ed25519@openssh.com + - ssh-ed25519-cert-v01@openssh.com + - ssh-ed25519 + - ssh-rsa-cert-v01@openssh.com + - ssh-rsa + - ssh-xmss-cert-v01@openssh.com + - ssh-xmss@openssh.com + + .. note:: + This list has been filtered out from the supported key types of + `OpenSSH `_ + source, where the ``sigonly`` keys are removed. See ``ssh_util`` for + more information. + + ``rsa``, ``ecdsa`` and ``ed25519`` are added for legacy, as they are + valid public keys in some older distros. They may be removed in the + future when support for the older distros is dropped. + + **Host keys** + + Host keys are for authenticating a specific instance. Many images have + default host SSH keys, which can be removed using ``ssh_deletekeys``. + + Host keys can be added using the ``ssh_keys`` configuration key. + + When host keys are generated the output of the ``ssh-keygen`` command(s) + can be displayed on the console using the ``ssh_quiet_keygen`` + configuration key. + + .. note:: + When specifying private host keys in cloud-config, take care to ensure + that communication between the data source and the instance is secure. + + If no host keys are specified using ``ssh_keys``, then keys will be + generated using ``ssh-keygen``. By default, one public/private pair of + each supported host key type will be generated. The key types to generate + can be specified using the ``ssh_genkeytypes`` config flag, which accepts a + list of host key types to use. For each host key type for which this module + has been instructed to create a keypair, if a key of the same type is + already present on the system (i.e. if ``ssh_deletekeys`` was set to + false), no key will be generated. + + Supported host key types for the ``ssh_keys`` and the ``ssh_genkeytypes`` + config flags are: + + - ecdsa + - ed25519 + - rsa + + Unsupported host key types for the ``ssh_keys`` and the ``ssh_genkeytypes`` + config flags are: + + - ecdsa-sk + - ed25519-sk + examples: + - comment: | + Example 1: + file: cc_ssh/example1.yaml + name: SSH + title: Configure SSH and SSH keys diff --git a/doc/module-docs/cc_ssh/example1.yaml b/doc/module-docs/cc_ssh/example1.yaml new file mode 100644 index 00000000000..cc54f6ede67 --- /dev/null +++ b/doc/module-docs/cc_ssh/example1.yaml @@ -0,0 +1,24 @@ +#cloud-config +allow_public_ssh_keys: true +disable_root: true +disable_root_opts: no-port-forwarding,no-agent-forwarding,no-X11-forwarding +ssh_authorized_keys: [ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEA3FSyQwBI6Z+nCSjUU ..., ssh-rsa + AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZ ...] +ssh_deletekeys: true +ssh_genkeytypes: [rsa, ecdsa, ed25519] +ssh_keys: {rsa_certificate: 'ssh-rsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt + ... + + ', rsa_private: '-----BEGIN RSA PRIVATE KEY----- + + MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qco + + ... + + -----END RSA PRIVATE KEY----- + + ', rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ...} +ssh_publish_hostkeys: + blacklist: [rsa] + enabled: true +ssh_quiet_keygen: true diff --git a/doc/module-docs/cc_ssh_authkey_fingerprints/data.yaml b/doc/module-docs/cc_ssh_authkey_fingerprints/data.yaml new file mode 100644 index 00000000000..c034d5d2a6e --- /dev/null +++ b/doc/module-docs/cc_ssh_authkey_fingerprints/data.yaml @@ -0,0 +1,14 @@ +cc_ssh_authkey_fingerprints: + description: | + Write fingerprints of authorized keys for each user to log. This is enabled + by default, but can be disabled using ``no_ssh_fingerprints``. The hash + type for the keys can be specified, but defaults to ``sha256``. + examples: + - comment: | + Example 1: + file: cc_ssh_authkey_fingerprints/example1.yaml + - comment: | + Example 2: + file: cc_ssh_authkey_fingerprints/example2.yaml + name: SSH AuthKey Fingerprints + title: Log fingerprints of user SSH keys diff --git a/doc/module-docs/cc_ssh_authkey_fingerprints/example1.yaml b/doc/module-docs/cc_ssh_authkey_fingerprints/example1.yaml new file mode 100644 index 00000000000..7bae7a9d71f --- /dev/null +++ b/doc/module-docs/cc_ssh_authkey_fingerprints/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +no_ssh_fingerprints: true diff --git a/doc/module-docs/cc_ssh_authkey_fingerprints/example2.yaml b/doc/module-docs/cc_ssh_authkey_fingerprints/example2.yaml new file mode 100644 index 00000000000..2212058fe89 --- /dev/null +++ b/doc/module-docs/cc_ssh_authkey_fingerprints/example2.yaml @@ -0,0 +1,2 @@ +#cloud-config +authkey_hash: sha512 diff --git a/doc/module-docs/cc_ssh_import_id/data.yaml b/doc/module-docs/cc_ssh_import_id/data.yaml new file mode 100644 index 00000000000..2421309836a --- /dev/null +++ b/doc/module-docs/cc_ssh_import_id/data.yaml @@ -0,0 +1,13 @@ +cc_ssh_import_id: + description: | + This module imports SSH keys from either a public keyserver (usually + Launchpad), or GitHub, using ``ssh-import-id``. Keys are referenced by the + username they are associated with on the keyserver. The keyserver can be + specified by prepending either ``lp:`` for Launchpad or ``gh:`` for + GitHub to the username. + examples: + - comment: | + Example 1: + file: cc_ssh_import_id/example1.yaml + name: SSH Import ID + title: Import SSH ID diff --git a/doc/module-docs/cc_ssh_import_id/example1.yaml b/doc/module-docs/cc_ssh_import_id/example1.yaml new file mode 100644 index 00000000000..c52bc9a5124 --- /dev/null +++ b/doc/module-docs/cc_ssh_import_id/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +ssh_import_id: [user, 'gh:user', 'lp:user'] diff --git a/doc/module-docs/cc_timezone/data.yaml b/doc/module-docs/cc_timezone/data.yaml new file mode 100644 index 00000000000..def136d735f --- /dev/null +++ b/doc/module-docs/cc_timezone/data.yaml @@ -0,0 +1,10 @@ +cc_timezone: + description: | + Sets the `system timezone `_ based on the + value provided. + examples: + - comment: | + Example 1: + file: cc_timezone/example1.yaml + name: Timezone + title: Set the system timezone diff --git a/doc/module-docs/cc_timezone/example1.yaml b/doc/module-docs/cc_timezone/example1.yaml new file mode 100644 index 00000000000..c54cadce4e2 --- /dev/null +++ b/doc/module-docs/cc_timezone/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +timezone: US/Eastern diff --git a/doc/module-docs/cc_ubuntu_drivers/data.yaml b/doc/module-docs/cc_ubuntu_drivers/data.yaml new file mode 100644 index 00000000000..d704a620fb1 --- /dev/null +++ b/doc/module-docs/cc_ubuntu_drivers/data.yaml @@ -0,0 +1,10 @@ +cc_ubuntu_drivers: + description: | + This module interacts with the ``ubuntu-drivers`` command to install third + party driver packages. + examples: + - comment: | + Example 1: + file: cc_ubuntu_drivers/example1.yaml + name: Ubuntu Drivers + title: Interact with third party drivers in Ubuntu diff --git a/doc/module-docs/cc_ubuntu_drivers/example1.yaml b/doc/module-docs/cc_ubuntu_drivers/example1.yaml new file mode 100644 index 00000000000..659c904bd2e --- /dev/null +++ b/doc/module-docs/cc_ubuntu_drivers/example1.yaml @@ -0,0 +1,3 @@ +#cloud-config +drivers: + nvidia: {license-accepted: true} diff --git a/doc/module-docs/cc_ubuntu_pro/data.yaml b/doc/module-docs/cc_ubuntu_pro/data.yaml new file mode 100644 index 00000000000..3706f638132 --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/data.yaml @@ -0,0 +1,52 @@ +cc_ubuntu_pro: + description: | + Attach machine to an existing Ubuntu Pro support contract and enable or + disable support services such as Livepatch, ESM, FIPS and FIPS Updates. + + When attaching a machine to Ubuntu Pro, one can also specify services to + enable. When the ``enable`` list is present, only named services will be + activated. If the ``enable`` list is not present, the contract's default + services will be enabled. + + On Pro instances, when ``ubuntu_pro`` config is provided to cloud-init, + Pro's auto-attach feature will be disabled and cloud-init will perform + the Pro auto-attach, ignoring the ``token`` key. The ``enable`` and + ``enable_beta`` values will strictly determine what services will be + enabled, ignoring contract defaults. + + Note that when enabling FIPS or FIPS updates you will need to schedule a + reboot to ensure the machine is running the FIPS-compliant kernel. See the + Power State Change module for information on how to configure cloud-init to + perform this reboot. + examples: + - comment: > + Example 1: Attach the machine to an Ubuntu Pro support contract with a + Pro contract token obtained from https://ubuntu.com/pro. + file: cc_ubuntu_pro/example1.yaml + - comment: > + Example 2: Attach the machine to an Ubuntu Pro support contract, enabling + only FIPS and ESM services. Services will only be enabled if the + environment supports that service. Otherwise, warnings will be logged for + incompatible services. + file: cc_ubuntu_pro/example2.yaml + - comment: > + Example 3: Attach the machine to an Ubuntu Pro support contract and + enable the FIPS service. Perform a reboot once cloud-init has completed. + file: cc_ubuntu_pro/example3.yaml + - comment: > + Example 4: Set a HTTP(s) proxy before attaching the machine to an Ubuntu + Pro support contract and enabling the FIPS service. + file: cc_ubuntu_pro/example4.yaml + - comment: > + Example 5: On Ubuntu Pro instances, auto-attach but don't enable any Pro + services. + file: cc_ubuntu_pro/example5.yaml + - comment: > + Example 6: Enable ESM and beta Real-time Ubuntu services in Ubuntu Pro + instances. + file: cc_ubuntu_pro/example6.yaml + - comment: > + Example 7: Disable auto-attach in Ubuntu Pro instances. + file: cc_ubuntu_pro/example7.yaml + name: Ubuntu Pro + title: Configure Ubuntu Pro support services diff --git a/doc/module-docs/cc_ubuntu_pro/example1.yaml b/doc/module-docs/cc_ubuntu_pro/example1.yaml new file mode 100644 index 00000000000..ae9cdf2f23c --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +ubuntu_pro: {token: } diff --git a/doc/module-docs/cc_ubuntu_pro/example2.yaml b/doc/module-docs/cc_ubuntu_pro/example2.yaml new file mode 100644 index 00000000000..c9cf5c3bf65 --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/example2.yaml @@ -0,0 +1,4 @@ +#cloud-config +ubuntu_pro: + enable: [fips, esm] + token: diff --git a/doc/module-docs/cc_ubuntu_pro/example3.yaml b/doc/module-docs/cc_ubuntu_pro/example3.yaml new file mode 100644 index 00000000000..e4aa01a01e1 --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/example3.yaml @@ -0,0 +1,5 @@ +#cloud-config +power_state: {mode: reboot} +ubuntu_pro: + enable: [fips] + token: diff --git a/doc/module-docs/cc_ubuntu_pro/example4.yaml b/doc/module-docs/cc_ubuntu_pro/example4.yaml new file mode 100644 index 00000000000..5fac5375081 --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/example4.yaml @@ -0,0 +1,12 @@ +#cloud-config +ubuntu_pro: + token: + config: + http_proxy: 'http://some-proxy:8088' + https_proxy: 'https://some-proxy:8088' + global_apt_https_proxy: 'https://some-global-apt-proxy:8088/' + global_apt_http_proxy: 'http://some-global-apt-proxy:8088/' + ua_apt_http_proxy: 'http://10.0.10.10:3128' + ua_apt_https_proxy: 'https://10.0.10.10:3128' + enable: + - fips diff --git a/doc/module-docs/cc_ubuntu_pro/example5.yaml b/doc/module-docs/cc_ubuntu_pro/example5.yaml new file mode 100644 index 00000000000..915c0c23d30 --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/example5.yaml @@ -0,0 +1,4 @@ +#cloud-config +ubuntu_pro: + enable: [] + enable_beta: [] diff --git a/doc/module-docs/cc_ubuntu_pro/example6.yaml b/doc/module-docs/cc_ubuntu_pro/example6.yaml new file mode 100644 index 00000000000..dfb1f083215 --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/example6.yaml @@ -0,0 +1,4 @@ +#cloud-config +ubuntu_pro: + enable: [esm] + enable_beta: [realtime-kernel] diff --git a/doc/module-docs/cc_ubuntu_pro/example7.yaml b/doc/module-docs/cc_ubuntu_pro/example7.yaml new file mode 100644 index 00000000000..72dd926bbda --- /dev/null +++ b/doc/module-docs/cc_ubuntu_pro/example7.yaml @@ -0,0 +1,3 @@ +#cloud-config +ubuntu_pro: + features: {disable_auto_attach: true} diff --git a/doc/module-docs/cc_update_etc_hosts/data.yaml b/doc/module-docs/cc_update_etc_hosts/data.yaml new file mode 100644 index 00000000000..c723a8a0dfe --- /dev/null +++ b/doc/module-docs/cc_update_etc_hosts/data.yaml @@ -0,0 +1,55 @@ +cc_update_etc_hosts: + description: | + This module will update the contents of the local hosts database (hosts + file, usually ``/etc/hosts``) based on the hostname/FQDN specified in + config. Management of the hosts file is controlled using + ``manage_etc_hosts``. If this is set to ``false``, cloud-init will not + manage the hosts file at all. This is the default behavior. + + If set to ``true``, cloud-init will generate the hosts file using the + template located in ``/etc/cloud/templates/hosts.tmpl``. In the + ``/etc/cloud/templates/hosts.tmpl`` template, the strings ``$hostname`` + and ``$fqdn`` will be replaced with the hostname and FQDN respectively. + + If ``manage_etc_hosts`` is set to ``localhost``, then cloud-init will not + rewrite the hosts file entirely, but rather will ensure that an entry for + the FQDN with a distribution-dependent IP is present (i.e., + ``ping `` will ping ``127.0.0.1`` or ``127.0.1.1`` or other IP). + + .. note:: + If ``manage_etc_hosts`` is set to ``true``, the contents of the + ``hosts`` file will be updated every boot. To make any changes to the + ``hosts`` file persistent they must be made in + ``/etc/cloud/templates/hosts.tmpl``. + + .. note:: + For instructions on specifying hostname and FQDN, see documentation for + the ``cc_set_hostname`` module. + examples: + - comment: > + Example 1: Do not update or manage ``/etc/hosts`` at all. This is the + default behavior. Whatever is present at instance boot time will be + present after boot. User changes will not be overwritten. + file: cc_update_etc_hosts/example1.yaml + - comment: > + Example 2: Manage ``/etc/hosts`` with cloud-init. On every boot, + ``/etc/hosts`` will be re-written from + ``/etc/cloud/templates/hosts.tmpl``. + + The strings ``$hostname`` and ``$fqdn`` are replaced in the template + with the appropriate values, either from the config-config ``fqdn`` or + ``hostname`` if provided. When absent, the cloud metadata will be + checked for ``local-hostname`` which can be split into + ``.``. + + To make modifications persistent across a reboot, you must modify + ``/etc/cloud/templates/hosts.tmpl``. + file: cc_update_etc_hosts/example2.yaml + - comment: > + Example 3: Update ``/etc/hosts`` every boot, providing a "localhost" + ``127.0.1.1`` entry with the latest hostname and FQDN as provided by + either IMDS or cloud-config. All other entries will be left alone. + ``ping hostname`` will ping ``127.0.1.1``. + file: cc_update_etc_hosts/example3.yaml + name: Update Etc Hosts + title: Update the hosts file (usually ``/etc/hosts``) diff --git a/doc/module-docs/cc_update_etc_hosts/example1.yaml b/doc/module-docs/cc_update_etc_hosts/example1.yaml new file mode 100644 index 00000000000..d8f31267f35 --- /dev/null +++ b/doc/module-docs/cc_update_etc_hosts/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +manage_etc_hosts: false diff --git a/doc/module-docs/cc_update_etc_hosts/example2.yaml b/doc/module-docs/cc_update_etc_hosts/example2.yaml new file mode 100644 index 00000000000..9bc3b33e3f2 --- /dev/null +++ b/doc/module-docs/cc_update_etc_hosts/example2.yaml @@ -0,0 +1,2 @@ +#cloud-config +manage_etc_hosts: true diff --git a/doc/module-docs/cc_update_etc_hosts/example3.yaml b/doc/module-docs/cc_update_etc_hosts/example3.yaml new file mode 100644 index 00000000000..e1f85816cc0 --- /dev/null +++ b/doc/module-docs/cc_update_etc_hosts/example3.yaml @@ -0,0 +1,2 @@ +#cloud-config +manage_etc_hosts: localhost diff --git a/doc/module-docs/cc_update_hostname/data.yaml b/doc/module-docs/cc_update_hostname/data.yaml new file mode 100644 index 00000000000..cfe5dc56993 --- /dev/null +++ b/doc/module-docs/cc_update_hostname/data.yaml @@ -0,0 +1,38 @@ +cc_update_hostname: + description: | + This module will update the system hostname and FQDN. If + ``preserve_hostname`` is set to ``true``, then the hostname will not be + altered. + + .. note:: + For instructions on specifying hostname and FQDN, see documentation for + the ``cc_set_hostname`` module. + examples: + - comment: > + Example 1: By default, when ``preserve_hostname`` is not specified, + cloud-init updates ``/etc/hostname`` per-boot based on the cloud provided + ``local-hostname`` setting. If you manually change ``/etc/hostname`` + after boot cloud-init will no longer modify it. + + This default cloud-init behavior is equivalent to this cloud-config: + file: cc_update_hostname/example1.yaml + - comment: | + Example 2: Prevent cloud-init from updating the system hostname. + file: cc_update_hostname/example2.yaml + - comment: | + Example 3: Prevent cloud-init from updating ``/etc/hostname``. + file: cc_update_hostname/example3.yaml + - comment: | + Example 4: Set hostname to ``external.fqdn.me`` instead of ``myhost``. + file: cc_update_hostname/example4.yaml + - comment: > + Example 5: Set hostname to ``external`` instead of ``external.fqdn.me`` when + cloud metadata provides the ``local-hostname``: ``external.fqdn.me``. + file: cc_update_hostname/example5.yaml + - comment: > + Example 6: On a machine without an ``/etc/hostname`` file, don''t create + it. In most clouds, this will result in a DHCP-configured hostname + provided by the cloud. + file: cc_update_hostname/example6.yaml + name: Update Hostname + title: Update hostname and FQDN diff --git a/doc/module-docs/cc_update_hostname/example1.yaml b/doc/module-docs/cc_update_hostname/example1.yaml new file mode 100644 index 00000000000..2ca4689d1d8 --- /dev/null +++ b/doc/module-docs/cc_update_hostname/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +preserve_hostname: false diff --git a/doc/module-docs/cc_update_hostname/example2.yaml b/doc/module-docs/cc_update_hostname/example2.yaml new file mode 100644 index 00000000000..6edec76adf6 --- /dev/null +++ b/doc/module-docs/cc_update_hostname/example2.yaml @@ -0,0 +1,2 @@ +#cloud-config +preserve_hostname: true diff --git a/doc/module-docs/cc_update_hostname/example3.yaml b/doc/module-docs/cc_update_hostname/example3.yaml new file mode 100644 index 00000000000..6edec76adf6 --- /dev/null +++ b/doc/module-docs/cc_update_hostname/example3.yaml @@ -0,0 +1,2 @@ +#cloud-config +preserve_hostname: true diff --git a/doc/module-docs/cc_update_hostname/example4.yaml b/doc/module-docs/cc_update_hostname/example4.yaml new file mode 100644 index 00000000000..8f7ecfbb420 --- /dev/null +++ b/doc/module-docs/cc_update_hostname/example4.yaml @@ -0,0 +1,5 @@ +#cloud-config +fqdn: external.fqdn.me +hostname: myhost +prefer_fqdn_over_hostname: true +create_hostname_file: true diff --git a/doc/module-docs/cc_update_hostname/example5.yaml b/doc/module-docs/cc_update_hostname/example5.yaml new file mode 100644 index 00000000000..51717bd5e61 --- /dev/null +++ b/doc/module-docs/cc_update_hostname/example5.yaml @@ -0,0 +1,2 @@ +#cloud-config +prefer_fqdn_over_hostname: false diff --git a/doc/module-docs/cc_update_hostname/example6.yaml b/doc/module-docs/cc_update_hostname/example6.yaml new file mode 100644 index 00000000000..785e6167d89 --- /dev/null +++ b/doc/module-docs/cc_update_hostname/example6.yaml @@ -0,0 +1,2 @@ +#cloud-config +create_hostname_file: false diff --git a/doc/module-docs/cc_users_groups/data.yaml b/doc/module-docs/cc_users_groups/data.yaml new file mode 100644 index 00000000000..03bb32594d7 --- /dev/null +++ b/doc/module-docs/cc_users_groups/data.yaml @@ -0,0 +1,107 @@ +cc_users_groups: + description: | + This module configures users and groups. For more detailed information + on user options, see the :ref:`Including users and groups ` + config example. + + Groups to add to the system can be specified under the ``groups`` key as a + string of comma-separated groups to create, or a list. Each item in the + list should either contain a string of a single group to create, or a + dictionary with the group name as the key and string of a single user as a + member of that group or a list of users who should be members of the group. + + .. note:: + Groups are added before users, so any users in a group list must already + exist on the system. + + Users to add can be specified as a string or list under the ``users`` key. + Each entry in the list should either be a string or a dictionary. If a + string is specified, that string can be comma-separated usernames to + create, or the reserved string ``default`` which represents the primary + admin user used to access the system. The ``default`` user varies per + distribution and is generally configured in ``/etc/cloud/cloud.cfg`` by the + ``default_user key``. + + Each ``users`` dictionary item must contain either a ``name`` or + ``snapuser`` key, otherwise it will be ignored. Omission of ``default`` as + the first item in the ``users`` list skips creation the default user. If + no ``users`` key is provided, the default behavior is to create the + default user via this config: + + .. code-block:: yaml + + users: + - default + + .. note:: + Specifying a hash of a user's password with ``passwd`` is a security + risk if the cloud-config can be intercepted. SSH authentication is + preferred. + + .. note:: + If specifying a ``doas`` rule for a user, ensure that the syntax for + the rule is valid, as the only checking performed by cloud-init is to + ensure that the user referenced in the rule is the correct user. + + .. note:: + If specifying a ``sudo`` rule for a user, ensure that the syntax for + the rule is valid, as it is not checked by cloud-init. + + .. note:: + Most of these configuration options will not be honored if the user + already exists. The following options are the exceptions, and are + applied to already-existing users; ``plain_text_passwd``, ``doas``, + ``hashed_passwd``, ``lock_passwd``, ``sudo``, ``ssh_authorized_keys``, + ``ssh_redirect_user``. + + The ``user`` key can be used to override the ``default_user`` configuration + defined in ``/etc/cloud/cloud.cfg``. The ``user`` value should be a + dictionary which supports the same config keys as the ``users`` dictionary + items. + examples: + - comment: > + Example 1: Add the ``default_user`` from ``/etc/cloud/cloud.cfg``. This + is also the default behavior of cloud-init when no ``users`` key is + provided. + file: cc_users_groups/example1.yaml + - comment: > + Example 2: Add the ``admingroup`` with members ``root`` and ``sys``, and + an empty group ``cloud-users``. + file: cc_users_groups/example2.yaml + - comment: > + Example 3: Skip creation of the ``default`` user and only create + ``newsuper``. Password-based login is rejected, but the GitHub user + ``TheRealFalcon`` and the Launchpad user ``falcojr`` can SSH as + ``newsuper``. The default shell for ``newsuper`` is bash instead of + system default. + file: cc_users_groups/example3.yaml + - comment: > + Example 4: Skip creation of the ``default`` user and only create + ``newsuper``. Password-based login is rejected, but the GitHub user + ``TheRealFalcon`` and the Launchpad user ``falcojr`` can SSH as + ``newsuper``. ``doas``/``opendoas`` is configured to permit this user to + run commands as other users (without being prompted for a password) + except not as root. + file: cc_users_groups/example4.yaml + - comment: > + Example 5: On a system with SELinux enabled, add ``youruser`` and set the + SELinux user to ``staff_u``. When omitted on SELinux, the system will + select the configured default SELinux user. + file: cc_users_groups/example5.yaml + - comment: > + Example 6: To redirect a legacy username to the ``default`` user for a + distribution, ``ssh_redirect_user`` will accept an SSH connection and + emit a message telling the client to SSH as the ``default`` user. + SSH clients will get the message; + file: cc_users_groups/example6.yaml + - comment: > + Example 7: Override any ``default_user`` config in + ``/etc/cloud/cloud.cfg`` with supplemental config options. This config + will make the default user ``mynewdefault`` and change the user to not + have ``sudo`` rights. + file: cc_users_groups/example7.yaml + - comment: > + Example 8: Avoid creating any ``default_user``. + file: cc_users_groups/example8.yaml + name: Users and Groups + title: Configure users and groups diff --git a/doc/module-docs/cc_users_groups/example1.yaml b/doc/module-docs/cc_users_groups/example1.yaml new file mode 100644 index 00000000000..b129d9e9c64 --- /dev/null +++ b/doc/module-docs/cc_users_groups/example1.yaml @@ -0,0 +1,2 @@ +#cloud-config +users: [default] diff --git a/doc/module-docs/cc_users_groups/example2.yaml b/doc/module-docs/cc_users_groups/example2.yaml new file mode 100644 index 00000000000..595f4e4546a --- /dev/null +++ b/doc/module-docs/cc_users_groups/example2.yaml @@ -0,0 +1,4 @@ +#cloud-config +groups: +- admingroup: [root, sys] +- cloud-users diff --git a/doc/module-docs/cc_users_groups/example3.yaml b/doc/module-docs/cc_users_groups/example3.yaml new file mode 100644 index 00000000000..ed79daa9d25 --- /dev/null +++ b/doc/module-docs/cc_users_groups/example3.yaml @@ -0,0 +1,9 @@ +#cloud-config +users: +- gecos: Big Stuff + groups: users, admin + lock_passwd: true + name: newsuper + shell: /bin/bash + ssh_import_id: ['lp:falcojr', 'gh:TheRealFalcon'] + sudo: ALL=(ALL) NOPASSWD:ALL diff --git a/doc/module-docs/cc_users_groups/example4.yaml b/doc/module-docs/cc_users_groups/example4.yaml new file mode 100644 index 00000000000..4c764b9c6fa --- /dev/null +++ b/doc/module-docs/cc_users_groups/example4.yaml @@ -0,0 +1,8 @@ +#cloud-config +users: +- doas: [permit nopass newsuper, deny newsuper as root] + gecos: Big Stuff + groups: users, admin + lock_passwd: true + name: newsuper + ssh_import_id: ['lp:falcojr', 'gh:TheRealFalcon'] diff --git a/doc/module-docs/cc_users_groups/example5.yaml b/doc/module-docs/cc_users_groups/example5.yaml new file mode 100644 index 00000000000..2de78aa97f7 --- /dev/null +++ b/doc/module-docs/cc_users_groups/example5.yaml @@ -0,0 +1,4 @@ +#cloud-config +users: +- default +- {name: youruser, selinux_user: staff_u} diff --git a/doc/module-docs/cc_users_groups/example6.yaml b/doc/module-docs/cc_users_groups/example6.yaml new file mode 100644 index 00000000000..05a7013dc88 --- /dev/null +++ b/doc/module-docs/cc_users_groups/example6.yaml @@ -0,0 +1,4 @@ +#cloud-config +users: +- default +- {name: nosshlogins, ssh_redirect_user: true} diff --git a/doc/module-docs/cc_users_groups/example7.yaml b/doc/module-docs/cc_users_groups/example7.yaml new file mode 100644 index 00000000000..98525e7ebd0 --- /dev/null +++ b/doc/module-docs/cc_users_groups/example7.yaml @@ -0,0 +1,3 @@ +#cloud-config +ssh_import_id: [chad.smith] +user: {name: mynewdefault, sudo: null} diff --git a/doc/module-docs/cc_users_groups/example8.yaml b/doc/module-docs/cc_users_groups/example8.yaml new file mode 100644 index 00000000000..c6e6d4c39c9 --- /dev/null +++ b/doc/module-docs/cc_users_groups/example8.yaml @@ -0,0 +1,2 @@ +#cloud-config +users: [] diff --git a/doc/module-docs/cc_wireguard/data.yaml b/doc/module-docs/cc_wireguard/data.yaml new file mode 100644 index 00000000000..d49af7ca813 --- /dev/null +++ b/doc/module-docs/cc_wireguard/data.yaml @@ -0,0 +1,43 @@ +cc_wireguard: + description: | + The WireGuard module provides a dynamic interface for configuring + WireGuard (as a peer or server) in a straightforward way. + + This module takes care of; + + - writing interface configuration files + - enabling and starting interfaces + - installing wireguard-tools package + - loading WireGuard kernel module + - executing readiness probes + + **What is a readiness probe?** + + The idea behind readiness probes is to ensure WireGuard connectivity before + continuing the cloud-init process. This could be useful if you need access + to specific services like an internal APT Repository Server (e.g., + Landscape) to install/update packages. + + **Example** + + An edge device can't access the internet but uses cloud-init modules which + will install packages (e.g. ``landscape``, ``packages``, + ``ubuntu_advantage``). Those modules will fail due to missing internet + connection. The ``wireguard`` module fixes that problem as it waits until + all readiness probes (which can be arbitrary commands, e.g. checking if a + proxy server is reachable over WireGuard network) are finished, before + continuing the cloud-init ``config`` stage. + + .. note:: + In order to use DNS with WireGuard you have to install the + ``resolvconf`` package or symlink it to systemd's ``resolvectl``, + otherwise ``wg-quick`` commands will throw an error message that + executable ``resolvconf`` is missing, which leads the ``wireguard`` + module to fail. + examples: + - comment: > + Configure one or more WireGuard interfaces and provide optional readiness + probes. + file: cc_wireguard/example1.yaml + name: Wireguard + title: Module to configure WireGuard tunnel diff --git a/doc/module-docs/cc_wireguard/example1.yaml b/doc/module-docs/cc_wireguard/example1.yaml new file mode 100644 index 00000000000..a7f0031375b --- /dev/null +++ b/doc/module-docs/cc_wireguard/example1.yaml @@ -0,0 +1,27 @@ +#cloud-config +wireguard: + interfaces: + - name: wg0 + config_path: /etc/wireguard/wg0.conf + content: | + [Interface] + PrivateKey = + Address =
+ [Peer] + PublicKey = + Endpoint = : + AllowedIPs = , , ... + - name: wg1 + config_path: /etc/wireguard/wg1.conf + content: | + [Interface] + PrivateKey = + Address =
+ [Peer] + PublicKey = + Endpoint = : + AllowedIPs = + readinessprobe: + - 'systemctl restart service' + - 'curl https://webhook.endpoint/example' + - 'nc -zv some-service-fqdn 443' diff --git a/doc/module-docs/cc_write_files/data.yaml b/doc/module-docs/cc_write_files/data.yaml new file mode 100644 index 00000000000..c59b8e2ea75 --- /dev/null +++ b/doc/module-docs/cc_write_files/data.yaml @@ -0,0 +1,40 @@ +cc_write_files: + description: | + Write out arbitrary content to files, optionally setting permissions. + Parent folders in the path are created if absent. Content can be specified + in plain text or binary. Data encoded with either base64 or binary gzip + data can be specified and will be decoded before being written. For empty + file creation, content can be omitted. + + .. note:: + If multi-line data is provided, care should be taken to ensure it + follows YAML formatting standards. To specify binary data, use the YAML + option ``!!binary``. + + .. note:: + Do not write files under ``/tmp`` during boot because of a race with + ``systemd-tmpfiles-clean`` that can cause temporary files to be cleaned + during the early boot process. Use ``/run/somedir`` instead to avoid + a race (LP: #1707222). + + .. warning:: + Existing files will be overridden. + examples: + - comment: | + Example 1: Write out base64-encoded content to ``/etc/sysconfig/selinux``. + file: cc_write_files/example1.yaml + - comment: | + Example 2: Appending content to an existing file. + file: cc_write_files/example2.yaml + - comment: | + Example 3: Provide gzipped binary content + file: cc_write_files/example3.yaml + - comment: | + Example 4: Create an empty file on the system + file: cc_write_files/example4.yaml + - comment: > + Example 5: Defer writing the file until after the package (Nginx) is + installed and its user is created. + file: cc_write_files/example5.yaml + name: Write Files + title: Write arbitrary files diff --git a/doc/module-docs/cc_write_files/example1.yaml b/doc/module-docs/cc_write_files/example1.yaml new file mode 100644 index 00000000000..ab2eb6f275e --- /dev/null +++ b/doc/module-docs/cc_write_files/example1.yaml @@ -0,0 +1,7 @@ +#cloud-config +write_files: +- encoding: b64 + content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4... + owner: root:root + path: /etc/sysconfig/selinux + permissions: '0644' diff --git a/doc/module-docs/cc_write_files/example2.yaml b/doc/module-docs/cc_write_files/example2.yaml new file mode 100644 index 00000000000..e7d7eda158b --- /dev/null +++ b/doc/module-docs/cc_write_files/example2.yaml @@ -0,0 +1,6 @@ +#cloud-config +write_files: +- content: | + 15 * * * * root ship_logs + path: /etc/crontab + append: true diff --git a/doc/module-docs/cc_write_files/example3.yaml b/doc/module-docs/cc_write_files/example3.yaml new file mode 100644 index 00000000000..749b914369e --- /dev/null +++ b/doc/module-docs/cc_write_files/example3.yaml @@ -0,0 +1,7 @@ +#cloud-config +write_files: +- encoding: gzip + content: !!binary | + H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= + path: /usr/bin/hello + permissions: '0755' diff --git a/doc/module-docs/cc_write_files/example4.yaml b/doc/module-docs/cc_write_files/example4.yaml new file mode 100644 index 00000000000..7c299bd9c2b --- /dev/null +++ b/doc/module-docs/cc_write_files/example4.yaml @@ -0,0 +1,3 @@ +#cloud-config +write_files: +- path: /root/CLOUD_INIT_WAS_HERE diff --git a/doc/module-docs/cc_write_files/example5.yaml b/doc/module-docs/cc_write_files/example5.yaml new file mode 100644 index 00000000000..a99bdf43449 --- /dev/null +++ b/doc/module-docs/cc_write_files/example5.yaml @@ -0,0 +1,15 @@ +#cloud-config +write_files: +- path: /etc/nginx/conf.d/example.com.conf + content: | + server { + server_name example.com; + listen 80; + root /var/www; + location / { + try_files $uri $uri/ $uri.html =404; + } + } + owner: 'nginx:nginx' + permissions: '0640' + defer: true diff --git a/doc/module-docs/cc_yum_add_repo/data.yaml b/doc/module-docs/cc_yum_add_repo/data.yaml new file mode 100644 index 00000000000..7875654fbf0 --- /dev/null +++ b/doc/module-docs/cc_yum_add_repo/data.yaml @@ -0,0 +1,27 @@ +cc_yum_add_repo: + description: | + Add yum repository configuration to ``/etc/yum.repos.d``. Configuration + files are named based on the opaque dictionary key under the ``yum_repos`` + they are specified with. If a config file already exists with the same + name as a config entry, the config entry will be skipped. + examples: + - comment: | + Example 1: + file: cc_yum_add_repo/example1.yaml + - comment: > + Example 2: Enable cloud-init upstream's daily testing repo for EPEL 8 to + install the latest cloud-init from tip of ``main`` for testing. + file: cc_yum_add_repo/example2.yaml + - comment: > + Example 3: Add the file ``/etc/yum.repos.d/epel_testing.repo`` which can + then subsequently be used by yum for later operations. + file: cc_yum_add_repo/example3.yaml + - comment: > + Example 4: Any yum repo configuration can be passed directly into the + repository file created. See ``man yum.conf`` for supported config keys. + + Write ``/etc/yum.conf.d/my-package-stream.repo`` with ``gpgkey`` checks + on the repo data of the repository enabled. + file: cc_yum_add_repo/example4.yaml + name: Yum Add Repo + title: Add yum repository configuration to the system diff --git a/doc/module-docs/cc_yum_add_repo/example1.yaml b/doc/module-docs/cc_yum_add_repo/example1.yaml new file mode 100644 index 00000000000..ed9642bb91a --- /dev/null +++ b/doc/module-docs/cc_yum_add_repo/example1.yaml @@ -0,0 +1,5 @@ +#cloud-config +yum_repos: + my_repo: + baseurl: http://blah.org/pub/epel/testing/5/$basearch/ +yum_repo_dir: /store/custom/yum.repos.d diff --git a/doc/module-docs/cc_yum_add_repo/example2.yaml b/doc/module-docs/cc_yum_add_repo/example2.yaml new file mode 100644 index 00000000000..f5ae4735c3b --- /dev/null +++ b/doc/module-docs/cc_yum_add_repo/example2.yaml @@ -0,0 +1,10 @@ +#cloud-config +yum_repos: + cloud-init-daily: + name: Copr repo for cloud-init-dev owned by @cloud-init + baseurl: https://download.copr.fedorainfracloud.org/results/@cloud-init/cloud-init-dev/epel-8-$basearch/ + type: rpm-md + skip_if_unavailable: true + gpgcheck: true + gpgkey: https://download.copr.fedorainfracloud.org/results/@cloud-init/cloud-init-dev/pubkey.gpg + enabled_metadata: 1 diff --git a/doc/module-docs/cc_yum_add_repo/example3.yaml b/doc/module-docs/cc_yum_add_repo/example3.yaml new file mode 100644 index 00000000000..717ca8c4e45 --- /dev/null +++ b/doc/module-docs/cc_yum_add_repo/example3.yaml @@ -0,0 +1,10 @@ +#cloud-config +yum_repos: +# The name of the repository + epel-testing: + baseurl: https://download.copr.fedorainfracloud.org/results/@cloud-init/cloud-init-dev/pubkey.gpg + enabled: false + failovermethod: priority + gpgcheck: true + gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL + name: Extra Packages for Enterprise Linux 5 - Testing diff --git a/doc/module-docs/cc_yum_add_repo/example4.yaml b/doc/module-docs/cc_yum_add_repo/example4.yaml new file mode 100644 index 00000000000..01188159dc1 --- /dev/null +++ b/doc/module-docs/cc_yum_add_repo/example4.yaml @@ -0,0 +1,8 @@ +#cloud-config +yum_repos: + my package stream: + baseurl: http://blah.org/pub/epel/testing/5/$basearch/ + mirrorlist: http://some-url-to-list-of-baseurls + repo_gpgcheck: 1 + enable_gpgcheck: true + gpgkey: https://url.to.ascii-armored-gpg-key diff --git a/doc/module-docs/cc_zypper_add_repo/data.yaml b/doc/module-docs/cc_zypper_add_repo/data.yaml new file mode 100644 index 00000000000..6ad5ad59c02 --- /dev/null +++ b/doc/module-docs/cc_zypper_add_repo/data.yaml @@ -0,0 +1,23 @@ +cc_zypper_add_repo: + description: | + Zypper behavior can be configured using the ``config`` key, which will + modify ``/etc/zypp/zypp.conf``. The configuration writer will only append + the provided configuration options to the configuration file. Any duplicate + options will be resolved by the way the ``zypp.conf`` INI file is parsed. + + .. note:: + Setting ``configdir`` is not supported and will be skipped. + + The ``repos`` key may be used to add repositories to the system. Beyond the + required ``id`` and ``baseurl`` attributions, no validation is performed on + the ``repos`` entries. + + It is assumed the user is familiar with the Zypper repository file format. + This configuration is also applicable for systems with + transactional-updates. + examples: + - comment: | + Example 1: + file: cc_zypper_add_repo/example1.yaml + name: Zypper Add Repo + title: Configure Zypper behavior and add Zypper repositories diff --git a/doc/module-docs/cc_zypper_add_repo/example1.yaml b/doc/module-docs/cc_zypper_add_repo/example1.yaml new file mode 100644 index 00000000000..720169db7fb --- /dev/null +++ b/doc/module-docs/cc_zypper_add_repo/example1.yaml @@ -0,0 +1,8 @@ +#cloud-config +zypper: + config: {download.use_deltarpm: true, reposdir: /etc/zypp/repos.dir, servicesdir: /etc/zypp/services.d} + repos: + - {autorefresh: 1, baseurl: 'http://dl.opensuse.org/dist/leap/v/repo/oss/', enabled: 1, + id: opensuse-oss, name: os-oss} + - {baseurl: 'http://dl.opensuse.org/dist/leap/v/update', id: opensuse-oss-update, + name: os-oss-up} diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index c78c9b981b9..1ca6a85a208 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -1,4 +1,5 @@ import datetime +import glob import os import sys @@ -274,22 +275,27 @@ def render_nested_properties(prop_cfg, defs, prefix): def render_module_schemas(): - from cloudinit.importer import find_module, import_module + from cloudinit.importer import import_module mod_docs = {} schema = get_schema() defs = schema.get("$defs", {}) - for key in schema["$defs"]: - if key[:3] != "cc_": - continue - locs, _ = find_module(key, ["", "cloudinit.config"], ["meta"]) - mod = import_module(locs[0]) - mod_docs[key] = { - "schema_doc": render_nested_properties( - schema["$defs"][key], defs, "" - ), + + for mod_path in glob.glob("../../cloudinit/config/cc_*py"): + mod_name = os.path.basename(mod_path).replace(".py", "") + mod = import_module(f"cloudinit.config.{mod_name}") + cc_key = mod.meta["id"] + mod_docs[cc_key] = { "meta": mod.meta, } + if cc_key in defs: + mod_docs[cc_key]["schema_doc"] = render_nested_properties( + defs[cc_key], defs, "" + ) + else: + mod_docs[cc_key][ + "schema_doc" + ] = "No schema definitions for this module" return mod_docs diff --git a/doc/rtd/development/docs_layout.rst b/doc/rtd/development/docs_layout.rst index 0f981a2afeb..1e4e7c76743 100644 --- a/doc/rtd/development/docs_layout.rst +++ b/doc/rtd/development/docs_layout.rst @@ -11,6 +11,7 @@ directory: /doc/ - examples/ - man/ + - module-docs/ - rtd/ - tutorial/ - howto/ @@ -29,17 +30,50 @@ directory: - sources/ -``examples/`` -============= +examples/ +========= -``man/`` -======== +man/ +==== This subdirectory contains the Linux man pages for the binaries provided by cloud-init. +module-docs/ +============ + +The documentation for modules is generated automatically using YAML files and +templates. Each module has its own sub-directory, containing: + +- ``data.yaml`` file: + Contains the text and descriptions rendered on the + :ref:`modules documentation ` page. +- ``example*.yaml`` files: + These examples stand alone as valid cloud-config. They always start with + ``#cloud-config``, and ideally, should also have some accompanying discussion + or context in the ``comment`` field in the ``data.yaml`` file to explain + what's happening. + +Edit existing module docs +------------------------- + +In the ``data.yaml`` file, the fields support reStructuredText markup in the +``description`` and ``comment`` fields. With the pipe character (``|``) +preceding these fields, the text will be preserved so that using rST directives +(such as notes or code blocks) will render correctly in the documentation. If +you don't need to use directives, you can use the greater-than character +(``>``), which will fold broken lines together into paragraphs (while +respecting empty lines). + +Create new module docs +---------------------- + +Creating documentation for a **new** module involves a little more work, and +the process for that is outlined in the :ref:`contributing modules ` +page. + ``rtd/`` ======== diff --git a/doc/rtd/development/logging.rst b/doc/rtd/development/logging.rst index c294ce4931b..fed9f35ba2c 100644 --- a/doc/rtd/development/logging.rst +++ b/doc/rtd/development/logging.rst @@ -271,7 +271,7 @@ With defaults used:: For more information on ``rsyslog`` configuration, see -:ref:`our module reference page`. +:ref:`our module reference page `. .. LINKS: .. _python logging config: https://docs.python.org/3/library/logging.config.html#configuration-file-format diff --git a/doc/rtd/development/module_creation.rst b/doc/rtd/development/module_creation.rst index 9c873ccaa91..32240ab3e91 100644 --- a/doc/rtd/development/module_creation.rst +++ b/doc/rtd/development/module_creation.rst @@ -7,8 +7,60 @@ Much of ``cloud-init``'s functionality is provided by :ref:`modules`. All modules follow a similar layout in order to provide consistent execution and documentation. Use the example provided here to create a new module. -Example -======= +.. _module_creation-Guidelines: + +Your Python module +================== + +Modules are located in the ``cloudinit/config/`` directory, where the naming +convention for modules is to use ``cc_`` (with underscores as the +separators). + +The handle function +------------------- + +Your module must include a ``handle`` function. The arguments are: + +- ``name``: The module name specified in the configuration. +- ``cfg``: A configuration object that is the result of the merging of + cloud-config configuration with any datasource-provided configuration. +- ``cloud``: A cloud object that can be used to access various datasource + and paths for the given distro and data provided by the various datasource + instance types. +- ``args``: An argument list. This is usually empty and is only populated + if the module is called independently from the command line or if the + module definition in :file:`/etc/cloud/cloud.cfg[.d]` has been modified + to pass arguments to this module. + +Schema definition +----------------- + +If your module introduces any new cloud-config keys, you must provide a schema +definition in `cloud-init-schema.json`_. + +- The ``meta`` variable must exist and be of type `MetaSchema`_. + + - ``id``: The module ID. In most cases this will be the filename without + the ``.py`` extension. + - ``distros``: Defines the list of supported distros. It can contain + any of the values (not keys) defined in the `OSFAMILIES`_ map or + ``[ALL_DISTROS]`` if there is no distro restriction. + - ``frequency``: Defines how often module runs. It must be one of: + + - ``PER_ALWAYS``: Runs on every boot. + - ``ONCE``: Runs only on first boot. + - ``PER_INSTANCE``: Runs once per instance. When exactly this happens + is dependent on the datasource, but may be triggered any time there + would be a significant change to the instance metadata. An example + could be an instance being moved to a different subnet. + + - ``activate_by_schema_keys``: Optional list of cloud-config keys that will + activate this module. When this list not empty, the config module will be + skipped unless one of the ``activate_by_schema_keys`` are present in merged + cloud-config instance-data. + +Example module.py file +====================== .. code-block:: python @@ -18,94 +70,87 @@ Example import logging from cloudinit.cloud import Cloud from cloudinit.config import Config - from cloudinit.config.schema import MetaSchema, get_meta_doc + from cloudinit.config.schema import MetaSchema from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE - MODULE_DESCRIPTION = """\ - Description that will be used in module documentation. - - This will likely take multiple lines. - """ - LOG = logging.getLogger(__name__) meta: MetaSchema = { "id": "cc_example", - "name": "Example Module", - "title": "Shows how to create a module", - "description": MODULE_DESCRIPTION, "distros": [ALL_DISTROS], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["example_key, example_other_key"], - "examples": [ - "example_key: example_value", - "example_other_key: ['value', 2]", - ], - } - - __doc__ = get_meta_doc(meta) - + } # type: ignore def handle( name: str, cfg: Config, cloud: Cloud, args: list ) -> None: LOG.debug(f"Hi from module {name}") -.. _module_creation-Guidelines: +Module documentation +==================== -Guidelines -========== - -* Create a new module in the :file:`cloudinit/config` directory with a ``cc_`` - prefix. -* Your module must include a ``handle`` function. The arguments are: - - * ``name``: The module name specified in the configuration. - * ``cfg``: A configuration object that is the result of the merging of - cloud-config configuration with any datasource-provided configuration. - * ``cloud``: A cloud object that can be used to access various datasource - and paths for the given distro and data provided by the various datasource - instance types. - * ``args``: An argument list. This is usually empty and is only populated - if the module is called independently from the command line or if the - module definition in :file:`/etc/cloud/cloud.cfg[.d]` has been modified - to pass arguments to this module. - -* If your module introduces any new cloud-config keys, you must provide a - schema definition in `cloud-init-schema.json`_. -* The ``meta`` variable must exist and be of type `MetaSchema`_. - - * ``id``: The module ID. In most cases this will be the filename without - the ``.py`` extension. - * ``distros``: Defines the list of supported distros. It can contain - any of the values (not keys) defined in the `OSFAMILIES`_ map or - ``[ALL_DISTROS]`` if there is no distro restriction. - * ``frequency``: Defines how often module runs. It must be one of: +Every module has a folder in the ``doc/module-docs/`` directory, containing +a ``data.yaml`` file, and one or more ``example*.yaml`` files. - * ``PER_ALWAYS``: Runs on every boot. - * ``ONCE``: Runs only on first boot. - * ``PER_INSTANCE``: Runs once per instance. When exactly this happens - is dependent on the datasource, but may triggered any time there - would be a significant change to the instance metadata. An example - could be an instance being moved to a different subnet. +- The ``data.yaml`` file contains most of the documentation fields. At a + minimum, your module should be provided with this file. Examples are not + strictly required, but are helpful to readers of the documentation so it is + preferred for at least one example to be included. +- The ``example*.yaml`` files are illustrative demonstrations of using the + module, but should be self-contained and in correctly-formatted YAML. These + will be automatically tested against the defined schema. - * ``activate_by_schema_keys``: Optional list of cloud-config keys that will - activate this module. When this list not empty, the config module will be - skipped unless one of the ``activate_by_schema_keys`` are present in merged - cloud-config instance-data. - * ``examples``: Lists examples of any cloud-config keys this module reacts - to. These examples will be rendered in the module reference documentation - and will automatically be tested against the defined schema - during testing. +Example data.yaml file +---------------------- + +.. code-block:: yaml + + cc_module_name: + description: > + This module provides some functionality, which you can describe here. + + For straightforward text examples, use a greater-than (``>``) symbol + next to ``description: `` to ensure proper rendering in the + documentation. Empty lines will be respected, but line-breaks are + folded together to create proper paragraphs. + + If you need to use call-outs or code blocks, use a pipe (``|``) symbol + instead of ``>`` so that reStructuredText formatting (e.g. for + directives, which take varying levels of indentation) is respected. + examples: + - comment: | + Example 1: (optional) description of the expected behavior of the example + file: cc_module_name/example1.yaml + - comment: | + Example 2: (optional) description of a second example. + file: cc_module_name/example2.yaml + name: Module Name + title: Very brief (1 sentence) tag line describing what your module does + +Rendering the module docs +------------------------- -* ``__doc__ = get_meta_doc(meta)`` is necessary to provide proper module - documentation. +The module documentation is auto-generated via the +:file:`doc/rtd/reference/modules.rst` file. + +For your module documentation to be shown in the cloud-init docs, you will +need to add an entry to this page. Modules are listed in alphabetical order. +The entry should be in the following reStructuredText format: + +.. code-block:: text + + .. datatemplate:yaml:: ../../module-docs/cc_ansible/data.yaml + :template: modules.tmpl + +The template pulls information from both your ``module.py`` file, and from its +corresponding entry in the the ``module-docs`` directory. Module execution ================ -In order for a module to be run, it must be defined in a module run section in +For a module to be run, it must be defined in a module run section in :file:`/etc/cloud/cloud.cfg` or :file:`/etc/cloud/cloud.cfg.d` on the launched instance. The three module sections are `cloud_init_modules`_, `cloud_config_modules`_, and `cloud_final_modules`_, @@ -119,7 +164,6 @@ dependencies or is not necessary for a later boot stage, it should be placed in the ``cloud_final_modules`` section before the ``final-message`` module. - .. _MetaSchema: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/config/schema.py#L58 .. _OSFAMILIES: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/distros/__init__.py#L35 .. _settings.py: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/settings.py#L66 diff --git a/doc/rtd/reference/base_config_reference.rst b/doc/rtd/reference/base_config_reference.rst index 5d291aac02b..9686d456d11 100644 --- a/doc/rtd/reference/base_config_reference.rst +++ b/doc/rtd/reference/base_config_reference.rst @@ -116,11 +116,11 @@ Both keys will be processed independently. - ``distro``: Name of distro being used. - ``default_user``: Defines the default user for the system using the same - user configuration as :ref:`Users and Groups`. Note that - this CAN be overridden if a ``users`` configuration + user configuration as :ref:`Users and Groups`. Note + that this CAN be overridden if a ``users`` configuration is specified without a ``- default`` entry. - ``ntp_client``: The default NTP client for the distro. Takes the same - form as ``ntp_client`` defined in :ref:`NTP`. + form as ``ntp_client`` defined in :ref:`NTP`. - ``package_mirrors``: Defines the package mirror info for apt. - ``ssh_svcname``: The SSH service name. For most distros this will be either ``ssh`` or ``sshd``. diff --git a/doc/rtd/reference/datasources/azure.rst b/doc/rtd/reference/datasources/azure.rst index 1907e4b5dcc..8cab989f390 100644 --- a/doc/rtd/reference/datasources/azure.rst +++ b/doc/rtd/reference/datasources/azure.rst @@ -52,7 +52,7 @@ The settings that may be configured are: * :command:`disk_aliases` A dictionary defining which device paths should be interpreted as ephemeral - images. See :ref:`cc_disk_setup ` module for more info. + images. See :ref:`cc_disk_setup ` module for more info. Configuration for the datasource can also be read from a ``dscfg`` entry in the ``LinuxProvisioningConfigurationSet``. Content in ``dscfg`` node is diff --git a/doc/rtd/reference/datasources/nocloud.rst b/doc/rtd/reference/datasources/nocloud.rst index 618473a925a..3033869f682 100644 --- a/doc/rtd/reference/datasources/nocloud.rst +++ b/doc/rtd/reference/datasources/nocloud.rst @@ -104,7 +104,7 @@ A valid ``seedfrom`` value consists of a URI which must contain a trailing Some optional keys may be used, but their use is discouraged and may be removed in the future. -* ``local-hostname`` alias: ``h`` (:ref:`cloud-config` +* ``local-hostname`` alias: ``h`` (:ref:`cloud-config` preferred) * ``instance-id`` alias: ``i`` (set instance id in :file:`meta-data` instead) diff --git a/doc/rtd/reference/datasources/wsl.rst b/doc/rtd/reference/datasources/wsl.rst index 4b602c3c5df..ab96f9490c4 100644 --- a/doc/rtd/reference/datasources/wsl.rst +++ b/doc/rtd/reference/datasources/wsl.rst @@ -188,8 +188,9 @@ include file. While creating users through cloud-init works as in any other platform, WSL has the concept of the *default user*, which is the user logged in by default. So, to create the default user with cloud-init, one must supply user - data to the :ref:`Users and Groups module ` and write the - entry in ``/etc/wsl.conf`` to make that user the default. See the example: + data to the :ref:`Users and Groups module ` and write + the entry in ``/etc/wsl.conf`` to make that user the default. See the + example: .. code-block:: yaml diff --git a/doc/rtd/reference/modules.rst b/doc/rtd/reference/modules.rst index 6962eb1351d..2a7d26d3068 100644 --- a/doc/rtd/reference/modules.rst +++ b/doc/rtd/reference/modules.rst @@ -4,18 +4,18 @@ Module reference **************** Deprecation schedule and versions ---------------------------------- -Keys may be documented as ``deprecated``, ``new``, or ``changed``. +================================= + +Keys can be documented as ``deprecated``, ``new``, or ``changed``. This allows cloud-init to evolve as requirements change, and to adopt better practices without maintaining design decisions indefinitely. -Keys that have been marked as deprecated or changed may be removed or -changed 5 years from the date of deprecation. For example, a key that is -deprecated in version ``22.1`` (which is the first release in 2022) is -scheduled to be removed in ``27.1`` (first release in 2027). Use of -deprecated keys may cause warnings in the logs. In the case that a -key's expected value changes, the key will be marked ``changed`` with a -date. A 5 year timeline may also be expected for changed keys. +Keys marked as ``deprecated`` or ``changed`` may be removed or changed 5 +years from the deprecation date. For example, if a key is deprecated in +version ``22.1`` (the first release in 2022) it is scheduled to be removed in +``27.1`` (first release in 2027). Use of deprecated keys may cause warnings in +the logs. If a key's expected value changes, the key will be marked +``changed`` with a date. A 5 year timeline also applies to changed keys. .. datatemplate:yaml:: ../../module-docs/cc_ansible/data.yaml :template: modules.tmpl @@ -35,9 +35,6 @@ date. A 5 year timeline may also be expected for changed keys. :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_disable_ec2_metadata/data.yaml :template: modules.tmpl - -.. _mod-disk_setup: - .. datatemplate:yaml:: ../../module-docs/cc_disk_setup/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_fan/data.yaml @@ -64,9 +61,6 @@ date. A 5 year timeline may also be expected for changed keys. :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_mounts/data.yaml :template: modules.tmpl - -.. _mod-ntp: - .. datatemplate:yaml:: ../../module-docs/cc_ntp/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_package_update_upgrade_install/data.yaml @@ -83,47 +77,55 @@ date. A 5 year timeline may also be expected for changed keys. :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_rh_subscription/data.yaml :template: modules.tmpl - -.. _mod-rsyslog: - -.. automodule:: cloudinit.config.cc_rsyslog - -.. _mod-runcmd: - -.. automodule:: cloudinit.config.cc_runcmd -.. automodule:: cloudinit.config.cc_salt_minion -.. automodule:: cloudinit.config.cc_scripts_per_boot -.. automodule:: cloudinit.config.cc_scripts_per_instance -.. automodule:: cloudinit.config.cc_scripts_per_once -.. automodule:: cloudinit.config.cc_scripts_user -.. automodule:: cloudinit.config.cc_scripts_vendor -.. automodule:: cloudinit.config.cc_seed_random - -.. _mod-set_hostname: - -.. automodule:: cloudinit.config.cc_set_hostname - -.. _mod-set_passwords: - -.. automodule:: cloudinit.config.cc_set_passwords -.. automodule:: cloudinit.config.cc_snap -.. automodule:: cloudinit.config.cc_spacewalk -.. automodule:: cloudinit.config.cc_ssh -.. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints -.. automodule:: cloudinit.config.cc_ssh_import_id -.. automodule:: cloudinit.config.cc_timezone -.. automodule:: cloudinit.config.cc_ubuntu_drivers -.. automodule:: cloudinit.config.cc_ubuntu_pro -.. automodule:: cloudinit.config.cc_update_etc_hosts -.. automodule:: cloudinit.config.cc_update_hostname - -.. _mod-users_groups: - -.. automodule:: cloudinit.config.cc_users_groups -.. automodule:: cloudinit.config.cc_wireguard - -.. _mod-write_files: - -.. automodule:: cloudinit.config.cc_write_files -.. automodule:: cloudinit.config.cc_yum_add_repo -.. automodule:: cloudinit.config.cc_zypper_add_repo +.. datatemplate:yaml:: ../../module-docs/cc_rsyslog/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_runcmd/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_salt_minion/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_scripts_per_boot/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_scripts_per_instance/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_scripts_per_once/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_scripts_user/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_scripts_vendor/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_seed_random/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_set_hostname/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_set_passwords/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_snap/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_spacewalk/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_ssh/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_ssh_authkey_fingerprints/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_ssh_import_id/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_timezone/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_ubuntu_drivers/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_ubuntu_pro/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_update_etc_hosts/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_update_hostname/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_users_groups/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_wireguard/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_write_files/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_yum_add_repo/data.yaml + :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_zypper_add_repo/data.yaml + :template: modules.tmpl diff --git a/doc/rtd/templates/modules.tmpl b/doc/rtd/templates/modules.tmpl index 47eccaf0a81..6f7750a7404 100644 --- a/doc/rtd/templates/modules.tmpl +++ b/doc/rtd/templates/modules.tmpl @@ -1,6 +1,9 @@ .. -*- mode: rst -*- {% for mod_id, mod_cfg in data.items() %} {% set mod_meta = config.html_context[mod_id]['meta'] -%} + +.. _mod_{{ mod_id }}: + {{ mod_cfg['name'] }}{% set name_len = mod_cfg['name']|length %} {{ '~' * name_len }} @@ -32,15 +35,18 @@ .. tab-item:: Examples - {% for example in mod_cfg['examples'] %} - {% for line in example["comment"].splitlines() %} - {{ line }} - {% endfor %} - - .. literalinclude:: ../../module-docs/{{ example["file"] }} - :language: yaml + {% if mod_cfg['examples'] %} + {% for example in mod_cfg['examples'] %} + {% for line in example["comment"].splitlines() %} + {{ line }} + {% endfor %} - {% endfor %} + .. literalinclude:: ../../module-docs/{{ example["file"] }} + :language: yaml + {% endfor %} + {% else %} + No examples for this module + {% endif %} {% endfor %} diff --git a/doc/rtd/tutorial/lxd.rst b/doc/rtd/tutorial/lxd.rst index 6d7453d826a..8bde79c852e 100644 --- a/doc/rtd/tutorial/lxd.rst +++ b/doc/rtd/tutorial/lxd.rst @@ -60,9 +60,9 @@ following file on your local filesystem at :file:`/tmp/my-user-data`: Here, we are defining our ``cloud-init`` user data in the :ref:`#cloud-config` format, using the -:ref:`runcmd module ` to define a command to run. When applied, it -will write ``Hello, World!`` to :file:`/var/tmp/hello-world.txt` (as we shall -see later!). +:ref:`runcmd module ` to define a command to run. When applied, +it will write ``Hello, World!`` to :file:`/var/tmp/hello-world.txt` (as we +shall see later!). Launch a LXD container with our user data ========================================= @@ -163,7 +163,7 @@ We can then remove the container completely using: What's next? ============ -In this tutorial, we used the :ref:`runcmd module ` to execute a +In this tutorial, we used the :ref:`runcmd module ` to execute a shell command. The full list of modules available can be found in our :ref:`modules documentation`. Each module contains examples of how to use it. diff --git a/doc/rtd/tutorial/qemu.rst b/doc/rtd/tutorial/qemu.rst index 27f0b980d27..4c1afedd8a1 100644 --- a/doc/rtd/tutorial/qemu.rst +++ b/doc/rtd/tutorial/qemu.rst @@ -86,7 +86,7 @@ Define our user data Now we need to create our :file:`user-data` file. This user data cloud-config sets the password of the default user, and sets that password to never expire. For more details you can refer to the -:ref:`Set Passwords module page`. +:ref:`Set Passwords module page`. Run the following command, which creates a file named :file:`user-data` containing our configuration data. @@ -127,7 +127,7 @@ machine instance. Multiple different format types are supported by :ref:`documentation describing different formats`. The second line, ``password: password``, as per -:ref:`the Users and Groups module docs`, sets the default +:ref:`the Users and Groups module docs`, sets the default user's password to ``password``. The third and fourth lines direct ``cloud-init`` to not require a password diff --git a/doc/rtd/tutorial/wsl.rst b/doc/rtd/tutorial/wsl.rst index 5e429b50e61..ed486809e2d 100644 --- a/doc/rtd/tutorial/wsl.rst +++ b/doc/rtd/tutorial/wsl.rst @@ -189,7 +189,7 @@ cloud-init. For more information, see the :ref:`documentation describing different formats`. The remaining lines, as per -:ref:`the Write Files module docs`, creates a file +:ref:`the Write Files module docs`, creates a file ``/var/tmp/hello-world.txt`` with the content ``Hello from cloud-init`` and permissions allowing anybody on the system to read or write the file. @@ -316,7 +316,7 @@ successfully! What's next? ============ -In this tutorial, we used the :ref:`Write Files module ` to +In this tutorial, we used the :ref:`Write Files module ` to write a file to our WSL instance. The full list of modules available can be found in our :ref:`modules documentation`. Each module contains examples of how to use it. diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index f8f0dcdc563..cad9b75b813 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -1041,8 +1041,8 @@ class TestSchemaDocMarkdown: "frequency": "frequency", "distros": ["debian", "rhel"], "examples": [ - 'prop1:\n [don\'t, expand, "this"]', - "prop2: true", + '\nExample 1:\nprop1:\n [don\'t, expand, "this"]', + "\nExample 2:\nprop2: true", ], } @@ -1091,10 +1091,10 @@ def test_get_meta_doc_returns_restructured_text( " * **prop1:** (array of integer) prop-description", " .. tab-item:: Examples", " ::", - " # --- Example1 ---", + " Example 1:", " prop1:", ' [don\'t, expand, "this"]', - " # --- Example2 ---", + " Example 2:", " prop2: true", ] @@ -1142,10 +1142,10 @@ def test_get_meta_doc_full_with_activate_by_schema_keys( " * **prop2:** (boolean) prop2-description.", " .. tab-item:: Examples", " ::", - " # --- Example1 ---", + " Example 1:", " prop1:", + " Example 2:", ' [don\'t, expand, "this"]', - " # --- Example2 ---", " prop2: true", ] @@ -1342,8 +1342,8 @@ def test_get_meta_doc_handles_string_examples(self, paths, mocker): full_schema.update( { "examples": [ - 'ex1:\n [don\'t, expand, "this"]', - "ex2: true", + 'Example 1:\nex1:\n [don\'t, expand, "this"]', + "Example 2:\nex2: true", ], "properties": { "prop1": { @@ -1359,10 +1359,10 @@ def test_get_meta_doc_handles_string_examples(self, paths, mocker): " * **prop1:** (array of integer) prop-description.\n\n", " .. tab-item:: Examples\n\n", " ::\n\n\n", - " # --- Example1 ---\n\n", + " Example 1:\n", " prop1:\n", ' [don\'t, expand, "this"]\n', - " # --- Example2 ---\n\n", + " Example 2:\n", " prop2: true", ] assert "".join(expected) in get_meta_doc(self.meta, full_schema) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 4315d89cd6e..3a92d29e261 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -327,7 +327,7 @@ def test_wb_schema_subcommand_parser(self, m_read_cfg, capsys): "sle_hpc, sle-micro, sles, TencentOS, ubuntu, virtuozzo", " **resize_rootfs:** ", "(``true``/``false``/``noblock``)", - "runcmd:\n - [ ls, -l, / ]\n", + "runcmd:\n - [ls, -l, /]\n", ], False, id="all_spot_check", diff --git a/tools/format_yaml_doc b/tools/format_yaml_doc new file mode 100755 index 00000000000..697be5b3a5a --- /dev/null +++ b/tools/format_yaml_doc @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Rewrite YAML file using standard yaml.dump index of 2 and sorting keys. + +Useful when creating standard rtd/module-docs/*/data.yaml files. +""" +import argparse +import yaml + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("yaml_file", help="YAML documentation file to reformat.") +parser.add_argument( + "--cloud-config", + action="store_true", + default=False, + help="Append #cloud-config header to rendered content" +) +args = parser.parse_args() +dump_kwargs = { + "indent": 2, + "sort_keys": True, +} +if args.cloud_config: + dump_kwargs["default_flow_style"] = None +formatted_content = yaml.dump( + yaml.safe_load(open(args.yaml_file).read()), **dump_kwargs +) +with open(args.yaml_file, "w") as stream: + if args.cloud_config: + stream.write("#cloud-config\n") + stream.write(formatted_content) + From 70f7e78ec2ba75ba6c813413789871c8deeafbcb Mon Sep 17 00:00:00 2001 From: Curt Moore Date: Thu, 27 Jun 2024 17:48:08 -0500 Subject: [PATCH 15/75] fix: Do not add the vlan_mac_address field into the VLAN object (#5365) This field is required by the OpenStack network_data.json parsing code but is not supported by the VLAN object in the cloudinit network schema v1. Fixes GH-5364 --- cloudinit/sources/helpers/openstack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 4f86de1f883..69a35db7241 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -700,7 +700,6 @@ def convert_net_json(network_json=None, known_macs=None): { "name": name, "vlan_id": link["vlan_id"], - "mac_address": link["vlan_mac_address"], } ) link_updates.append((cfg, "vlan_link", "%s", link["vlan_link"])) From b3618d44a37ae6345f0c3d935b77ae0ae9dd1c92 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 27 Jun 2024 18:12:31 -0600 Subject: [PATCH 16/75] fix(schema): permit deprecated hyphenated keys under users key (#5456) Both hyphenated and underscore delimited key names are permitted by cloudinit/distros/ug_util.py#L114 due to magic replacement of key names. Since this is still valid json schema, add the necessary hyphenated aliases for all users/groups keys. Because the goal in the future is to only support one config key for a given configuraion option, add deprecated keys to those schema definitions. Also drop the description key from the deprecates lock-passwd schema key. Any deprecated schema key which provides a suggested replacement should not provide duplicated key descriptions as the preferred replacement will provided the necessary context. Fixes GH-5454 --- .../schemas/schema-cloud-config-v1.json | 103 +++++++++++++----- tests/unittests/config/test_cc_grub_dpkg.py | 4 +- .../test_cc_package_update_upgrade_install.py | 11 +- .../unittests/config/test_cc_users_groups.py | 33 +++--- tests/unittests/config/test_schema.py | 21 ++-- 5 files changed, 111 insertions(+), 61 deletions(-) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index 5115c3af45e..f1ed3dd8f4d 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -267,9 +267,9 @@ "patternProperties": { "^.+$": { "label": "", - "description": "When providing an object for users.groups the ```` keys are the groups to add this user to", "deprecated": true, "deprecated_version": "23.1", + "deprecated_description": "The use of ``object`` type is deprecated. Use ``string`` or ``array`` of ``string`` instead.", "type": [ "null" ], @@ -292,9 +292,7 @@ "type": "string" }, "lock-passwd": { - "default": true, "type": "boolean", - "description": "Default: ``true``", "deprecated": true, "deprecated_version": "22.3", "deprecated_description": "Use ``lock_passwd`` instead." @@ -304,16 +302,34 @@ "description": "Disable password login. Default: ``true``", "type": "boolean" }, + "no-create-home": { + "type": "boolean", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``no_create_home`` instead." + }, "no_create_home": { "default": false, "description": "Do not create home directory. Default: ``false``", "type": "boolean" }, + "no-log-init": { + "type": "boolean", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``no_log_init`` instead." + }, "no_log_init": { "default": false, "description": "Do not initialize lastlog and faillog for user. Default: ``false``", "type": "boolean" }, + "no-user-group": { + "type": "boolean", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``no_user_group`` instead." + }, "no_user_group": { "default": false, "description": "Do not create group named after user. Default: ``false``", @@ -323,24 +339,54 @@ "description": "Hash of user password applied when user does not exist. This will NOT be applied if the user already exists. To generate this hash, run: ``mkpasswd --method=SHA-512 --rounds=500000`` **Note:** Your password might possibly be visible to unprivileged users on your system, depending on your cloud's security model. Check if your cloud's IMDS server is visible from an unprivileged user to evaluate risk.", "type": "string" }, + "hashed-passwd": { + "type": "string", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``hashed_passwd`` instead." + }, "hashed_passwd": { "description": "Hash of user password to be applied. This will be applied even if the user is preexisting. To generate this hash, run: ``mkpasswd --method=SHA-512 --rounds=500000``. **Note:** Your password might possibly be visible to unprivileged users on your system, depending on your cloud's security model. Check if your cloud's IMDS server is visible from an unprivileged user to evaluate risk.", "type": "string" }, + "plain-text-passwd": { + "type": "string", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``plain_text_passwd`` instead." + }, "plain_text_passwd": { "description": "Clear text of user password to be applied. This will be applied even if the user is preexisting. **Note:** SSH keys or certificates are a safer choice for logging in to your system. For local escalation, supplying a hashed password is a safer choice than plain text. Your password might possibly be visible to unprivileged users on your system, depending on your cloud's security model. An exposed plain text password is an immediate security concern. Check if your cloud's IMDS server is visible from an unprivileged user to evaluate risk.", "type": "string" }, + "create-groups": { + "type": "boolean", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``create_groups`` instead." + }, "create_groups": { "default": true, "description": "Boolean set ``false`` to disable creation of specified user ``groups``. Default: ``true``.", "type": "boolean" }, + "primary-group": { + "type": "string", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``primary_group`` instead." + }, "primary_group": { "default": "````", "description": "Primary group for user. Default: ````", "type": "string" }, + "selinux-user": { + "type": "string", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``selinux_user`` instead." + }, "selinux_user": { "description": "SELinux user for user's login. Default: the default SELinux user.", "type": "string" @@ -362,20 +408,24 @@ "minItems": 1 }, "ssh-authorized-keys": { - "allOf": [ - { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1 - }, - { - "deprecated": true, - "deprecated_version": "18.3", - "deprecated_description": "Use ``ssh_authorized_keys`` instead." - } - ] + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "deprecated": true, + "deprecated_version": "18.3", + "deprecated_description": "Use ``ssh_authorized_keys`` instead." + }, + "ssh-import-id": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``ssh_import_id`` instead." }, "ssh_import_id": { "description": "List of ssh ids to import for user. Can not be combined with ``ssh_redirect_user``. See the man page[1] for more details. [1] https://manpages.ubuntu.com/manpages/noble/en/man1/ssh-import-id.1.html", @@ -385,6 +435,12 @@ }, "minItems": 1 }, + "ssh-redirect-user": { + "type": "boolean", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use ``ssh_redirect_user`` instead." + }, "ssh_redirect_user": { "type": "boolean", "default": false, @@ -487,7 +543,6 @@ "properties": { "remove-defaults": { "type": "boolean", - "default": false, "deprecated": true, "deprecated_version": "22.3", "deprecated_description": "Use ``remove_defaults`` instead." @@ -605,9 +660,9 @@ }, "system_info": { "type": "object", - "description": "System and/or distro specific settings. This is not intended to be overridden by user data or vendor data.", "deprecated": true, - "deprecated_version": "24.2" + "deprecated_version": "24.2", + "deprecated_description": "System and/or distro specific settings. This is not intended to be overridden by user data or vendor data." } } }, @@ -1580,7 +1635,6 @@ }, "grub-dpkg": { "type": "object", - "description": "An alias for ``grub_dpkg``", "deprecated": true, "deprecated_version": "22.2", "deprecated_description": "Use ``grub_dpkg`` instead." @@ -2167,24 +2221,18 @@ }, "apt_update": { "type": "boolean", - "default": false, - "description": "Default: ``false``.", "deprecated": true, "deprecated_version": "22.2", "deprecated_description": "Use ``package_update`` instead." }, "apt_upgrade": { "type": "boolean", - "default": false, - "description": "Default: ``false``.", "deprecated": true, "deprecated_version": "22.2", "deprecated_description": "Use ``package_upgrade`` instead." }, "apt_reboot_if_required": { "type": "boolean", - "default": false, - "description": "Default: ``false``.", "deprecated": true, "deprecated_version": "22.2", "deprecated_description": "Use ``package_reboot_if_required`` instead." @@ -2882,7 +2930,6 @@ } ], "minItems": 1, - "description": "List of ``username:password`` pairs. Each user will have the corresponding password set. A password can be randomly generated by specifying ``RANDOM`` or ``R`` as a user's password. A hashed password, created by a tool like ``mkpasswd``, can be specified. A regex (``r'\\$(1|2a|2y|5|6)(\\$.+){2}'``) is used to determine if a password value should be treated as a hash.", "deprecated": true, "deprecated_version": "22.2", "deprecated_description": "Use ``users`` instead." diff --git a/tests/unittests/config/test_cc_grub_dpkg.py b/tests/unittests/config/test_cc_grub_dpkg.py index b4bd48df154..36ef7fd9821 100644 --- a/tests/unittests/config/test_cc_grub_dpkg.py +++ b/tests/unittests/config/test_cc_grub_dpkg.py @@ -300,8 +300,8 @@ class TestGrubDpkgSchema: pytest.raises( SchemaValidationError, match=( - "Cloud config schema deprecations: grub-dpkg: An alias" - " for ``grub_dpkg`` Deprecated in version 22.2. Use " + "Cloud config schema deprecations: grub-dpkg:" + " Deprecated in version 22.2. Use " "``grub_dpkg`` instead." ), ), diff --git a/tests/unittests/config/test_cc_package_update_upgrade_install.py b/tests/unittests/config/test_cc_package_update_upgrade_install.py index 08db05a03fb..ad3651ad7b9 100644 --- a/tests/unittests/config/test_cc_package_update_upgrade_install.py +++ b/tests/unittests/config/test_cc_package_update_upgrade_install.py @@ -300,16 +300,16 @@ class TestPackageUpdateUpgradeSchema: ( {"apt_update": False}, ( - "Cloud config schema deprecations: apt_update: " - "Default: ``false``. Deprecated in version 22.2. " + "Cloud config schema deprecations: apt_update: " + "Deprecated in version 22.2. " "Use ``package_update`` instead." ), ), ( {"apt_upgrade": False}, ( - "Cloud config schema deprecations: apt_upgrade: " - "Default: ``false``. Deprecated in version 22.2. " + "Cloud config schema deprecations: apt_upgrade: " + "Deprecated in version 22.2. " "Use ``package_upgrade`` instead." ), ), @@ -317,8 +317,7 @@ class TestPackageUpdateUpgradeSchema: {"apt_reboot_if_required": False}, ( "Cloud config schema deprecations: " - "apt_reboot_if_required: Default: ``false``. " - "Deprecated in version 22.2. Use " + "apt_reboot_if_required: Deprecated in version 22.2. Use " "``package_reboot_if_required`` instead." ), ), diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index ceb13a3dd10..9fac84d771b 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -373,9 +373,20 @@ class TestUsersGroupsSchema: SchemaValidationError, match=( "Cloud config schema deprecations: " - "users.0.lock-passwd: Default: ``true`` " - "Deprecated in version 22.3. Use " - "``lock_passwd`` instead." + "users.0.lock-passwd: Deprecated in version 22.3." + " Use ``lock_passwd`` instead." + ), + ), + False, + ), + ( + {"users": [{"name": "bbsw", "no-create-home": True}]}, + pytest.raises( + SchemaValidationError, + match=( + "Cloud config schema deprecations: " + "users.0.no-create-home: Deprecated in version 24.2." + " Use ``no_create_home`` instead." ), ), False, @@ -396,13 +407,10 @@ class TestUsersGroupsSchema: SchemaValidationError, match=( "Cloud config schema deprecations: " - "users.0.groups.adm: When providing an object " - "for users.groups the ```` keys " - "are the groups to add this user to Deprecated" - " in version 23.1., users.0.groups.sudo: When " - "providing an object for users.groups the " - "```` keys are the groups to add " - "this user to Deprecated in version 23.1." + "users.0.groups.adm: Deprecated in version 23.1. " + "The use of ``object`` type is deprecated. Use " + "``string`` or ``array`` of ``string`` instead., " + "users.0.groups.sudo: Deprecated in version 23.1." ), ), False, @@ -458,10 +466,7 @@ class TestUsersGroupsSchema: SchemaValidationError, match=( "Cloud config schema deprecations: " - "user.groups.sbuild: When providing an object " - "for users.groups the ```` keys " - "are the groups to add this user to Deprecated" - " in version 23.1." + "user.groups.sbuild: Deprecated in version 23.1." ), ), False, diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index cad9b75b813..e9e8aa64ce1 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -2773,9 +2773,9 @@ def test_handle_schema_unable_to_read_cfg_paths( apt_reboot_if_required: true # D3 # Deprecations: ------------- - # D1: Default: ``false``. Deprecated in version 22.2. Use ``package_update`` instead. - # D2: Default: ``false``. Deprecated in version 22.2. Use ``package_upgrade`` instead. - # D3: Default: ``false``. Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. + # D1: Deprecated in version 22.2. Use ``package_update`` instead. + # D2: Deprecated in version 22.2. Use ``package_upgrade`` instead. + # D3: Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. Valid schema {cfg_file} """ # noqa: E501 @@ -2795,9 +2795,9 @@ def test_handle_schema_unable_to_read_cfg_paths( apt_reboot_if_required: true # D3 # Deprecations: ------------- - # D1: Default: ``false``. Deprecated in version 22.2. Use ``package_update`` instead. - # D2: Default: ``false``. Deprecated in version 22.2. Use ``package_upgrade`` instead. - # D3: Default: ``false``. Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. + # D1: Deprecated in version 22.2. Use ``package_update`` instead. + # D2: Deprecated in version 22.2. Use ``package_upgrade`` instead. + # D3: Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. Valid schema {cfg_file} """ # noqa: E501 @@ -2810,11 +2810,10 @@ def test_handle_schema_unable_to_read_cfg_paths( dedent( """\ Cloud config schema deprecations: \ -apt_reboot_if_required: Default: ``false``. Deprecated in version 22.2.\ - Use ``package_reboot_if_required`` instead., apt_update: Default: \ -``false``. Deprecated in version 22.2. Use ``package_update`` instead.,\ - apt_upgrade: Default: ``false``. Deprecated in version 22.2. Use \ -``package_upgrade`` instead.\ +apt_reboot_if_required: Deprecated in version 22.2. Use\ + ``package_reboot_if_required`` instead., apt_update: Deprecated in version\ + 22.2. Use ``package_update`` instead., apt_upgrade: Deprecated in version\ + 22.2. Use ``package_upgrade`` instead.\ Valid schema {cfg_file} """ # noqa: E501 ), From 371b2362bbd78ce53cd1b8f69d55db5855434e61 Mon Sep 17 00:00:00 2001 From: Curt Moore Date: Tue, 4 Jun 2024 12:45:32 -0500 Subject: [PATCH 17/75] fix: Ensure properties for bonded interfaces are properly translated (#5367) There is a discrepancy between the properties key name formatting in the OpenStack network_data.json and cloudinit network-config.json specifications. Ensure `bond_` is translated to `bond-` when the OpenStack configuration is parsed by cloudinit. Fixes GH-5366 Co-authored-by: Alberto Contreras --- cloudinit/sources/helpers/openstack.py | 9 ++++++++- tests/unittests/sources/helpers/test_openstack.py | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 69a35db7241..70998dda2ee 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -672,7 +672,14 @@ def convert_net_json(network_json=None, known_macs=None): if k == "bond_links": continue elif k.startswith("bond"): - params.update({k: v}) + # There is a difference in key name formatting for + # bond parameters in the cloudinit and OpenStack + # network schemas. The keys begin with 'bond-' in the + # cloudinit schema but 'bond_' in OpenStack + # network_data.json schema. Translate them to what + # is expected by cloudinit. + translated_key = "bond-{}".format(k.split("bond_", 1)[-1]) + params.update({translated_key: v}) # openstack does not provide a name for the bond. # they do provide an 'id', but that is possibly non-sensical. diff --git a/tests/unittests/sources/helpers/test_openstack.py b/tests/unittests/sources/helpers/test_openstack.py index 312d66a0186..663f6c2db3e 100644 --- a/tests/unittests/sources/helpers/test_openstack.py +++ b/tests/unittests/sources/helpers/test_openstack.py @@ -192,9 +192,9 @@ def test_bond_mac(self): "name": "bond0", "mac_address": "xx:xx:xx:xx:xx:00", "params": { - "bond_miimon": 100, - "bond_mode": "802.3ad", - "bond_xmit_hash_policy": "layer3+4", + "bond-miimon": 100, + "bond-mode": "802.3ad", + "bond-xmit_hash_policy": "layer3+4", }, "subnets": [], "type": "bond", From debafbc9b3b7c2e6555ca7678016a418e0ea336e Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Fri, 28 Jun 2024 11:31:43 +0200 Subject: [PATCH 18/75] test: fix unit test openstack vlan mac_address (#5367) After 70f7e78ec2ba75ba6c813413789871c8deeafbcb, mac_address is not rendered for in openstack for network config v1. Fix the unit test to reflect that. --- tests/unittests/sources/helpers/test_openstack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittests/sources/helpers/test_openstack.py b/tests/unittests/sources/helpers/test_openstack.py index 663f6c2db3e..7ae164140a0 100644 --- a/tests/unittests/sources/helpers/test_openstack.py +++ b/tests/unittests/sources/helpers/test_openstack.py @@ -200,7 +200,6 @@ def test_bond_mac(self): "type": "bond", }, { - "mac_address": "xx:xx:xx:xx:xx:00", "name": "bond0.123", "subnets": [ { From 121539b0f7fd70eafc30c3b16f15a5447fa956ba Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 28 Jun 2024 13:17:34 -0600 Subject: [PATCH 19/75] fix(schema): Don't report changed keys as deprecated (#5464) --- cloudinit/config/schema.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index cf7fd10763e..292c1efe664 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -362,7 +362,7 @@ def _validator( ): """Jsonschema validator for `deprecated` items. - It raises a instance of `error_type` if deprecated that must be handled, + It yields an instance of `error_type` if deprecated that must be handled, otherwise the instance is consider faulty. """ if deprecated: @@ -797,8 +797,9 @@ def validate_cloudconfig_schema( ): # pylint: disable=W1116 if ( "devel" != features.DEPRECATION_INFO_BOUNDARY - and Version.from_str(schema_error.version) - > Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + and (schema_error.version == "devel" + or Version.from_str(schema_error.version) + > Version.from_str(features.DEPRECATION_INFO_BOUNDARY)) ): info_deprecations.append( SchemaProblem(path, schema_error.message) From 726159b9a4b7e4446817b7249171d906678ca278 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 28 Jun 2024 13:19:30 -0600 Subject: [PATCH 20/75] fix(test): Mock version boundary (#5464) --- tests/unittests/config/test_schema.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index e9e8aa64ce1..2ea42b62d5b 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -815,12 +815,13 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( def test_validateconfig_logs_deprecations( self, schema, config, expected_msg, log_deprecations, caplog ): - validate_cloudconfig_schema( - config, - schema=schema, - strict_metaschema=True, - log_deprecations=log_deprecations, - ) + with mock.patch.object(features, "DEPRECATION_INFO_BOUNDARY", "devel"): + validate_cloudconfig_schema( + config, + schema=schema, + strict_metaschema=True, + log_deprecations=log_deprecations, + ) if expected_msg is None: return log_record = (M_PATH[:-1], DEPRECATED_LOG_LEVEL, expected_msg) From 25669f736f015bb2e7e7a9e790ca310a7088f006 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Fri, 28 Jun 2024 15:55:23 -0500 Subject: [PATCH 21/75] fix: dont double-log deprecated INFOs (#5465) --- cloudinit/util.py | 1 - tests/unittests/test_log.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index f42e641440b..8cac7dc8b2d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3255,7 +3255,6 @@ def deprecate( "devel" != features.DEPRECATION_INFO_BOUNDARY and Version.from_str(features.DEPRECATION_INFO_BOUNDARY) < version ): - LOG.info(deprecate_msg) level = logging.INFO elif hasattr(LOG, "deprecated"): level = log.DEPRECATED diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index e68dcc48029..78b53f5a9f9 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -84,7 +84,12 @@ def test_deprecated_log_level(self, caplog): ), ) def test_deprecate_log_level_based_on_features( - self, expected_log_level, deprecation_info_boundary, caplog, mocker + self, + expected_log_level, + deprecation_info_boundary, + caplog, + mocker, + clear_deprecation_log, ): """Deprecation log level depends on key deprecation_version From 5ce2ee36e243b4cc178fef28bc0da06212ac7607 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Fri, 28 Jun 2024 16:02:05 -0500 Subject: [PATCH 22/75] chore: fix schema.py formatting (#5465) --- cloudinit/config/schema.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 292c1efe664..c7c23731aac 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -795,11 +795,10 @@ def validate_cloudconfig_schema( if isinstance( schema_error, SchemaDeprecationError ): # pylint: disable=W1116 - if ( - "devel" != features.DEPRECATION_INFO_BOUNDARY - and (schema_error.version == "devel" + if "devel" != features.DEPRECATION_INFO_BOUNDARY and ( + schema_error.version == "devel" or Version.from_str(schema_error.version) - > Version.from_str(features.DEPRECATION_INFO_BOUNDARY)) + > Version.from_str(features.DEPRECATION_INFO_BOUNDARY) ): info_deprecations.append( SchemaProblem(path, schema_error.message) From 0a698a57f920cb50199cfea40a9aa994965252c0 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Fri, 28 Jun 2024 18:55:08 -0500 Subject: [PATCH 23/75] test: Fix deprecation test failures (#5466) 8906e17e is causing some test failures on older releases. Fix them --- cloudinit/config/schema.py | 14 ++++++-------- cloudinit/util.py | 17 +++++++++++++---- tests/unittests/distros/test_create_users.py | 15 +++++++++++++-- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index c7c23731aac..c72c4ed5f1c 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -31,18 +31,18 @@ import yaml -from cloudinit import features, importer, safeyaml +from cloudinit import importer, safeyaml from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers import INCLUSION_TYPES_MAP, type_from_starts_with from cloudinit.helpers import Paths from cloudinit.sources import DataSourceNotFoundException from cloudinit.temp_utils import mkdtemp from cloudinit.util import ( - Version, error, get_modules_from_dir, load_text_file, load_yaml, + should_log_deprecation, write_file, ) @@ -795,16 +795,14 @@ def validate_cloudconfig_schema( if isinstance( schema_error, SchemaDeprecationError ): # pylint: disable=W1116 - if "devel" != features.DEPRECATION_INFO_BOUNDARY and ( - schema_error.version == "devel" - or Version.from_str(schema_error.version) - > Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + if schema_error.version == "devel" or should_log_deprecation( + schema_error.version ): + deprecations.append(SchemaProblem(path, schema_error.message)) + else: info_deprecations.append( SchemaProblem(path, schema_error.message) ) - else: - deprecations.append(SchemaProblem(path, schema_error.message)) else: errors.append(SchemaProblem(path, schema_error.message)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 8cac7dc8b2d..4380cce3535 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3210,6 +3210,18 @@ def _compare_version(self, other: "Version") -> int: return -1 +def should_log_deprecation(version: str) -> bool: + """Determine if a deprecation message should be logged. + + :param version: The version in which the thing was deprecated. + + :return: True if the message should be logged, else False. + """ + return features.DEPRECATION_INFO_BOUNDARY == "devel" or Version.from_str( + version + ) <= Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + + def deprecate( *, deprecated: str, @@ -3251,10 +3263,7 @@ def deprecate( f"{deprecated_version} and scheduled to be removed in " f"{version_removed}. {message}" ).rstrip() - if ( - "devel" != features.DEPRECATION_INFO_BOUNDARY - and Version.from_str(features.DEPRECATION_INFO_BOUNDARY) < version - ): + if not should_log_deprecation(deprecated_version): level = logging.INFO elif hasattr(LOG, "deprecated"): level = log.DEPRECATED diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 039723aaad2..86c79fb1775 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -5,6 +5,7 @@ import pytest from cloudinit import distros, ssh_util +from cloudinit.util import should_log_deprecation from tests.unittests.helpers import mock from tests.unittests.util import abstract_to_concrete @@ -142,7 +143,12 @@ def test_create_groups_with_dict_deprecated( ] assert m_subp.call_args_list == expected - assert caplog.records[0].levelname in ["WARNING", "DEPRECATED"] + expected_levels = ( + ["WARNING", "DEPRECATED"] + if should_log_deprecation("23.1") + else ["INFO"] + ) + assert caplog.records[0].levelname in expected_levels assert ( "The user foo_user has a 'groups' config value of type dict" in caplog.records[0].message @@ -170,7 +176,12 @@ def test_explicit_sudo_false(self, m_subp, dist, caplog): mock.call(["passwd", "-l", USER]), ] - assert caplog.records[1].levelname in ["WARNING", "DEPRECATED"] + expected_levels = ( + ["WARNING", "DEPRECATED"] + if should_log_deprecation("22.3") + else ["INFO"] + ) + assert caplog.records[1].levelname in expected_levels assert ( "The value of 'false' in user foo_user's 'sudo' " "config is deprecated in 22.3 and scheduled to be removed" From 7693ee2a6846ab8953b37eee36d275898eb73001 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 06:58:28 -0600 Subject: [PATCH 24/75] tests: update keyserver PPA key fur curtin-dev (#5472) Commit 8470af00 changes the integration test keyserver PPA to ppa:curtin-dev/daily. Update the assertion key fingerprint. --- tests/integration_tests/modules/test_apt_functionality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 1b49722fd03..b86e217556e 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -124,7 +124,7 @@ r"deb-src http://badsecurity.ubuntu.com/ubuntu [a-z]+-security multiverse", ] -TEST_KEYSERVER_KEY = "110E 21D8 B0E2 A1F0 243A F682 0856 F197 B892 ACEA" +TEST_KEYSERVER_KEY = "1BC3 0F71 5A3B 8612 47A8 1A5E 55FE 7C8C 0165 013E" TEST_PPA_KEY = "3552 C902 B4DD F7BD 3842 1821 015D 28D7 4416 14D8" TEST_KEY = "1FF0 D853 5EF7 E719 E5C8 1B9C 083D 06FB E4D3 04DF" TEST_SIGNED_BY_KEY = "A2EB 2DEC 0BD7 519B 7B38 BE38 376A 290E C806 8B11" From d59724a1516c6d434063a5677bd3ea67257f83a6 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 07:07:05 -0600 Subject: [PATCH 25/75] tests: integration tests aware of features.DEPRECATION_INFO_BOUNDARY Adapt util.should_log_deprecation to expect two params: - version: where the key/feature was deprecated - boundary_version: at which deprecation level logs occur --- cloudinit/config/schema.py | 4 +- cloudinit/util.py | 11 +++-- tests/integration_tests/cmd/test_schema.py | 42 +++++++++++-------- tests/integration_tests/cmd/test_status.py | 39 ++++++++++++----- .../modules/test_combined.py | 18 ++++++-- tests/unittests/distros/test_create_users.py | 10 +++-- 6 files changed, 83 insertions(+), 41 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index c72c4ed5f1c..e59fde4ee92 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -31,7 +31,7 @@ import yaml -from cloudinit import importer, safeyaml +from cloudinit import features, importer, safeyaml from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers import INCLUSION_TYPES_MAP, type_from_starts_with from cloudinit.helpers import Paths @@ -796,7 +796,7 @@ def validate_cloudconfig_schema( schema_error, SchemaDeprecationError ): # pylint: disable=W1116 if schema_error.version == "devel" or should_log_deprecation( - schema_error.version + schema_error.version, features.DEPRECATION_INFO_BOUNDARY ): deprecations.append(SchemaProblem(path, schema_error.message)) else: diff --git a/cloudinit/util.py b/cloudinit/util.py index 4380cce3535..5b8f07683f6 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3210,16 +3210,17 @@ def _compare_version(self, other: "Version") -> int: return -1 -def should_log_deprecation(version: str) -> bool: +def should_log_deprecation(version: str, boundary_version: str) -> bool: """Determine if a deprecation message should be logged. :param version: The version in which the thing was deprecated. + :param boundary_version: The version at which deprecation level is logged. :return: True if the message should be logged, else False. """ - return features.DEPRECATION_INFO_BOUNDARY == "devel" or Version.from_str( + return boundary_version == "devel" or Version.from_str( version - ) <= Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + ) <= Version.from_str(boundary_version) def deprecate( @@ -3263,7 +3264,9 @@ def deprecate( f"{deprecated_version} and scheduled to be removed in " f"{version_removed}. {message}" ).rstrip() - if not should_log_deprecation(deprecated_version): + if not should_log_deprecation( + deprecated_version, features.DEPRECATION_INFO_BOUNDARY + ): level = logging.INFO elif hasattr(LOG, "deprecated"): level = log.DEPRECATED diff --git a/tests/integration_tests/cmd/test_schema.py b/tests/integration_tests/cmd/test_schema.py index 70eb0a67c6f..3155a07919b 100644 --- a/tests/integration_tests/cmd/test_schema.py +++ b/tests/integration_tests/cmd/test_schema.py @@ -3,9 +3,13 @@ import pytest +from cloudinit.util import should_log_deprecation from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.releases import CURRENT_RELEASE, MANTIC -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import ( + get_feature_flag_value, + verify_clean_log, +) USER_DATA = """\ #cloud-config @@ -62,10 +66,19 @@ class TestSchemaDeprecations: def test_clean_log(self, class_client: IntegrationInstance): log = class_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log, ignore_deprecations=True) - assert "DEPRECATED]: Deprecated cloud-config provided:" in log - assert "apt_reboot_if_required: Default: ``false``. Deprecated " in log - assert "apt_update: Default: ``false``. Deprecated in version" in log - assert "apt_upgrade: Default: ``false``. Deprecated in version" in log + version_boundary = get_feature_flag_value( + class_client, "DEPRECATION_INFO_BOUNDARY" + ) + # the deprecation_version is 22.2 in schema for apt_* keys in + # user-data. Pass 22.2 in against the client's version_boundary. + if should_log_deprecation("22.2", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + assert f"{log_level}]: Deprecated cloud-config provided:" in log + assert "apt_reboot_if_required: Deprecated " in log + assert "apt_update: Deprecated in version" in log + assert "apt_upgrade: Deprecated in version" in log def test_network_config_schema_validation( self, class_client: IntegrationInstance @@ -139,17 +152,10 @@ def test_schema_deprecations(self, class_client: IntegrationInstance): ), "`schema` cmd must return 0 even with deprecated configs" assert not result.stderr assert "Cloud config schema deprecations:" in result.stdout + assert "apt_update: Deprecated in version" in result.stdout + assert "apt_upgrade: Deprecated in version" in result.stdout assert ( - "apt_update: Default: ``false``. Deprecated in version" - in result.stdout - ) - assert ( - "apt_upgrade: Default: ``false``. Deprecated in version" - in result.stdout - ) - assert ( - "apt_reboot_if_required: Default: ``false``. Deprecated in version" - in result.stdout + "apt_reboot_if_required: Deprecated in version" in result.stdout ) annotated_result = class_client.execute( @@ -167,9 +173,9 @@ def test_schema_deprecations(self, class_client: IntegrationInstance): apt_reboot_if_required: false\t\t# D3 # Deprecations: ------------- - # D1: Default: ``false``. Deprecated in version 22.2. Use ``package_update`` instead. - # D2: Default: ``false``. Deprecated in version 22.2. Use ``package_upgrade`` instead. - # D3: Default: ``false``. Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. + # D1: Deprecated in version 22.2. Use ``package_update`` instead. + # D2: Deprecated in version 22.2. Use ``package_upgrade`` instead. + # D3: Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. Valid schema /root/user-data""" # noqa: E501 diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index 4eda01e82fb..20c78433257 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -3,12 +3,14 @@ import pytest +from cloudinit.util import should_log_deprecation from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, JAMMY from tests.integration_tests.util import ( + get_feature_flag_value, push_and_enable_systemd_unit, wait_for_cloud_init, ) @@ -79,17 +81,32 @@ def test_status_json_errors(client): "DEPRECATED" ) - status_json = client.execute("cloud-init status --format json").stdout - assert "Deprecated cloud-config provided:\nca-certs:" in json.loads( - status_json - )["init"]["recoverable_errors"].get("DEPRECATED").pop(0) - assert "Deprecated cloud-config provided:\nca-certs:" in json.loads( - status_json - )["recoverable_errors"].get("DEPRECATED").pop(0) - assert "cloud-config failed schema validation" in json.loads(status_json)[ - "init" - ]["recoverable_errors"].get("WARNING").pop(0) - assert "cloud-config failed schema validation" in json.loads(status_json)[ + status_json = json.loads( + client.execute("cloud-init status --format json").stdout + ) + version_boundary = get_feature_flag_value( + client, "DEPRECATION_INFO_BOUNDARY" + ) + # The deprecation_version is 22.3 in schema for ca-certs. + # Expect an extra deprecation level log in status if boundary > 22.3 + if should_log_deprecation("22.3", version_boundary): + assert "Deprecated cloud-config provided: ca-certs" in status_json[ + "init" + ]["recoverable_errors"].get("DEPRECATED").pop(0) + assert "Deprecated cloud-config provided: ca-certs" in status_json[ + "recoverable_errors" + ].get("DEPRECATED").pop(0) + + assert "Key 'ca-certs' is deprecated in 22.1" in status_json["init"][ + "recoverable_errors" + ].get("DEPRECATED").pop(0) + assert "Key 'ca-certs' is deprecated in 22.1" in status_json[ + "recoverable_errors" + ].get("DEPRECATED").pop(0) + assert "cloud-config failed schema validation" in status_json["init"][ + "recoverable_errors" + ].get("WARNING").pop(0) + assert "cloud-config failed schema validation" in status_json[ "recoverable_errors" ].get("WARNING").pop(0) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 8a73e7e8623..4a7c2dfc9e0 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -15,7 +15,7 @@ import pytest import cloudinit.config -from cloudinit.util import is_true +from cloudinit.util import is_true, should_log_deprecation from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM @@ -131,8 +131,20 @@ def test_deprecated_message(self, class_client: IntegrationInstance): """Check that deprecated key produces a log warning""" client = class_client log = client.read_from_file("/var/log/cloud-init.log") - assert "Deprecated cloud-config provided" in log - assert "The value of 'false' in user craig's 'sudo' config is " in log + version_boundary = get_feature_flag_value( + class_client, "DEPRECATION_INFO_BOUNDARY" + ) + # the deprecation_version is 22.2 in schema for apt_* keys in + # user-data. Pass 22.2 in against the client's version_boundary. + if should_log_deprecation("22.2", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + + assert ( + f"[{log_level}]: The value of 'false' in user craig's 'sudo'" + " config is deprecated" in log + ) assert 2 == log.count("DEPRECATE") def test_ntp_with_apt(self, class_client: IntegrationInstance): diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 86c79fb1775..893b47060d4 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -4,7 +4,7 @@ import pytest -from cloudinit import distros, ssh_util +from cloudinit import distros, features, ssh_util from cloudinit.util import should_log_deprecation from tests.unittests.helpers import mock from tests.unittests.util import abstract_to_concrete @@ -145,7 +145,9 @@ def test_create_groups_with_dict_deprecated( expected_levels = ( ["WARNING", "DEPRECATED"] - if should_log_deprecation("23.1") + if should_log_deprecation( + "23.1", features.DEPRECATION_INFO_BOUNDARY + ) else ["INFO"] ) assert caplog.records[0].levelname in expected_levels @@ -178,7 +180,9 @@ def test_explicit_sudo_false(self, m_subp, dist, caplog): expected_levels = ( ["WARNING", "DEPRECATED"] - if should_log_deprecation("22.3") + if should_log_deprecation( + "22.3", features.DEPRECATION_INFO_BOUNDARY + ) else ["INFO"] ) assert caplog.records[1].levelname in expected_levels From 681b7de1a598bc5ab1b7d9868dcf2f32fd45aba3 Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Mon, 1 Jul 2024 21:04:21 +0200 Subject: [PATCH 26/75] fix(rh_subscription): add string type to org (#5453) Per [1], org's correct type is string. Added as new type and deprecated integer. References: [1] https://github.com/candlepin/subscription-manager/blob/b6fad11e7783ae414fe88fdecee57d8db5c8e292/man/subscription-manager.8#L589 Fixes GH-5382 Co-authored-by: pneigel-ca --- .../config/schemas/schema-cloud-config-v1.json | 14 ++++++++++++-- doc/module-docs/cc_rh_subscription/example2.yaml | 2 +- .../unittests/config/test_cc_rh_subscription.py | 16 +++++++++++++++- tools/.github-cla-signers | 1 + 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index f1ed3dd8f4d..f5609c539fc 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -2535,8 +2535,18 @@ "description": "The activation key to use. Must be used with ``org``. Should not be used with ``username`` or ``password``" }, "org": { - "type": "integer", - "description": "The organization number to use. Must be used with ``activation-key``. Should not be used with ``username`` or ``password``" + "description": "The organization to use. Must be used with ``activation-key``. Should not be used with ``username`` or ``password``", + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use of type integer for this value is deprecated. Use a string instead." + } + ] }, "auto-attach": { "type": "boolean", diff --git a/doc/module-docs/cc_rh_subscription/example2.yaml b/doc/module-docs/cc_rh_subscription/example2.yaml index b6fff8c44d1..72328f93811 100644 --- a/doc/module-docs/cc_rh_subscription/example2.yaml +++ b/doc/module-docs/cc_rh_subscription/example2.yaml @@ -1,4 +1,4 @@ #cloud-config rh_subscription: activation-key: foobar - org: 12345 + org: "ABC" diff --git a/tests/unittests/config/test_cc_rh_subscription.py b/tests/unittests/config/test_cc_rh_subscription.py index 955b092ba16..d811d16a5b6 100644 --- a/tests/unittests/config/test_cc_rh_subscription.py +++ b/tests/unittests/config/test_cc_rh_subscription.py @@ -184,7 +184,7 @@ class TestBadInput(CiTestCase): "rh_subscription": { "activation-key": "abcdef1234", "fookey": "bar", - "org": "123", + "org": "ABC", } } @@ -330,6 +330,20 @@ class TestRhSubscriptionSchema: {"rh_subscription": {"disable-repo": "name"}}, "'name' is not of type 'array'", ), + ( + { + "rh_subscription": { + "activation-key": "foobar", + "org": "ABC", + } + }, + None, + ), + ( + {"rh_subscription": {"activation-key": "foobar", "org": 314}}, + "Deprecated in version 24.2. Use of type integer for this" + " value is deprecated. Use a string instead.", + ), ], ) @skipUnlessJsonSchema() diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 3aea01b0023..d9accd11460 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -144,6 +144,7 @@ outscale-mdr philsphicas phsm phunyguy +pneigel-ca qubidt r00ta RedKrieg From ee1b25b73d8c08bf079a34af3837760e283715b0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 13:53:20 -0600 Subject: [PATCH 27/75] tests: update nocloud deprecation test for boundary version (#5474) --- .../integration_tests/datasources/test_nocloud.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/datasources/test_nocloud.py b/tests/integration_tests/datasources/test_nocloud.py index a21f91cfa21..c6c440840a3 100644 --- a/tests/integration_tests/datasources/test_nocloud.py +++ b/tests/integration_tests/datasources/test_nocloud.py @@ -6,10 +6,12 @@ from pycloudlib.lxd.instance import LXDInstance from cloudinit.subp import subp +from cloudinit.util import should_log_deprecation from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL from tests.integration_tests.util import ( + get_feature_flag_value, override_kernel_command_line, verify_clean_boot, verify_clean_log, @@ -193,9 +195,18 @@ def test_smbios_seed_network(self, client: IntegrationInstance): assert client.execute("cloud-init clean --logs").ok client.restart() assert client.execute("test -f /var/tmp/smbios_test_file").ok - assert "'nocloud-net' datasource name is deprecated" in client.execute( - "cloud-init status --format json" + version_boundary = get_feature_flag_value( + client, "DEPRECATION_INFO_BOUNDARY" ) + # nocloud-net deprecated in version 24.1 + if should_log_deprecation("24.1", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + client.execute( + rf"grep \"{log_level}]: The 'nocloud-net' datasource name is" + ' deprecated" /var/log/cloud-init.log' + ).ok @pytest.mark.skipif(PLATFORM != "lxd_vm", reason="Modifies grub config") From c45280e981a0658d637af8736036e653ffe9646e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 15:04:45 -0600 Subject: [PATCH 28/75] tests: update ubuntu_pro test to account for info-level deprecations (#5475) The ubuntu_advantage key is deprecated in 24.1 in favor of ubuntu_pro. Any version less than 24.1 will have only an info level log resulting in no DEPRECATED keys from cloud-init status --format=yaml. Parse log level messages in /var/log/cloud-init.log instead. --- .../modules/test_ubuntu_pro.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/integration_tests/modules/test_ubuntu_pro.py b/tests/integration_tests/modules/test_ubuntu_pro.py index 20bdd5b9510..f4438163425 100644 --- a/tests/integration_tests/modules/test_ubuntu_pro.py +++ b/tests/integration_tests/modules/test_ubuntu_pro.py @@ -5,6 +5,7 @@ import pytest from pycloudlib.cloud import ImageType +from cloudinit.util import should_log_deprecation from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.conftest import get_validated_source from tests.integration_tests.instances import ( @@ -19,7 +20,10 @@ IS_UBUNTU, JAMMY, ) -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import ( + get_feature_flag_value, + verify_clean_log, +) LOG = logging.getLogger("integration_testing.test_ubuntu_pro") @@ -135,12 +139,18 @@ def test_valid_token(self, client: IntegrationInstance): "sed -i 's/ubuntu_pro$/ubuntu_advantage/' /etc/cloud/cloud.cfg" ) client.restart() - status_resp = client.execute("cloud-init status --format json") - status = json.loads(status_resp.stdout) - assert ( - "Module has been renamed from cc_ubuntu_advantage to cc_ubuntu_pro" - in "\n".join(status["recoverable_errors"]["DEPRECATED"]) + version_boundary = get_feature_flag_value( + client, "DEPRECATION_INFO_BOUNDARY" ) + # ubuntu_advantage key is deprecated in version 24.1 + if should_log_deprecation("24.1", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + client.execute( + rf"grep \"{log_level}]: Module has been renamed from" + " cc_ubuntu_advantage to cc_ubuntu_pro /var/log/cloud-init.log" + ).ok assert is_attached(client) @pytest.mark.user_data(ATTACH.format(token=CLOUD_INIT_UA_TOKEN)) From 40e2eb4aea83299b3de497b6237cb64ae48cfa74 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 1 Jul 2024 16:40:49 -0500 Subject: [PATCH 29/75] test: Update integration tests to pass on focal (#5476) - Update deprecation test to include focal. - Update test using `cloud-id` appropriately --- tests/integration_tests/cmd/test_status.py | 45 +++++++++------------ tests/integration_tests/test_ds_identify.py | 4 +- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index 20c78433257..84d9b694f63 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -3,14 +3,12 @@ import pytest -from cloudinit.util import should_log_deprecation from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, JAMMY from tests.integration_tests.util import ( - get_feature_flag_value, push_and_enable_systemd_unit, wait_for_cloud_init, ) @@ -65,8 +63,10 @@ def test_wait_when_no_datasource(session_cloud: IntegrationCloud, setup_image): USER_DATA = """\ #cloud-config +users: + - name: something + ssh-authorized-keys: ["something"] ca-certs: - remove_defaults: false invalid_key: true """ @@ -81,32 +81,23 @@ def test_status_json_errors(client): "DEPRECATED" ) - status_json = json.loads( - client.execute("cloud-init status --format json").stdout + status_json = client.execute("cloud-init status --format json").stdout + assert ( + "Deprecated cloud-config provided: users.0.ssh-authorized-keys" + in json.loads(status_json)["init"]["recoverable_errors"] + .get("DEPRECATED") + .pop(0) ) - version_boundary = get_feature_flag_value( - client, "DEPRECATION_INFO_BOUNDARY" + assert ( + "Deprecated cloud-config provided: users.0.ssh-authorized-keys:" + in json.loads(status_json)["recoverable_errors"] + .get("DEPRECATED") + .pop(0) ) - # The deprecation_version is 22.3 in schema for ca-certs. - # Expect an extra deprecation level log in status if boundary > 22.3 - if should_log_deprecation("22.3", version_boundary): - assert "Deprecated cloud-config provided: ca-certs" in status_json[ - "init" - ]["recoverable_errors"].get("DEPRECATED").pop(0) - assert "Deprecated cloud-config provided: ca-certs" in status_json[ - "recoverable_errors" - ].get("DEPRECATED").pop(0) - - assert "Key 'ca-certs' is deprecated in 22.1" in status_json["init"][ - "recoverable_errors" - ].get("DEPRECATED").pop(0) - assert "Key 'ca-certs' is deprecated in 22.1" in status_json[ - "recoverable_errors" - ].get("DEPRECATED").pop(0) - assert "cloud-config failed schema validation" in status_json["init"][ - "recoverable_errors" - ].get("WARNING").pop(0) - assert "cloud-config failed schema validation" in status_json[ + assert "cloud-config failed schema validation" in json.loads(status_json)[ + "init" + ]["recoverable_errors"].get("WARNING").pop(0) + assert "cloud-config failed schema validation" in json.loads(status_json)[ "recoverable_errors" ].get("WARNING").pop(0) diff --git a/tests/integration_tests/test_ds_identify.py b/tests/integration_tests/test_ds_identify.py index 2375dde7909..59d3edd778f 100644 --- a/tests/integration_tests/test_ds_identify.py +++ b/tests/integration_tests/test_ds_identify.py @@ -1,7 +1,7 @@ """test that ds-identify works as expected""" from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.integration_settings import OS_IMAGE, PLATFORM +from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.util import verify_clean_log, wait_for_cloud_init DATASOURCE_LIST_FILE = "/etc/cloud/cloud.cfg.d/90_dpkg.cfg" @@ -29,8 +29,6 @@ def test_ds_identify(client: IntegrationInstance): assert client.execute("cloud-init status --wait") datasource = MAP_PLATFORM_TO_DATASOURCE.get(PLATFORM, PLATFORM) - if "lxd" == datasource and "focal" == OS_IMAGE: - datasource = "nocloud" cloud_id = client.execute("cloud-id") assert cloud_id.ok assert datasource == cloud_id.stdout.rstrip() From 41c375c4f011ac030e2cf59d6741b24a28fbe257 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 12:56:50 -0600 Subject: [PATCH 30/75] ci(mypy): Set default follow_imports value (#5350) Remove code for detecting a virtualenv. This has been broken since ~3.7. --- setup.py | 17 +++++++---------- setup_utils.py | 9 --------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 3becd7bbb1d..3e33d0062bd 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ # isort: off from setup_utils import ( # noqa: E402 get_version, - in_virtualenv, is_f, is_generator, pkg_config_read, @@ -266,13 +265,12 @@ def finalize_options(self): self.distribution.reinitialize_command("install_data", True) -if not in_virtualenv(): - USR = "/" + USR - ETC = "/" + ETC - USR_LIB_EXEC = "/" + USR_LIB_EXEC - LIB = "/" + LIB - for k in INITSYS_ROOTS.keys(): - INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k] +USR = "/" + USR +ETC = "/" + ETC +USR_LIB_EXEC = "/" + USR_LIB_EXEC +LIB = "/" + LIB +for k in INITSYS_ROOTS.keys(): + INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k] data_files = [ (ETC + "/cloud", [render_tmpl("config/cloud.cfg.tmpl", is_yaml=True)]), @@ -308,8 +306,7 @@ def finalize_options(self): ] if not platform.system().endswith("BSD"): RULES_PATH = pkg_config_read("udev", "udevdir") - if not in_virtualenv(): - RULES_PATH = "/" + RULES_PATH + RULES_PATH = "/" + RULES_PATH data_files.extend( [ diff --git a/setup_utils.py b/setup_utils.py index f88905afdfb..0ff7581070c 100644 --- a/setup_utils.py +++ b/setup_utils.py @@ -35,15 +35,6 @@ def pkg_config_read(library: str, var: str) -> str: return path -def in_virtualenv() -> bool: - # TODO: sys.real_prefix doesn't exist on any currently supported - # version of python. This function can never return True - try: - return sys.real_prefix != sys.prefix - except AttributeError: - return False - - def version_to_pep440(version: str) -> str: # read-version can spit out something like 22.4-15-g7f97aee24 # which is invalid under PEP 440. If we replace the first - with a + From 16018b244922f71f6e2882567c58f8cbf5b4cdff Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:14:34 -0600 Subject: [PATCH 31/75] fix(typing): Remove invalid type annotations (#5350) Invalid type annotations confuse mypy. When it sees a type annotation that doesn't conflict with other annotations, it is trusted. This means that future attempts to conditionally check a variable's type will cause mypy to produce an unreachable code path warning. --- cloudinit/config/cc_lxd.py | 4 +--- tests/unittests/sources/test_akamai.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 85e50679209..aa77e9ef901 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -30,9 +30,7 @@ } # type: ignore -def supplemental_schema_validation( - init_cfg: dict, bridge_cfg: dict, preseed_str: str -): +def supplemental_schema_validation(init_cfg, bridge_cfg, preseed_str): """Validate user-provided lxd network and bridge config option values. @raises: ValueError describing invalid values provided. diff --git a/tests/unittests/sources/test_akamai.py b/tests/unittests/sources/test_akamai.py index 7b6987bfb47..2480269f6e6 100644 --- a/tests/unittests/sources/test_akamai.py +++ b/tests/unittests/sources/test_akamai.py @@ -1,5 +1,5 @@ from contextlib import suppress -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import pytest @@ -224,7 +224,7 @@ def test_get_network_context_managers( get_interfaces_by_mac, local_stage: bool, ds_cfg: Dict[str, Any], - expected_manager_config: List[Tuple[Tuple[bool, bool], bool]], + expected_manager_config: List, expected_interface: str, ): """ From 555028524368facc375e28e3b4631d595e80c95f Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:28:00 -0600 Subject: [PATCH 32/75] fix(typing): Remove type annotation for unused variable (#5350) Later use of the same variable name leads mypy to report unreachable code paths. --- cloudinit/config/cc_power_state_change.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 69420d6f960..5bd6b96d847 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -78,8 +78,8 @@ def check_condition(cond): def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: try: - (args, timeout, condition) = load_power_state(cfg, cloud.distro) - if args is None: + (arg_list, timeout, condition) = load_power_state(cfg, cloud.distro) + if arg_list is None: LOG.debug("no power_state provided. doing nothing") return except Exception as e: @@ -99,7 +99,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: devnull_fp = open(os.devnull, "w") - LOG.debug("After pid %s ends, will execute: %s", mypid, " ".join(args)) + LOG.debug("After pid %s ends, will execute: %s", mypid, " ".join(arg_list)) util.fork_cb( run_after_pid_gone, @@ -108,7 +108,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: timeout, condition, execmd, - [args, devnull_fp], + [arg_list, devnull_fp], ) From a8cf24b7c8d777913974ea07e25e6f1cd50b82bd Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:46:37 -0600 Subject: [PATCH 33/75] fix(typing): Add / update type annotations (#5350) Mypy infers an unannotated variable's type from its initial value. When a variable's type may change (such as from None to non-None), the variable's type needs to be annotated, otherwise mypy may warn of unreachable code paths. --- cloudinit/config/schema.py | 2 +- cloudinit/net/dhcp.py | 4 +++- cloudinit/net/ephemeral.py | 4 ++-- cloudinit/sources/DataSourceAzure.py | 6 +++--- cloudinit/url_helper.py | 3 ++- tests/unittests/cmd/test_status.py | 2 +- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index e59fde4ee92..790fe1d1e80 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -127,7 +127,7 @@ class MetaSchema(TypedDict): title: str description: str distros: typing.List[str] - examples: typing.List[str] + examples: typing.List[Union[dict, str]] frequency: str activate_by_schema_keys: NotRequired[List[str]] diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 49d5ae80147..6b3aabc8603 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -82,7 +82,9 @@ class NoDHCPLeaseMissingDhclientError(NoDHCPLeaseError): """Raised when unable to find dhclient.""" -def maybe_perform_dhcp_discovery(distro, nic=None, dhcp_log_func=None): +def maybe_perform_dhcp_discovery( + distro, nic=None, dhcp_log_func=None +) -> Dict[str, Any]: """Perform dhcp discovery if nic valid and dhclient command exists. If the nic is invalid or undiscoverable or dhclient command is not found, diff --git a/cloudinit/net/ephemeral.py b/cloudinit/net/ephemeral.py index 4ac2aa12736..c8730fb1e8a 100644 --- a/cloudinit/net/ephemeral.py +++ b/cloudinit/net/ephemeral.py @@ -285,8 +285,8 @@ def __init__( dhcp_log_func=None, ): self.iface = iface - self._ephipv4 = None - self.lease = None + self._ephipv4: Optional[EphemeralIPv4Network] = None + self.lease: Optional[Dict[str, Any]] = None self.dhcp_log_func = dhcp_log_func self.connectivity_url_data = connectivity_url_data self.distro = distro diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 04a928c31af..44b1e194fa4 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -331,7 +331,7 @@ def __init__(self, sys_cfg, distro, paths): ) self._iso_dev = None self._network_config = None - self._ephemeral_dhcp_ctx = None + self._ephemeral_dhcp_ctx: Optional[EphemeralDHCPv4] = None self._route_configured_for_imds = False self._route_configured_for_wireserver = False self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT @@ -426,7 +426,7 @@ def _setup_ephemeral_networking( dhcp_log_func=dhcp_log_cb, ) - lease = None + lease: Optional[Dict[str, Any]] = None start_time = monotonic() deadline = start_time + timeout_minutes * 60 with events.ReportEventStack( @@ -1252,7 +1252,7 @@ def _wait_for_pps_unknown_reuse(self): def _poll_imds(self) -> bytes: """Poll IMDs for reprovisiondata XML document data.""" dhcp_attempts = 0 - reprovision_data = None + reprovision_data: Optional[bytes] = None while not reprovision_data: if not self._is_ephemeral_networking_up(): dhcp_attempts += 1 diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index a897194661a..777aa6a70e3 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -279,7 +279,8 @@ def __init__(self, response: requests.Response): @property def contents(self) -> bytes: if self._response.content is None: - return b"" + # typeshed bug: https://github.com/python/typeshed/pull/12180 + return b"" # type: ignore return self._response.content @property diff --git a/tests/unittests/cmd/test_status.py b/tests/unittests/cmd/test_status.py index ddf738560f8..22241139ff1 100644 --- a/tests/unittests/cmd/test_status.py +++ b/tests/unittests/cmd/test_status.py @@ -746,7 +746,7 @@ def test_status_output( assert_file, cmdargs: MyArgs, expected_retcode: int, - expected_status: str, + expected_status: Union[str, dict], config: Config, capsys, ): From b26d388ceb7c8f0cf2e8fc214049f9477090ddc4 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:52:01 -0600 Subject: [PATCH 34/75] refactor(typing): Remove unused code paths (#5350) Enable mypy's "warn_unreachable" checks. Remove adjacent unused mocks. Note: mypy can incorrectly report typing errors as "unused code paths" --- cloudinit/config/schema.py | 10 ----- cloudinit/sources/DataSourceLXD.py | 4 -- cloudinit/sources/DataSourceNWCS.py | 12 +++-- cloudinit/sources/__init__.py | 10 ++++- cloudinit/temp_utils.py | 12 +---- cloudinit/util.py | 2 - pyproject.toml | 1 + tests/integration_tests/test_upgrade.py | 2 - .../unittests/config/test_cc_apk_configure.py | 11 +++-- tests/unittests/config/test_cc_ntp.py | 44 +++++++++---------- tests/unittests/config/test_schema.py | 23 ---------- tests/unittests/test_temp_utils.py | 5 --- 12 files changed, 42 insertions(+), 94 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 790fe1d1e80..062ab92ecd8 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -939,16 +939,6 @@ def annotate( if not schema_errors and not schema_deprecations: return self._original_content lines = self._original_content.split("\n") - if not isinstance(self._cloudconfig, dict): - # Return a meaningful message on empty cloud-config - return "\n".join( - lines - + [ - self._build_footer( - "Errors", ["# E1: Cloud-config is not a YAML dict."] - ) - ] - ) errors_by_line = self._build_errors_by_line(schema_errors) deprecations_by_line = self._build_errors_by_line(schema_deprecations) annotated_content = self._annotate_content( diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py index a85853ec44a..4f69d90eb70 100644 --- a/cloudinit/sources/DataSourceLXD.py +++ b/cloudinit/sources/DataSourceLXD.py @@ -213,10 +213,6 @@ def _get_data(self) -> bool: user_metadata = _raw_instance_data_to_dict( "user.meta-data", user_metadata ) - if not isinstance(self.metadata, dict): - self.metadata = util.mergemanydict( - [util.load_yaml(self.metadata), user_metadata] - ) if "user-data" in self._crawled_metadata: self.userdata_raw = self._crawled_metadata["user-data"] if "network-config" in self._crawled_metadata: diff --git a/cloudinit/sources/DataSourceNWCS.py b/cloudinit/sources/DataSourceNWCS.py index 03a86254891..7c89713cb33 100644 --- a/cloudinit/sources/DataSourceNWCS.py +++ b/cloudinit/sources/DataSourceNWCS.py @@ -45,6 +45,11 @@ def __init__(self, sys_cfg, distro, paths): self.dsmode = sources.DSMODE_NETWORK self.metadata_full = None + def _unpickle(self, ci_pkl_version: int) -> None: + super()._unpickle(ci_pkl_version) + if not self._network_config: + self._network_config = sources.UNSET + def _get_data(self): md = self.get_metadata() @@ -95,13 +100,6 @@ def get_metadata(self): def network_config(self): LOG.debug("Attempting network configuration") - if self._network_config is None: - LOG.warning( - "Found None as cached _network_config, resetting to %s", - sources.UNSET, - ) - self._network_config = sources.UNSET - if self._network_config != sources.UNSET: return self._network_config diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index df19b764559..1420862ea21 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -317,7 +317,7 @@ def __init__(self, sys_cfg, distro: Distro, paths: Paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro self.paths = paths - self.userdata = None + self.userdata: Optional[Any] = None self.metadata: dict = {} self.userdata_raw: Optional[str] = None self.vendordata = None @@ -359,7 +359,6 @@ def _unpickle(self, ci_pkl_version: int) -> None: if not hasattr(self, "check_if_fallback_is_allowed"): setattr(self, "check_if_fallback_is_allowed", lambda: False) - if hasattr(self, "userdata") and self.userdata is not None: # If userdata stores MIME data, on < python3.6 it will be # missing the 'policy' attribute that exists on >=python3.6. @@ -376,6 +375,7 @@ def _unpickle(self, ci_pkl_version: int) -> None: ) raise DatasourceUnpickleUserDataError() from e + def __str__(self): return type_utils.obj_name(self) @@ -484,6 +484,12 @@ def get_data(self) -> bool: """ self._dirty_cache = True return_value = self._check_and_get_data() + # TODO: verify that datasource types are what they are expected to be + # each datasource uses different logic to get userdata, metadata, etc + # and then the rest of the codebase assumes the types of this data + # it would be prudent to have a type check here that warns, when the + # datatype is incorrect, rather than assuming types and throwing + # exceptions later if/when they get used incorrectly. if not return_value: return return_value self.persist_instance_data() diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py index 957433474aa..faa4aaa287a 100644 --- a/cloudinit/temp_utils.py +++ b/cloudinit/temp_utils.py @@ -10,7 +10,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) -_TMPDIR = None _ROOT_TMPDIR = "/run/cloud-init/tmp" _EXE_ROOT_TMPDIR = "/var/tmp/cloud-init" @@ -20,8 +19,6 @@ def get_tmp_ancestor(odir=None, needs_exe: bool = False): return odir if needs_exe: return _EXE_ROOT_TMPDIR - if _TMPDIR: - return _TMPDIR if os.getuid() == 0: return _ROOT_TMPDIR return os.environ.get("TMPDIR", "/tmp") @@ -53,18 +50,11 @@ def _tempfile_dir_arg(odir=None, needs_exe: bool = False): " mounted as noexec", tdir, ) - - if odir is None and not needs_exe: - global _TMPDIR - _TMPDIR = tdir - return tdir def ExtendedTemporaryFile(**kwargs): - kwargs["dir"] = _tempfile_dir_arg( - kwargs.pop("dir", None), kwargs.pop("needs_exe", False) - ) + kwargs["dir"] = _tempfile_dir_arg() fh = tempfile.NamedTemporaryFile(**kwargs) # Replace its unlink with a quiet version # that does not raise errors when the diff --git a/cloudinit/util.py b/cloudinit/util.py index 5b8f07683f6..93ba6e0dcbf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -357,8 +357,6 @@ def read_conf(fname, *, instance_data_file=None) -> Dict: config_file, repr(e), ) - if config_file is None: - return {} return load_yaml(config_file, default={}) # pyright: ignore diff --git a/pyproject.toml b/pyproject.toml index 49811eb7477..a0419d31666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ follow_imports = "silent" check_untyped_defs = true warn_redundant_casts = true warn_unused_ignores = true +warn_unreachable = true exclude = [] [[tool.mypy.overrides]] diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py index 4d29b29f832..970a2406d8a 100644 --- a/tests/integration_tests/test_upgrade.py +++ b/tests/integration_tests/test_upgrade.py @@ -62,7 +62,6 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud): source = get_validated_source(session_cloud) if not source.installs_new_version(): pytest.skip(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) - return # type checking doesn't understand that skip raises launch_kwargs = { "image_id": session_cloud.initial_image_id, } @@ -194,7 +193,6 @@ def test_subsequent_boot_of_upgraded_package(session_cloud: IntegrationCloud): pytest.fail(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) else: pytest.skip(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) - return # type checking doesn't understand that skip raises launch_kwargs = {"image_id": session_cloud.initial_image_id} diff --git a/tests/unittests/config/test_cc_apk_configure.py b/tests/unittests/config/test_cc_apk_configure.py index 544820a9335..6d09c5738ad 100644 --- a/tests/unittests/config/test_cc_apk_configure.py +++ b/tests/unittests/config/test_cc_apk_configure.py @@ -10,7 +10,7 @@ import pytest -from cloudinit import cloud, helpers, temp_utils, util +from cloudinit import cloud, helpers, util from cloudinit.config import cc_apk_configure from cloudinit.config.schema import ( SchemaValidationError, @@ -51,7 +51,7 @@ def test_no_config(self): class TestConfig(FilesystemMockingTestCase): def setUp(self): - super(TestConfig, self).setUp() + super().setUp() self.new_root = self.tmp_dir() self.new_root = self.reRoot(root=self.new_root) for dirname in ["tmp", "etc/apk"]: @@ -60,11 +60,14 @@ def setUp(self): self.name = "apk_configure" self.cloud = cloud.Cloud(None, self.paths, None, None, None) self.args = [] - temp_utils._TMPDIR = self.new_root + self.mock = mock.patch( + "cloudinit.temp_utils.get_tmp_ancestor", lambda *_: self.new_root + ) + self.mock.start() def tearDown(self): + self.mock.stop() super().tearDown() - temp_utils._TMPDIR = None @mock.patch(CC_APK + "._write_repositories_file") def test_no_repo_settings(self, m_write_repos): diff --git a/tests/unittests/config/test_cc_ntp.py b/tests/unittests/config/test_cc_ntp.py index 6f6c3360de7..ead6f5213f3 100644 --- a/tests/unittests/config/test_cc_ntp.py +++ b/tests/unittests/config/test_cc_ntp.py @@ -138,16 +138,14 @@ def test_write_ntp_config_template_uses_ntp_conf_distro_no_servers(self): servers = [] pools = ["10.0.0.1", "10.0.0.2"] (confpath, template_fn) = self._generate_template() - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): - cc_ntp.write_ntp_config_template( - "ubuntu", - servers=servers, - pools=pools, - path=confpath, - template_fn=template_fn, - template=None, - ) + cc_ntp.write_ntp_config_template( + "ubuntu", + servers=servers, + pools=pools, + path=confpath, + template_fn=template_fn, + template=None, + ) self.assertEqual( "servers []\npools ['10.0.0.1', '10.0.0.2']\n", util.load_text_file(confpath), @@ -163,16 +161,14 @@ def test_write_ntp_config_template_defaults_pools_w_empty_lists(self): pools = cc_ntp.generate_server_names(distro) servers = [] (confpath, template_fn) = self._generate_template() - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): - cc_ntp.write_ntp_config_template( - distro, - servers=servers, - pools=pools, - path=confpath, - template_fn=template_fn, - template=None, - ) + cc_ntp.write_ntp_config_template( + distro, + servers=servers, + pools=pools, + path=confpath, + template_fn=template_fn, + template=None, + ) self.assertEqual( "servers []\npools {0}\n".format(pools), util.load_text_file(confpath), @@ -672,8 +668,8 @@ def test_ntp_user_provided_config_with_template(self, m_install): } for distro in cc_ntp.distros: mycloud = self._get_cloud(distro) - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): + mock_path = "cloudinit.config.cc_ntp.temp_utils.get_tmp_ancestor" + with mock.patch(mock_path, lambda *_: self.new_root): cc_ntp.handle("notimportant", cfg, mycloud, None) self.assertEqual( "servers []\npools ['mypool.org']\n%s" % custom, @@ -712,8 +708,8 @@ def test_ntp_user_provided_config_template_only( ) confpath = ntpconfig["confpath"] m_select.return_value = ntpconfig - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): + mock_path = "cloudinit.config.cc_ntp.temp_utils.get_tmp_ancestor" + with mock.patch(mock_path, lambda *_: self.new_root): cc_ntp.handle("notimportant", {"ntp": cfg}, mycloud, None) self.assertEqual( "servers []\npools ['mypool.org']\n%s" % custom, diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 2ea42b62d5b..184857583fb 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -1767,29 +1767,6 @@ def test_annotated_cloudconfig_file_no_schema_errors(self): schema_errors=[], ) - def test_annotated_cloudconfig_file_with_non_dict_cloud_config(self): - """Error when empty non-dict cloud-config is provided. - - OurJSON validation when user-data is None type generates a bunch - schema validation errors of the format: - ('', "None is not of type 'object'"). Ignore those symptoms and - report the general problem instead. - """ - content = "\n\n\n" - expected = "\n".join( - [ - content, - "# Errors: -------------", - "# E1: Cloud-config is not a YAML dict.\n\n", - ] - ) - assert expected == annotated_cloudconfig_file( - None, - content, - schemamarks={}, - schema_errors=[SchemaProblem("", "None is not of type 'object'")], - ) - def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self): """With schema_errors, error lines are annotated and a footer added.""" content = dedent( diff --git a/tests/unittests/test_temp_utils.py b/tests/unittests/test_temp_utils.py index d47852d66d8..6cbc372823c 100644 --- a/tests/unittests/test_temp_utils.py +++ b/tests/unittests/test_temp_utils.py @@ -25,7 +25,6 @@ def fake_mkdtemp(*args, **kwargs): { "os.getuid": 1000, "tempfile.mkdtemp": {"side_effect": fake_mkdtemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkdtemp, @@ -46,7 +45,6 @@ def fake_mkdtemp(*args, **kwargs): { "os.getuid": 1000, "tempfile.mkdtemp": {"side_effect": fake_mkdtemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, "util.has_mount_opt": True, }, @@ -69,7 +67,6 @@ def fake_mkdtemp(*args, **kwargs): { "os.getuid": 0, "tempfile.mkdtemp": {"side_effect": fake_mkdtemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkdtemp, @@ -90,7 +87,6 @@ def fake_mkstemp(*args, **kwargs): { "os.getuid": 1000, "tempfile.mkstemp": {"side_effect": fake_mkstemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkstemp, @@ -111,7 +107,6 @@ def fake_mkstemp(*args, **kwargs): { "os.getuid": 0, "tempfile.mkstemp": {"side_effect": fake_mkstemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkstemp, From 97146b57f85e4faafc232b61e9ca0aa3bb0a2812 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 19:41:46 -0600 Subject: [PATCH 35/75] chore(typing): Remove type ignores and casts (#5350) Python's provides type annotation features which allow validating types throughout the code base. Use, rather than ignore, these features. Note that since a dict's .get() may return None, using this method when the only possible return value is a literal will confuse mypy. --- cloudinit/distros/__init__.py | 3 ++- cloudinit/distros/bsd.py | 2 +- cloudinit/net/netplan.py | 25 +++++++++++++---------- cloudinit/sources/__init__.py | 1 - cloudinit/url_helper.py | 4 ++-- cloudinit/util.py | 9 ++++---- tests/unittests/conftest.py | 2 +- tests/unittests/net/test_network_state.py | 2 +- tests/unittests/test_log.py | 7 ++----- 9 files changed, 28 insertions(+), 27 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 789af4a7d69..3c956408090 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,6 +50,7 @@ from cloudinit.distros.parsers import hosts from cloudinit.features import ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES from cloudinit.net import activators, dhcp, renderers +from cloudinit.net.netops import NetOps from cloudinit.net.network_state import parse_net_config_data from cloudinit.net.renderer import Renderer @@ -142,7 +143,7 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): # This is used by self.shutdown_command(), and can be overridden in # subclasses shutdown_options_map = {"halt": "-H", "poweroff": "-P", "reboot": "-r"} - net_ops = iproute2.Iproute2 + net_ops: Type[NetOps] = iproute2.Iproute2 _ci_pkl_version = 1 prefer_fqdn = False diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 5bef9203c3d..25b374ba3bc 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -28,7 +28,7 @@ class BSD(distros.Distro): # There is no update/upgrade on OpenBSD pkg_cmd_update_prefix: Optional[List[str]] = None pkg_cmd_upgrade_prefix: Optional[List[str]] = None - net_ops = bsd_netops.BsdNetOps # type: ignore + net_ops = bsd_netops.BsdNetOps def __init__(self, name, cfg, paths): super().__init__(name, cfg, paths) diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 4e1a01ae94b..532442dcb83 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -6,7 +6,7 @@ import os import textwrap from tempfile import SpooledTemporaryFile -from typing import Callable, List, Optional, cast +from typing import Callable, List, Optional from cloudinit import features, safeyaml, subp, util from cloudinit.net import ( @@ -454,10 +454,7 @@ def _render_content(self, network_state: NetworkState) -> str: bond_config = {} # extract bond params and drop the bond_ prefix as it's # redundant in v2 yaml format - v2_bond_map = cast(dict, NET_CONFIG_TO_V2.get("bond")) - # Previous cast is needed to help mypy to know that the key is - # present in `NET_CONFIG_TO_V2`. This could probably be removed - # by using `Literal` when supported. + v2_bond_map = NET_CONFIG_TO_V2["bond"] for match in ["bond_", "bond-"]: bond_params = _get_params_dict_by_match(ifcfg, match) for param, value in bond_params.items(): @@ -478,9 +475,18 @@ def _render_content(self, network_state: NetworkState) -> str: elif if_type == "bridge": # required_keys = ['name', 'bridge_ports'] + # + # Rather than raise an exception on `sorted(None)`, log a + # warning and skip this interface when invalid configuration is + # received. bridge_ports = ifcfg.get("bridge_ports") - # mypy wrong error. `copy(None)` is supported: - ports = sorted(copy.copy(bridge_ports)) # type: ignore + if bridge_ports is None: + LOG.warning( + "Invalid config. The key", + f"'bridge_ports' is required in {config}.", + ) + continue + ports = sorted(copy.copy(bridge_ports)) bridge: dict = { "interfaces": ports, } @@ -492,10 +498,7 @@ def _render_content(self, network_state: NetworkState) -> str: # v2 yaml uses different names for the keys # and at least one value format change - v2_bridge_map = cast(dict, NET_CONFIG_TO_V2.get("bridge")) - # Previous cast is needed to help mypy to know that the key is - # present in `NET_CONFIG_TO_V2`. This could probably be removed - # by using `Literal` when supported. + v2_bridge_map = NET_CONFIG_TO_V2["bridge"] for param, value in params.items(): newname = v2_bridge_map.get(param) if newname is None: diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 1420862ea21..27c37ee1e13 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -375,7 +375,6 @@ def _unpickle(self, ci_pkl_version: int) -> None: ) raise DatasourceUnpickleUserDataError() from e - def __str__(self): return type_utils.obj_name(self) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 777aa6a70e3..d409e322858 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -560,7 +560,7 @@ def dual_stack( """ return_result = None returned_address = None - last_exception = None + last_exception: Optional[BaseException] = None exceptions = [] is_done = threading.Event() @@ -620,7 +620,7 @@ def dual_stack( "Timed out waiting for addresses: %s, " "exception(s) raised while waiting: %s", " ".join(addresses), - " ".join(exceptions), # type: ignore + " ".join(map(str, exceptions)), ) finally: executor.shutdown(wait=False) diff --git a/cloudinit/util.py b/cloudinit/util.py index 93ba6e0dcbf..98dd66d59fc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3251,8 +3251,8 @@ def deprecate( Note: uses keyword-only arguments to improve legibility """ - if not hasattr(deprecate, "_log"): - deprecate._log = set() # type: ignore + if not hasattr(deprecate, "log"): + setattr(deprecate, "log", set()) message = extra_message or "" dedup = hash(deprecated + message + deprecated_version + str(schedule)) version = Version.from_str(deprecated_version) @@ -3270,8 +3270,9 @@ def deprecate( level = log.DEPRECATED else: level = logging.WARN - if not skip_log and dedup not in deprecate._log: # type: ignore - deprecate._log.add(dedup) # type: ignore + log_cache = getattr(deprecate, "log") + if not skip_log and dedup not in log_cache: + log_cache.add(dedup) LOG.log(level, deprecate_msg) return DeprecationLog(level, deprecate_msg) diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index e912c90ad28..e0baa63b99b 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -152,7 +152,7 @@ def clear_deprecation_log(): # Since deprecations are de-duped, the existance (or non-existance) of # a deprecation warning in a previous test can cause the next test to # fail. - util.deprecate._log = set() + setattr(util.deprecate, "log", set()) PYTEST_VERSION_TUPLE = tuple(map(int, pytest.__version__.split("."))) diff --git a/tests/unittests/net/test_network_state.py b/tests/unittests/net/test_network_state.py index 19b1b1b30e2..eaad90dc8e1 100644 --- a/tests/unittests/net/test_network_state.py +++ b/tests/unittests/net/test_network_state.py @@ -215,7 +215,7 @@ def test_v2_warns_deprecated_gateways( In netplan targets we perform a passthrough and the warning is not needed. """ - util.deprecate._log = set() # type: ignore + util.deprecate.__dict__["log"] = set() ncfg = yaml.safe_load( cfg.format( gateway4="gateway4: 10.54.0.1", diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 78b53f5a9f9..87996310349 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -63,8 +63,7 @@ def test_logger_uses_gmtime(self): class TestDeprecatedLogs: def test_deprecated_log_level(self, caplog): - logger = logging.getLogger() - logger.deprecated("deprecated message") + logging.getLogger().deprecated("deprecated message") assert "DEPRECATED" == caplog.records[0].levelname assert "deprecated message" in caplog.text @@ -116,7 +115,6 @@ def test_deprecate_log_level_based_on_features( ) def test_log_deduplication(self, caplog): - log.define_deprecation_logger() util.deprecate( deprecated="stuff", deprecated_version="19.1", @@ -139,6 +137,5 @@ def test_log_deduplication(self, caplog): def test_logger_prints_to_stderr(capsys): message = "to stdout" log.setup_basic_logging() - LOG = logging.getLogger() - LOG.warning(message) + logging.getLogger().warning(message) assert message in capsys.readouterr().err From e4c05e80b8ba7f1449674ffd0240dff435ebbdf2 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 3 Jun 2024 15:05:31 -0600 Subject: [PATCH 36/75] fix(tox): Update tox.ini (#5350) Enable positional arguments for various environments. Add missing deps for tip-mypy. The -tip envs were broken when a non-tip env existed. In these cases, {[testenv:mypy]commands} resolved to the non-tip environment. Fix it. --- tox.ini | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/tox.ini b/tox.ini index 37e61f120c4..52d6505e9ad 100644 --- a/tox.ini +++ b/tox.ini @@ -45,31 +45,33 @@ version = cloudinit/config/schemas/versions.schema.cloud-config.json [testenv:ruff] deps = ruff=={[format_deps]ruff} -commands = {envpython} -m ruff check . +commands = {envpython} -m ruff check {posargs:.} [testenv:pylint] deps = pylint=={[format_deps]pylint} -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -commands = {envpython} -m pylint {posargs:cloudinit/ tests/ tools/ conftest.py setup.py} +commands = {envpython} -m pylint {posargs:.} [testenv:black] deps = black=={[format_deps]black} -commands = {envpython} -m black . --check +commands = {envpython} -m black --check {posargs:.} [testenv:isort] deps = isort=={[format_deps]isort} -commands = {envpython} -m isort . --check-only --diff +commands = {envpython} -m isort --check-only --diff {posargs:.} [testenv:mypy] deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt hypothesis=={[format_deps]hypothesis} hypothesis_jsonschema=={[format_deps]hypothesis_jsonschema} mypy=={[format_deps]mypy} - pytest=={[format_deps]pytest} types-jsonschema=={[format_deps]types-jsonschema} types-Jinja2=={[format_deps]types-Jinja2} types-passlib=={[format_deps]types-passlib} @@ -78,7 +80,7 @@ deps = types-requests=={[format_deps]types-requests} types-setuptools=={[format_deps]types-setuptools} typing-extensions=={[format_deps]typing-extensions} -commands = {envpython} -m mypy cloudinit/ tests/ tools/ +commands = {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:check_format] deps = @@ -89,16 +91,18 @@ deps = isort=={[format_deps]isort} mypy=={[format_deps]mypy} pylint=={[format_deps]pylint} - pytest=={[format_deps]pytest} types-jsonschema=={[format_deps]types-jsonschema} + types-Jinja2=={[format_deps]types-Jinja2} types-oauthlib=={[format_deps]types-oauthlib} types-passlib=={[format_deps]types-passlib} types-pyyaml=={[format_deps]types-PyYAML} + types-oauthlib=={[format_deps]types-oauthlib} types-requests=={[format_deps]types-requests} types-setuptools=={[format_deps]types-setuptools} typing-extensions=={[format_deps]typing-extensions} -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt commands = {[testenv:black]commands} {[testenv:ruff]commands} @@ -115,15 +119,17 @@ deps = isort mypy pylint - pytest types-jsonschema + types-Jinja2 types-oauthlib + types-passlib types-pyyaml + types-oauthlib types-requests types-setuptools - typing-extensions -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt commands = {[testenv:check_format]commands} @@ -151,7 +157,8 @@ commands = {envpython} -m pytest \ -vvvv --showlocals \ --durations 10 \ -m "not hypothesis_slow" \ - {posargs:--cov=cloudinit --cov-branch tests/unittests} + --cov=cloudinit --cov-branch \ + {posargs:tests/unittests} # experimental [testenv:py3-fast] @@ -168,15 +175,16 @@ deps = -r{toxinidir}/test-requirements.txt commands = {envpython} -m pytest \ -m hypothesis_slow \ - {posargs:--hypothesis-show-statistics tests/unittests} + --hypothesis-show-statistics \ + {posargs:tests/unittests} #commands = {envpython} -X tracemalloc=40 -Werror::ResourceWarning:cloudinit -m pytest \ [testenv:py3-leak] deps = {[testenv:py3]deps} commands = {envpython} -X tracemalloc=40 -Wall -m pytest \ --durations 10 \ - {posargs:--cov=cloudinit --cov-branch \ - tests/unittests} + --cov=cloudinit --cov-branch \ + {posargs:tests/unittests} [testenv:lowest-supported] @@ -242,16 +250,24 @@ commands = {[testenv:ruff]commands} [testenv:tip-mypy] deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt hypothesis hypothesis_jsonschema mypy pytest + types-Jinja2 types-jsonschema types-oauthlib types-PyYAML + types-passlib + types-pyyaml + types-oauthlib types-requests types-setuptools -commands = {[testenv:mypy]commands} + typing-extensions +commands = {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:tip-pylint] deps = @@ -260,7 +276,8 @@ deps = # test-requirements -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -commands = {[testenv:pylint]commands} +commands = {envpython} -m pylint {posargs:.} + [testenv:tip-black] deps = black From 73a5c512a5e1bd9dcdbf6741b6b97d3cfc644c3b Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 3 Jun 2024 15:18:18 -0600 Subject: [PATCH 37/75] chore: Auto-format network jsonschema in ci (#5350) Fix the current network v2 schema as well. --- .../schemas/schema-network-config-v2.json | 112 ++++++++++-------- tox.ini | 4 + 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/cloudinit/config/schemas/schema-network-config-v2.json b/cloudinit/config/schemas/schema-network-config-v2.json index 64c29f5ba16..0a3741d65ac 100644 --- a/cloudinit/config/schemas/schema-network-config-v2.json +++ b/cloudinit/config/schemas/schema-network-config-v2.json @@ -10,43 +10,43 @@ ] }, "dhcp-overrides": { - "type": "object", - "description": "DHCP behaviour overrides. Overrides will only have an effect if the corresponding DHCP type is enabled.", - "additionalProperties": false, - "properties": { - "hostname": { - "type": "string", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "route-metric": { - "type": "integer", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "send-hostname": { - "type": "boolean", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "use-dns": { - "type": "boolean" - }, - "use-domains": { - "type": "string" - }, - "use-hostname": { - "type": "boolean" - }, - "use-mtu": { - "type": "boolean", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "use-ntp": { - "type": "boolean" - }, - "use-routes": { - "type": "boolean", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - } + "type": "object", + "description": "DHCP behaviour overrides. Overrides will only have an effect if the corresponding DHCP type is enabled.", + "additionalProperties": false, + "properties": { + "hostname": { + "type": "string", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "route-metric": { + "type": "integer", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "send-hostname": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "use-dns": { + "type": "boolean" + }, + "use-domains": { + "type": "string" + }, + "use-hostname": { + "type": "boolean" + }, + "use-mtu": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "use-ntp": { + "type": "boolean" + }, + "use-routes": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." } + } }, "gateway": { "type": "string", @@ -56,17 +56,33 @@ "type": "object", "properties": { "renderer": { - "$ref": "#/$defs/renderer" + "$ref": "#/$defs/renderer" }, "dhcp4": { - "type": ["boolean", "string"], + "type": [ + "boolean", + "string" + ], "description": "Enable DHCP for IPv4. Off by default.", - "enum": ["yes", "no", true, false] + "enum": [ + "yes", + "no", + true, + false + ] }, "dhcp6": { - "type": ["boolean", "string"], + "type": [ + "boolean", + "string" + ], "description": "Enable DHCP for IPv6. Off by default.", - "enum": ["yes", "no", true, false] + "enum": [ + "yes", + "no", + true, + false + ] }, "dhcp4-overrides": { "$ref": "#/$defs/dhcp-overrides" @@ -89,7 +105,7 @@ }, "mtu": { "type": "integer", - "description": "The MTU key represents a device’s Maximum Transmission Unit, the largest size packet or frame, specified in octets (eight-bit bytes), that can be sent in a packet- or frame-based network. Specifying mtu is optional." + "description": "The MTU key represents a device\u2019s Maximum Transmission Unit, the largest size packet or frame, specified in octets (eight-bit bytes), that can be sent in a packet- or frame-based network. Specifying mtu is optional." }, "nameservers": { "type": "object", @@ -152,7 +168,7 @@ }, "macaddress": { "type": "string", - "description": "Device’s MAC address in the form xx:xx:xx:xx:xx:xx. Globs are not allowed. Letters must be lowercase." + "description": "Device\u2019s MAC address in the form xx:xx:xx:xx:xx:xx. Globs are not allowed. Letters must be lowercase." }, "driver": { "type": "string", @@ -162,7 +178,7 @@ }, "set-name": { "type": "string", - "description": "When matching on unique properties such as path or MAC, or with additional assumptions such as ''there will only ever be one wifi device'', match rules can be written so that they only match one device. Then this property can be used to give that device a more specific/desirable/nicer name than the default from udev’s ifnames. Any additional device that satisfies the match rules will then fail to get renamed and keep the original kernel name (and dmesg will show an error)." + "description": "When matching on unique properties such as path or MAC, or with additional assumptions such as ''there will only ever be one wifi device'', match rules can be written so that they only match one device. Then this property can be used to give that device a more specific/desirable/nicer name than the default from udev\u2019s ifnames. Any additional device that satisfies the match rules will then fail to get renamed and keep the original kernel name (and dmesg will show an error)." }, "wakeonlan": { "type": "boolean", @@ -257,7 +273,7 @@ "description": "Configure how ARP replies are to be validated when using ARP link monitoring.", "enum": [ "none", - "active", + "active", "backup", "all" ] @@ -356,7 +372,7 @@ }, "stp": { "type": "boolean", - "description": "Define whether the bridge should use Spanning Tree Protocol. The default value is “true”, which means that Spanning Tree should be used." + "description": "Define whether the bridge should use Spanning Tree Protocol. The default value is \u201ctrue\u201d, which means that Spanning Tree should be used." } } } @@ -393,13 +409,13 @@ ] }, "renderer": { - "$ref": "#/$defs/renderer" + "$ref": "#/$defs/renderer" }, "ethernets": { "type": "object", "additionalProperties": { "$ref": "#/$defs/mapping_physical" - } + } }, "bonds": { "type": "object", diff --git a/tox.ini b/tox.ini index 52d6505e9ad..a43ef53f3c2 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,8 @@ typing-extensions==4.1.1 [files] schema = cloudinit/config/schemas/schema-cloud-config-v1.json version = cloudinit/config/schemas/versions.schema.cloud-config.json +network_v1 = cloudinit/config/schemas/schema-network-config-v1.json +network_v2 = cloudinit/config/schemas/schema-network-config-v2.json [testenv:ruff] deps = @@ -142,6 +144,8 @@ commands = {envpython} -m black . {envpython} -m json.tool --indent 2 {[files]schema} {[files]schema} {envpython} -m json.tool --indent 2 {[files]version} {[files]version} + {envpython} -m json.tool --indent 2 {[files]network_v1} {[files]network_v1} + {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} [testenv:do_format_tip] deps = From 8ad0bbaf6a35020eb78e002a58e17d64fc71a757 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 10 Jun 2024 17:52:35 -0600 Subject: [PATCH 38/75] type: Add stub types for network v1/v2 config (#5350) This doesn't add anything useful on its own, but it defines a custom type which can easily be overridden. See GH-5398 for more details. --- cloudinit/config/__init__.py | 2 ++ cloudinit/stages.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index e5670257208..01da83fcffb 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -1 +1,3 @@ Config = dict +Netv1 = dict +Netv2 = dict diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 894eeac5960..52876e72434 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -11,7 +11,7 @@ import sys from collections import namedtuple from contextlib import suppress -from typing import Dict, Iterable, List, Optional, Set +from typing import Dict, Iterable, List, Optional, Set, Tuple, Union from cloudinit import ( atomic_helper, @@ -26,6 +26,7 @@ type_utils, util, ) +from cloudinit.config import Netv1, Netv2 from cloudinit.event import EventScope, EventType, userdata_to_events # Default handlers (used if not overridden) @@ -944,7 +945,7 @@ def _consume_userdata(self, frequency=PER_INSTANCE): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) - def _get_network_key_contents(self, cfg) -> dict: + def _get_network_key_contents(self, cfg) -> Union[Netv1, Netv2, None]: """ Network configuration can be passed as a dict under a "network" key, or optionally at the top level. In both cases, return the config. @@ -953,7 +954,9 @@ def _get_network_key_contents(self, cfg) -> dict: return cfg["network"] return cfg - def _find_networking_config(self): + def _find_networking_config( + self, + ) -> Tuple[Union[Netv1, Netv2, None], Union[NetworkConfigSource, str]]: disable_file = os.path.join( self.paths.get_cpath("data"), "upgraded-network" ) @@ -978,7 +981,9 @@ def _find_networking_config(self): order = sources.DataSource.network_config_sources for cfg_source in order: if not isinstance(cfg_source, NetworkConfigSource): - LOG.warning( + # This won't happen in the cloud-init codebase, but out-of-tree + # datasources might have an invalid type that mypy cannot know. + LOG.warning( # type: ignore "data source specifies an invalid network cfg_source: %s", cfg_source, ) From c95ec1a4f8bf3dd9c2c34c7f3daea6ffca125dad Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 20 Jun 2024 20:11:18 -0600 Subject: [PATCH 39/75] chore(mypy): Drop unused missing import exclusions (#5350) --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0419d31666..7408488f975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ exclude = [] module = [ "apport.*", "BaseHTTPServer", - "cloudinit.feature_overrides", "configobj", "debconf", "httplib", @@ -33,7 +32,6 @@ module = [ "paramiko.*", "pip.*", "pycloudlib.*", - "responses", "serial", "tests.integration_tests.user_settings", "uaclient.*", From 2bca69a9a41e2f1e3e822cedd0860d2536e9cda3 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 1 Jul 2024 11:48:31 -0600 Subject: [PATCH 40/75] fix(test): Fix ip printing for non-lxd instances (#5350) This also satisfies mypy via type narrowing, which hasattr() does not accomplish. --- tests/integration_tests/instances.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index bff28da7605..da7c4fb9ae3 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -8,6 +8,7 @@ from tempfile import NamedTemporaryFile from typing import Union +from pycloudlib.lxd.instance import LXDInstance from pycloudlib.gce.instance import GceInstance from pycloudlib.instance import BaseInstance from pycloudlib.result import Result @@ -289,7 +290,7 @@ def ip(self) -> str: try: # in some cases that ssh is not used, an address is not assigned if ( - hasattr(self.instance, "execute_via_ssh") + isinstance(self.instance, LXDInstance) and self.instance.execute_via_ssh ): self._ip = self.instance.ip From 291aabeb74c25b8af05449b481db197c36d8788a Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 1 Jul 2024 11:50:04 -0600 Subject: [PATCH 41/75] fix(test): Fix pycloudlib types in integration tests (#5350) --- tests/integration_tests/clouds.py | 31 ++++++++++--------- tests/integration_tests/instances.py | 2 +- .../modules/test_apt_functionality.py | 4 +-- .../modules/test_combined.py | 8 +++++ 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index b7786edb280..82acde409e2 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -21,7 +21,7 @@ Openstack, Qemu, ) -from pycloudlib.cloud import BaseCloud, ImageType +from pycloudlib.cloud import ImageType from pycloudlib.ec2.instance import EC2Instance from pycloudlib.lxd.cloud import _BaseLXD from pycloudlib.lxd.instance import BaseInstance, LXDInstance @@ -55,7 +55,6 @@ def _get_ubuntu_series() -> list: class IntegrationCloud(ABC): datasource: str - cloud_instance: BaseCloud def __init__( self, @@ -64,7 +63,7 @@ def __init__( ): self._image_type = image_type self.settings = settings - self.cloud_instance: BaseCloud = self._get_cloud_instance() + self.cloud_instance = self._get_cloud_instance() self.initial_image_id = self._get_initial_image() self.snapshot_id = None @@ -183,7 +182,7 @@ def snapshot(self, instance): def delete_snapshot(self): if self.snapshot_id: - if self.settings.KEEP_IMAGE: + if self.settings.KEEP_IMAGE: # type: ignore log.info( "NOT deleting snapshot image created for this testrun " "because KEEP_IMAGE is True: %s", @@ -200,7 +199,7 @@ def delete_snapshot(self): class Ec2Cloud(IntegrationCloud): datasource = "ec2" - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> EC2: return EC2(tag="ec2-integration-test") def _get_initial_image(self, **kwargs) -> str: @@ -228,7 +227,7 @@ def _perform_launch( class GceCloud(IntegrationCloud): datasource = "gce" - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> GCE: return GCE( tag="gce-integration-test", ) @@ -243,7 +242,7 @@ class AzureCloud(IntegrationCloud): datasource = "azure" cloud_instance: Azure - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> Azure: return Azure(tag="azure-integration-test") def _get_initial_image(self, **kwargs) -> str: @@ -265,7 +264,7 @@ def destroy(self): class OciCloud(IntegrationCloud): datasource = "oci" - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> OCI: return OCI( tag="oci-integration-test", ) @@ -276,11 +275,7 @@ class _LxdIntegrationCloud(IntegrationCloud): instance_tag: str cloud_instance: _BaseLXD - def _get_cloud_instance(self): - return self.pycloudlib_instance_cls(tag=self.instance_tag) - - @staticmethod - def _get_or_set_profile_list(release): + def _get_or_set_profile_list(self, release): return None @staticmethod @@ -355,15 +350,21 @@ class LxdContainerCloud(_LxdIntegrationCloud): pycloudlib_instance_cls = LXDContainer instance_tag = "lxd-container-integration-test" + def _get_cloud_instance(self) -> LXDContainer: + return self.pycloudlib_instance_cls(tag=self.instance_tag) + class LxdVmCloud(_LxdIntegrationCloud): datasource = "lxd_vm" cloud_instance: LXDVirtualMachine pycloudlib_instance_cls = LXDVirtualMachine instance_tag = "lxd-vm-integration-test" - _profile_list = None + _profile_list: list = [] - def _get_or_set_profile_list(self, release): + def _get_cloud_instance(self) -> LXDVirtualMachine: + return self.pycloudlib_instance_cls(tag=self.instance_tag) + + def _get_or_set_profile_list(self, release) -> list: if self._profile_list: return self._profile_list self._profile_list = self.cloud_instance.build_necessary_profiles( diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index da7c4fb9ae3..32281756cd1 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -8,9 +8,9 @@ from tempfile import NamedTemporaryFile from typing import Union -from pycloudlib.lxd.instance import LXDInstance from pycloudlib.gce.instance import GceInstance from pycloudlib.instance import BaseInstance +from pycloudlib.lxd.instance import LXDInstance from pycloudlib.result import Result from tests.helpers import cloud_init_project_dir diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index b86e217556e..2af9e590ce0 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -152,10 +152,10 @@ def get_keys(self, class_client: IntegrationInstance): keys = class_client.execute(list_cmd + cc_apt_configure.APT_LOCAL_KEYS) files = class_client.execute( "ls " + cc_apt_configure.APT_TRUSTED_GPG_DIR - ) + ).stdout for file in files.split(): path = cc_apt_configure.APT_TRUSTED_GPG_DIR + file - keys += class_client.execute(list_cmd + path) or "" + keys += class_client.execute(list_cmd + path).stdout class_client.execute("gpgconf --homedir /root/tmpdir --kill all") return keys diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 4a7c2dfc9e0..0b25a0fbe1a 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -13,6 +13,8 @@ from pathlib import Path import pytest +from pycloudlib.ec2.instance import EC2Instance +from pycloudlib.gce.instance import GceInstance import cloudinit.config from cloudinit.util import is_true, should_log_deprecation @@ -473,6 +475,9 @@ def test_instance_json_ec2(self, class_client: IntegrationInstance): "/run/cloud-init/cloud-id-aws" ) assert v1_data["subplatform"].startswith("metadata") + + # type narrow since availability_zone is not a BaseInstance attribute + assert isinstance(client.instance, EC2Instance) assert ( v1_data["availability_zone"] == client.instance.availability_zone ) @@ -495,6 +500,9 @@ def test_instance_json_gce(self, class_client: IntegrationInstance): "/run/cloud-init/cloud-id-gce" ) assert v1_data["subplatform"].startswith("metadata") + # type narrow since zone and instance_id are not BaseInstance + # attributes + assert isinstance(client.instance, GceInstance) assert v1_data["availability_zone"] == client.instance.zone assert v1_data["instance_id"] == client.instance.instance_id assert v1_data["local_hostname"] == client.instance.name From b0d6c7ddc4aa6adad064464e87ee0431037d57bd Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 2 Jul 2024 10:56:29 -0500 Subject: [PATCH 42/75] test: Add jsonschema guard in test_cc_ubuntu_pro.py (#5479) jsonschema is still a soft dependency, so don't fail it not present. --- tests/unittests/config/test_cc_ubuntu_pro.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unittests/config/test_cc_ubuntu_pro.py b/tests/unittests/config/test_cc_ubuntu_pro.py index f68a688f9fc..df47e7ae41e 100644 --- a/tests/unittests/config/test_cc_ubuntu_pro.py +++ b/tests/unittests/config/test_cc_ubuntu_pro.py @@ -5,7 +5,6 @@ import sys from collections import namedtuple -import jsonschema import pytest from cloudinit import subp @@ -28,6 +27,11 @@ from tests.unittests.helpers import does_not_raise, mock, skipUnlessJsonSchema from tests.unittests.util import get_cloud +try: + import jsonschema +except ImportError: + jsonschema = None # type: ignore + # Module path used in mocks MPATH = "cloudinit.config.cc_ubuntu_pro" From 0a7036ed748387e5855e301e194c77b3830f14a8 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 07:12:31 -0600 Subject: [PATCH 43/75] fix: correct deprecated_version=22.2 for users.sudo Update test to reduce expected deprecation count when below deprecation boundary. changed_version messages are still logged. --- cloudinit/distros/__init__.py | 2 +- tests/integration_tests/modules/test_combined.py | 9 +++++++-- tests/unittests/distros/test_create_users.py | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 3c956408090..4557d4320ee 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -847,7 +847,7 @@ def create_user(self, name, **kwargs): util.deprecate( deprecated=f"The value of 'false' in user {name}'s " "'sudo' config", - deprecated_version="22.3", + deprecated_version="22.2", extra_message="Use 'null' instead.", ) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 0b25a0fbe1a..0bf1b3d49e8 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -136,18 +136,23 @@ def test_deprecated_message(self, class_client: IntegrationInstance): version_boundary = get_feature_flag_value( class_client, "DEPRECATION_INFO_BOUNDARY" ) - # the deprecation_version is 22.2 in schema for apt_* keys in + # the changed_version is 22.2 in schema for user.sudo key in # user-data. Pass 22.2 in against the client's version_boundary. if should_log_deprecation("22.2", version_boundary): log_level = "DEPRECATED" + deprecation_count = 2 else: + # Expect the distros deprecated call to be redacted. + # jsonschema still emits deprecation log due to changed_version + # instead of deprecated_version log_level = "INFO" + deprecation_count = 1 assert ( f"[{log_level}]: The value of 'false' in user craig's 'sudo'" " config is deprecated" in log ) - assert 2 == log.count("DEPRECATE") + assert deprecation_count == log.count("DEPRECATE") def test_ntp_with_apt(self, class_client: IntegrationInstance): """LP #1628337. diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 893b47060d4..8fa7f0cc092 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -181,15 +181,15 @@ def test_explicit_sudo_false(self, m_subp, dist, caplog): expected_levels = ( ["WARNING", "DEPRECATED"] if should_log_deprecation( - "22.3", features.DEPRECATION_INFO_BOUNDARY + "22.2", features.DEPRECATION_INFO_BOUNDARY ) else ["INFO"] ) assert caplog.records[1].levelname in expected_levels assert ( "The value of 'false' in user foo_user's 'sudo' " - "config is deprecated in 22.3 and scheduled to be removed" - " in 27.3. Use 'null' instead." + "config is deprecated in 22.2 and scheduled to be removed" + " in 27.2. Use 'null' instead." ) in caplog.text def test_explicit_sudo_none(self, m_subp, dist, caplog): From 0a4c43d47cebd25c5f6d2f7631752349e956e32e Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 2 Jul 2024 13:22:28 -0500 Subject: [PATCH 44/75] test: Fix no default user in test_status.py (#5478) Also use ca_certs instead of ca-certs deprecated key --- tests/integration_tests/cmd/test_status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index 84d9b694f63..23509c57cef 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -66,7 +66,8 @@ def test_wait_when_no_datasource(session_cloud: IntegrationCloud, setup_image): users: - name: something ssh-authorized-keys: ["something"] -ca-certs: + - default +ca_certs: invalid_key: true """ From fd63297550a3c153b2f92720100a1fa5d266144c Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 2 Jul 2024 20:44:08 -0500 Subject: [PATCH 45/75] Release 24.2 (#5481) Bump the version in cloudinit/version.py to 24.2 and update ChangeLog. --- ChangeLog | 364 +++++++++++++++++++++++++++++++++++++++++++ cloudinit/version.py | 2 +- 2 files changed, 365 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 880d134750f..260c460df46 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,367 @@ +24.2 + - test: Fix no default user in test_status.py (#5478) + - fix: correct deprecated_version=22.2 for users.sudo + - test: Add jsonschema guard in test_cc_ubuntu_pro.py (#5479) + - fix(test): Fix pycloudlib types in integration tests (#5350) + - fix(test): Fix ip printing for non-lxd instances (#5350) + - chore(mypy): Drop unused missing import exclusions (#5350) + - type: Add stub types for network v1/v2 config (#5350) + - chore: Auto-format network jsonschema in ci (#5350) + - fix(tox): Update tox.ini (#5350) + - chore(typing): Remove type ignores and casts (#5350) + - refactor(typing): Remove unused code paths (#5350) + - fix(typing): Add / update type annotations (#5350) + - fix(typing): Remove type annotation for unused variable (#5350) + - fix(typing): Remove invalid type annotations (#5350) + - ci(mypy): Set default follow_imports value (#5350) + - test: Update integration tests to pass on focal (#5476) + - tests: update ubuntu_pro test to account for info-level deprecations + (#5475) + - tests: update nocloud deprecation test for boundary version (#5474) + - fix(rh_subscription): add string type to org (#5453) + - tests: integration tests aware of features.DEPRECATION_INFO_BOUNDARY + - tests: update keyserver PPA key fur curtin-dev (#5472) + - test: Fix deprecation test failures (#5466) + - chore: fix schema.py formatting (#5465) + - fix: dont double-log deprecated INFOs (#5465) + - fix(test): Mock version boundary (#5464) + - fix(schema): Don't report changed keys as deprecated (#5464) + - test: fix unit test openstack vlan mac_address (#5367) + - fix: Ensure properties for bonded interfaces are properly translated + (#5367) [Curt Moore] + - fix(schema): permit deprecated hyphenated keys under users key (#5456) + - fix: Do not add the vlan_mac_address field into the VLAN object (#5365) + [Curt Moore] + - doc(refactor): Convert module docs to new system (#5427) [Sally] + - test: Add unit tests for features.DEPRECATION_INFO_BOUNDARY (#5411) + - feat: Add deprecation boundary support to schema validator (#5411) + - feat: Add deprecation boundary to logger (#5411) + - fix: Gracefully handle missing files (#5397) [Curt Moore] + - test(openstack): Test bond mac address (#5369) + - fix(openstack): Fix bond mac_address (#5369) [Curt Moore] + - test: Add ds-identify integration test coverage (#5394) + - chore(cmdline): Update comments (#5458) + - fix: Add get_connection_with_tls_context() for requests 2.32.2+ (#5435) + [eaglegai] + - fix(net): klibc ipconfig PROTO compatibility (#5437) + [Alexsander de Souza] (LP: #2065787) + - Support metalink in yum repository config (#5444) [Ani Sinha] + - tests: hard-code curtin-dev ppa instead of canonical-kernel-team (#5450) + - ci: PR update checklist GH- anchors to align w/ later template (#5449) + - test: update validate error message in test_networking (#5436) + - ci: Add PR checklist (#5446) + - chore: fix W0105 in t/u/s/h/test_netlink.py (#5409) + - chore(pyproject.toml): migrate to booleans (#5409) + - typing: add check_untyped_defs (#5409) + - fix(openstack): Append interface / scope_id for IPv6 link-local metadata + address (#5419) [Christian Rohmann] + - test: Update validation error in test_cli.py test (#5430) + - test: Update schema validation error in integration test (#5429) + - test: bump pycloudlib to get azure oracular images (#5428) + - fix(azure): fix discrepancy for monotonic() vs time() (#5420) + [Chris Patterson] + - fix(pytest): Fix broken pytest gdb flag (#5415) + - fix: Use monotonic time (#5423) + - docs: Remove mention of resolv.conf (#5424) + - perf(netplan): Improve network v1 -> network v2 performance (#5391) + - perf(set_passwords): Run module in Network stage (#5395) + - fix(test): Remove temporary directory side effect (#5416) + - Improve schema validator warning messages (#5404) [Ani Sinha] + - feat(sysconfig): Add DNS from interface config to resolv.conf (#5401) + [Ani Sinha] + - typing: add no_implicit_optional lint (#5408) + - doc: update examples to reflect alternative ways to provide `sudo` + option (#5418) [Ani Sinha] + - fix(jsonschema): Add missing sudo definition (#5418) + - chore(doc): migrate cc modules i through r to templates (#5313) + - chore(doc): migrate grub_dpkg to tmpl add changed/deprecation (#5313) + - chore(json): migrate cc_apt_configure and json schema indents (#5313) + - chore(doc): migrate ca_certs/chef to template, flatten schema (#5313) + - chore(doc): migrate cc_byobu to templates (#5313) + - chore(doc): migrate cc_bootcmd to templates (#5313) + - fix(apt): Enable calling apt update multiple times (#5230) + - chore(VMware): Modify section of instance-id in the customization config + (#5356) [PengpengSun] + - fix(treewide): Remove dead code (#5332) [Shreenidhi Shedi] + - doc: network-config v2 ethernets are of type object (#5381) [Malte Poll] + - Release 24.1.7 (#5375) + - fix(azure): url_helper: specify User-Agent when using headers_cb with + readurl() (#5298) [Ksenija Stanojevic] + - fix: Stop attempting to resize ZFS in cc_growpart on Linux (#5370) + - doc: update docs adding YAML 1.1 spec and jinja template references + - fix(final_message): do not warn on datasourcenone when single ds + - fix(growpart): correct growpart log message to include value of mode + - feat(hotplug): disable hotplugd.socket (#5058) + - feat(hotlug): trigger hotplug after cloud-init.service (#5058) + - test: add function to push and enable systemd units (#5058) + - test(util): fix wait_until_cloud_init exit code 2 (#5058) + - test(hotplug): fix race getting ipv6 (#5271) + - docs: Adjust CSS to increase font weight across the docs (#5363) [Sally] + - fix(ec2): Correctly identify netplan renderer (#5361) + - tests: fix expect logging from growpart on devent with partition (#5360) + - test: Add v2 test coverage to test_net.py (#5247) + - refactor: Simplify collect_logs() in logs.py (#5268) + - fix: Ensure no subp from logs.py import (#5268) + - tests: fix integration tests for ubuntu pro 32.3 release (#5351) + - tests: add oracular's hello package for pkg upgrade test (#5354) + - growpart: Fix behaviour for ZFS datasets (#5169) [Mina Galić] + - device_part_info: do not recurse if we did not match anything (#5169) + [Mina Galić] + - feat(alpine): add support for Busybox adduser/addgroup (#5176) + [dermotbradley] + - ci: Move lint tip and py3-dev jobs to daily (#5347) + - fix(netplan): treat netplan warnings on stderr as debug for cloud-init + (#5348) + - feat(disk_setup): Add support for nvme devices (#5263) + - fix(log): Do not warn when doing requested operation (#5263) + - Support sudoers in the "/usr/usr merge" location (#5161) + [Robert Schweikert] + - doc(nocloud): Document network-config file (#5204) + - fix(netplan): Fix predictable interface rename issue (#5339) + - cleanup: Don't execute code on import (#5295) + - fix(net): Make duplicate route add succeed. (#5343) + - fix(freebsd): correct configuration of IPv6 routes (#5291) [Théo Bertin] + - fix(azure): disable use-dns for secondary nics (#5314) + - chore: fix lint failure (#5320) + - Update pylint version to support python 3.12 (#5338) [Ani Sinha] + - fix(tests): use regex to avoid focal whitespace in jinja debug test + (#5335) + - chore: Add docstrings and types to Version class (#5262) + - ci(mypy): add type-jinja2 stubs (#5337) + - tests(alpine): github trust lxc mounted source dir cloud-init-ro (#5329) + - test: Add oracular release to integration tests (#5328) + - Release 24.1.6 (#5326) + - test: Fix failing test_ec2.py test (#5324) + - fix: Check renderer for netplan-specific code (#5321) + - docs: Removal of top-level --file breaking change (#5308) + - fix: typo correction of delaycompress (#5317) + - docs: Renderers/Activators have downstream overrides (#5322) + - fix(ec2): Ensure metadata exists before configuring PBR (#5287) + - fix(lxd): Properly handle unicode from LXD socket (#5309) + - docs: Prefer "artifact" over "artefact" (#5311) [Arthur Le Maitre] + - chore(doc): migrate cc_byobu to templates + - chore(doc): migrate cc_bootcmd to templates + - chore(doc): migrate apt_pipelining and apk_configure to templates + - tests: in_place mount module-docs into lxd vm/container + - feat(docs): generate rtd module schema from rtd/module-docs + - feat: Set RH ssh key permissions when no 'ssh_keys' group (#5296) + [Ani Sinha] + - test: Avoid circular import in Azure tests (#5280) + - test: Fix test_failing_userdata_modules_exit_codes (#5279) + - chore: Remove CPY check from ruff (#5281) + - chore: Clean up docstrings + - chore(ruff): Bump to version 0.4.3 + - feat(systemd): Improve AlmaLinux OS and CloudLinux OS support (#5265) + [Elkhan Mammadli] + - feat(ca_certs): Add AlmaLinux OS and CloudLinux OS support (#5264) + [Elkhan Mammadli] + - docs: cc_apt_pipelining docstring typo fix (#5273) [Alex Ratner] + - feat(azure): add request identifier to IMDS requests (#5218) + [Ksenija Stanojevic] + - test: Fix TestFTP integration test (#5237) [d1r3ct0r] + - feat(ifconfig): prepare for CIDR output (#5272) [Mina Galić] + - fix: stop manually dropping dhcp6 key in integration test (#5267) + [Alec Warren] + - test: Remove some CiTestCase tests (#5256) + - fix: Warn when signal is handled (#5186) + - fix(snapd): ubuntu do not snap refresh when snap absent (LP: #2064300) + - feat(landscape-client): handle already registered client (#4784) + [Fabian Lichtenegger-Lukas] + - doc: Show how to debug external services blocking cloud-init (#5255) + - fix(pdb): Enable running cloud-init under pdb (#5217) + - chore: Update systemd description (#5250) + - fix(time): Harden cloud-init to system clock changes + - fix: Update analyze timestamp uptime + - fix(schema): no network validation on netplan systems without API + - fix(mount): Don't run cloud-init.service if cloud-init disabled (#5226) + - fix(ntp): Fix AlmaLinux OS and CloudLinux OS support (#5235) + [Elkhan Mammadli] + - tests: force version of cloud-init from PPA regardless of version (#5251) + - ci: Print isort diff (#5242) + - test: Fix integration test dependencies (#5248) + - fix(ec2): Fix broken uuid match with other-endianness (#5236) + - fix(schema): allow networkv2 schema without top-level key (#5239) + [Cat Red] + - fix(cmd): Do not hardcode reboot command (#5208) + - test: Run Alpine tests without network (#5220) + - docs: Add base config reference from explanation (#5241) + - docs: Remove preview from WSL tutorial (#5225) + - chore: Remove broken maas code (#5219) + - feat(WSL): Add support for Ubuntu Pro configs (#5116) [Ash] + - chore: sync ChangeLog and version.py from 24.1.x (#5228) + - bug(package_update): avoid snap refresh in images without snap command + (LP: #2064132) + - ci: Skip package build on tox runs (#5210) + - chore: Fix test skip message + - test(ec2): adopt pycloudlib public ip creation while launching instances + - test(ec2): add ipv6 testing for multi-nic instances + - test(ec2): adopt pycloudlib enable_ipv6 while launching instances + - feat: tool to print diff between netplan and networkv2 schema (#5200) + [Cat Red] + - test: mock internet access in test_upgrade (#5212) + - ci: Add timezone for alpine unit tests (#5216) + - fix: Ensure dump timestamps parsed as UTC (#5214) + - docs: Add WSL tutorial (#5206) + - feature(schema): add networkv2 schema (#4892) [Cat Red] + - Add alpine unittests to ci (#5121) + - test: Fix invalid openstack datasource name (#4905) + - test: Fix MAAS test and mark xfail (#4905) + - chore(ds-identify): Update shellcheck ignores (#4905) + - fix(ds-identify): Prevent various false positives and false negatives + (#4905) + - Use grep for faster parsing of cloud config in ds-identify (#4905) + [Scott Moser] (LP: #2030729) + - tests: validate netplan API YAML instead of strict content (#5195) + - chore(templates): update ubuntu universe wording (#5199) + - Deprecate the users ssh-authorized-keys property (#5162) + [Anders Björklund] + - doc(nocloud): Describe ftp and ftp over tls implementation (#5193) + - feat(net): provide network config to netplan.State for render (#4981) + - docs: Add breaking datasource identification changes (#5171) + - fix(openbsd): Update build-on-openbsd python dependencies (#5172) + [Hyacinthe Cartiaux] + - fix: Add subnet ipv4/ipv6 to network schema (#5191) + - docs: Add deprecated system_info to schema (#5168) + - docs: Add DataSourceNone documentation (#5165) + - test: Skip test if console log is None (#5188) + - fix(dhcp): Enable interactively running cloud-init init --local (#5166) + - test: Update message for netplan apply dbus issue + - test: install software-properties-common if absent during PPA setup + - test: bump pycloudlib to use latest version + - test: Update version of hello package installed on noble + - test: universally ignore netplan apply dbus issue (#5178) + - chore: Remove obsolete nose workaround + - feat: Add support for FTP and FTP over TLS (#4834) + - feat(opennebula): Add support for posix shell + - test: Make analyze tests not depend on GNU date + - test: Eliminate bash dependency from subp tests + - docs: Add breaking changes section to reference docs (#5147) [Cat Red] + - util: add log_level kwarg for logexc() (#5125) [Chris Patterson] + - refactor: Make device info part of distro definition (#5067) + - refactor: Distro-specific growpart code (#5067) + - test(ec2): fix mocking with responses==0.9.0 (focal) (#5163) + - chore(safeyaml): Remove unicode helper for Python2 (#5142) + - Revert "test: fix upgrade dhcp6 on ec2 (#5131)" (#5148) + - refactor(net): Reuse netops code + - refactor(iproute2): Make expressions multi-line for legibility + - feat(freebsd): support freebsd find part by gptid and ufsid (#5122) + [jinkangkang] + - feat: Determining route metric based on NIC name (#5070) [qidong.ld] + - test: Enable profiling in integration tests (#5130) + - dhcp: support configuring static routes for dhclient's unknown-121 + option (#5146) [Chris Patterson] + - feat(azure): parse ProvisionGuestProxyAgent as bool (#5126) + [Ksenija Stanojevic] + - fix(url_helper): fix TCP connection leak on readurl() retries (#5144) + [Chris Patterson] + - test: pytest-ify t/u/sources/test_ec2.py + - Revert "ec2: Do not enable dhcp6 on EC2 (#5104)" (#5145) [Major Hayden] + - fix: Logging sensitive data + - test: Mock ds-identify systemd path (#5119) + - fix(dhcpcd): Make lease parsing more robust (#5129) + - test: fix upgrade dhcp6 on ec2 (#5131) + - net/dhcp: raise InvalidDHCPLeaseFileError on error parsing dhcpcd lease + (#5128) [Chris Patterson] + - fix: Fix runtime file locations for cloud-init (#4820) + - ci: fix linkcheck.yml invalid yaml (#5123) + - net/dhcp: bump dhcpcd timeout to 300s (#5127) [Chris Patterson] + - ec2: Do not enable dhcp6 on EC2 (#5104) [Major Hayden] + - fix: Fall back to cached local ds if no valid ds found (#4997) + [PengpengSun] + - ci: Make linkcheck a scheduled job (#5118) + - net: Warn when interface rename fails + - ephemeral(dhcpcd): Set dhcpcd interface down + - Release 24.1.3 + - chore: Handle all level 1 TiCS security violations (#5103) + - fix: Always use single datasource if specified (#5098) + - fix(tests): Leaked mocks (#5097) + - fix(rhel)!: Fix network boot order in upstream cloud-init + - fix(rhel): Fix network ordering in sysconfig + - feat: Use NetworkManager renderer by default in RHEL family + - fix: Allow caret at the end of apt package (#5099) + - test: Add missing mocks to prevent bleed through (#5082) + [Robert Schweikert] + - fix: Ensure network config in DataSourceOracle can be unpickled (#5073) + - docs: set the home directory using homedir, not home (#5101) + [Olivier Gayot] (LP: #2047796) + - fix(cacerts): Correct configuration customizations for Photon (#5077) + [Christopher McCann] + - fix(test): Mock systemd fs path for non-systemd distros + - fix(tests): Leaked subp.which mock + - fix(networkd): add GatewayOnLink flag when necessary (#4996) [王煎饼] + - Release 24.1.2 + - test: fix `disable_sysfs_net` mock (#5065) + - refactor: don't import subp function directly (#5065) + - test: Remove side effects from tests (#5074) + - refactor: Import log module rather than functions (#5074) + - fix: Fix breaking changes in package install (#5069) + - fix: Undeprecate 'network' in schema route definition (#5072) + - refactor(ec2): simplify convert_ec2_metadata_network_config + - fix(ec2): fix ipv6 policy routing + - fix: document and add 'accept-ra' to network schema (#5060) + - bug(maas): register the correct DatasourceMAASLocal in init-local + (#5068) (LP: #2057763) + - ds-identify: Improve ds-identify testing flexibility (#5047) + - fix(ansible): Add verify_commit and inventory to ansible.pull schema + (#5032) [Fionn Fitzmaurice] + - doc: Explain breaking change in status code (#5049) + - gpg: Handle temp directory containing files (#5063) + - distro(freebsd): add_user: respect homedir (#5061) [Mina Galić] + - doc: Install required dependencies (#5054) + - networkd: Always respect accept-ra if set (#4928) [Phil Sphicas] + - chore: ignore all cloud-init_*.tar.gz in .gitignore (#5059) + - test: Don't assume ordering of ThreadPoolExecutor submissions (#5052) + - feat: Add new distro 'azurelinux' for Microsoft Azure Linux. (#4931) + [Dan Streetman] + - fix(gpg): Make gpg resilient to host configuration changes (#5026) + - Sync 24.1.1 changelog and version + - DS VMware: Fix ipv6 addr converter from netinfo to netifaces (#5029) + [PengpengSun] + - packages/debian: remove dependency on isc-dhcp-client (#5041) + [Chris Patterson] + - test: Allow fake_filesystem to work with TemporaryDirectory (#5035) + - tests: Don't wait for GCE instance teardown (#5037) + - fix: Include DataSourceCloudStack attribute in unpickle test (#5039) + - bug(vmware): initialize new DataSourceVMware attributes at unpickle + (#5021) (LP: #2056439) + - fix(apt): Don't warn on apt 822 source format (#5028) + - fix(atomic_helper.py): ensure presence of parent directories (#4938) + [Shreenidhi Shedi] + - fix: Add "broadcast" to network v1 schema (#5034) (LP: #2056460) + - pro: honor but warn on custom ubuntu_advantage in /etc/cloud/cloud.cfg + (#5030) + - net/dhcp: handle timeouts for dhcpcd (#5022) [Chris Patterson] + - fix: Make wait_for_url respect explicit arguments + - test: Fix scaleway retry assumptions + - fix: Make DataSourceOracle more resilient to early network issues + (#5025) (LP: #2056194) + - chore(cmd-modules): fix exit code when --mode init (#5017) + - feat: pylint: enable W0201 - attribute-defined-outside-init + - refactor: Ensure no attributes defined outside __init__ + - chore: disable attribute-defined-outside-init check in tests + - refactor: Use _unpickle rather than hasattr() in sources + - chore: remove unused vendordata "_pure" variables + - chore(cmd-modules): deprecate --mode init (#5005) + - tests: drop CiTestCase and convert to pytest + - bug(tests): mock reads of host's /sys/class/net via get_sys_class_path + - fix: log correct disabled path in ds-identify (#5016) + - tests: ec2 dont spend > 1 second retrying 19 times when 3 times will do + - tests: openstack mock expected ipv6 IMDS + - bug(wait_for_url): when exceptions occur url is unset, use url_exc + (LP: #2055077) + - feat(run-container): Run from arbitrary commitish (#5015) + - tests: Fix wsl test (#5008) + - feat(ds-identify): Don't run unnecessary systemd-detect-virt (#4633) + - chore(ephemeral): add debug log when bringing up ephemeral network + (#5010) [Alec Warren] + - release: sync changelog and version (#5011) + - Cleanup test_net.py (#4840) + - refactor: remove dependency on netifaces (#4634) [Cat Red] + - feat: make lxc binary configurable (#5000) + - docs: update 404 page for new doc site and bug link + - test(aws): local network connectivity on multi-nics (#4982) + - test: Make integration test output more useful (#4984) + 24.1.7 - fix(ec2): Correctly identify netplan renderer (#5361) diff --git a/cloudinit/version.py b/cloudinit/version.py index 9b141b4f87e..b6bc8227d66 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "24.1.7" +__VERSION__ = "24.2" _PACKAGED_VERSION = "@@PACKAGED_VERSION@@" FEATURES = [ From 9357c3899f6f6551c1969573f59b7f99568c34c1 Mon Sep 17 00:00:00 2001 From: Yuanhang Sun Date: Thu, 4 Jul 2024 03:10:22 +0800 Subject: [PATCH 46/75] feat(aosc): Add 'AOSC OS' support (#5310) --- cloudinit/config/cc_ca_certs.py | 10 +- cloudinit/config/cc_ntp.py | 7 ++ cloudinit/distros/__init__.py | 1 + cloudinit/distros/aosc.py | 148 +++++++++++++++++++++++++++ cloudinit/util.py | 1 + config/cloud.cfg.tmpl | 10 +- doc/rtd/reference/distros.rst | 1 + tests/unittests/distros/test_aosc.py | 10 ++ tests/unittests/test_cli.py | 3 +- tools/.github-cla-signers | 1 + tools/render-template | 1 + 11 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 cloudinit/distros/aosc.py create mode 100644 tests/unittests/distros/test_aosc.py diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 61345fcb58d..4e80947fd13 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -23,6 +23,13 @@ "ca_cert_update_cmd": ["update-ca-certificates"], } DISTRO_OVERRIDES = { + "aosc": { + "ca_cert_path": "/etc/ssl/certs/", + "ca_cert_local_path": "/etc/ssl/certs/", + "ca_cert_filename": "cloud-init-ca-cert-{cert_index}.pem", + "ca_cert_config": "/etc/ca-certificates/conf.d/cloud-init.conf", + "ca_cert_update_cmd": ["update-ca-bundle"], + }, "fedora": { "ca_cert_path": "/etc/pki/ca-trust/", "ca_cert_local_path": "/usr/share/pki/ca-trust-source/", @@ -71,6 +78,7 @@ distros = [ "almalinux", + "aosc", "cloudlinux", "alpine", "debian", @@ -149,7 +157,7 @@ def disable_default_ca_certs(distro_name, distro_cfg): """ if distro_name in ["rhel", "photon"]: remove_default_ca_certs(distro_cfg) - elif distro_name in ["alpine", "debian", "ubuntu"]: + elif distro_name in ["alpine", "aosc", "debian", "ubuntu"]: disable_system_ca_certs(distro_cfg) if distro_name in ["debian", "ubuntu"]: diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 3d659525eef..e2b83191a19 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -24,6 +24,7 @@ distros = [ "almalinux", "alpine", + "aosc", "azurelinux", "centos", "cloudlinux", @@ -109,6 +110,12 @@ "service_name": "ntpd", }, }, + "aosc": { + "systemd-timesyncd": { + "check_exe": "/usr/lib/systemd/systemd-timesyncd", + "confpath": "/etc/systemd/timesyncd.conf", + }, + }, "azurelinux": { "chrony": { "service_name": "chronyd", diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 4557d4320ee..73873cebeca 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -60,6 +60,7 @@ OSFAMILIES = { "alpine": ["alpine"], + "aosc": ["aosc"], "arch": ["arch"], "debian": ["debian", "ubuntu"], "freebsd": ["freebsd", "dragonfly"], diff --git a/cloudinit/distros/aosc.py b/cloudinit/distros/aosc.py new file mode 100644 index 00000000000..0460c740d5c --- /dev/null +++ b/cloudinit/distros/aosc.py @@ -0,0 +1,148 @@ +# Copyright (C) 2024 AOSC Developers +# +# Author: Yuanhang Sun +# +# This file is part of cloud-init. See LICENSE file for license information. +import logging + +from cloudinit import distros, helpers, subp, util +from cloudinit.distros import PackageList +from cloudinit.distros.parsers.hostname import HostnameConf +from cloudinit.distros.parsers.sys_conf import SysConf +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + systemd_locale_conf_fn = "/etc/locale.conf" + init_cmd = ["systemctl"] + network_conf_dir = "/etc/sysconfig/network" + resolve_conf_fn = "/etc/systemd/resolved.conf" + tz_local_fn = "/etc/localtime" + + dhclient_lease_directory = "/var/lib/NetworkManager" + dhclient_lease_file_regex = r"dhclient-[\w-]+\.lease" + + renderer_configs = { + "sysconfig": { + "control": "etc/sysconfig/network", + "iface_templates": "%(base)s/network-scripts/ifcfg-%(name)s", + "route_templates": { + "ipv4": "%(base)s/network-scripts/route-%(name)s", + "ipv6": "%(base)s/network-scripts/route6-%(name)s", + }, + } + } + + prefer_fqdn = False + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + self._runner = helpers.Runners(paths) + self.osfamily = "aosc" + self.default_locale = "en_US.UTF-8" + cfg["ssh_svcname"] = "sshd" + + def apply_locale(self, locale, out_fn=None): + if not out_fn: + out_fn = self.systemd_locale_conf_fn + locale_cfg = { + "LANG": locale, + } + update_locale_conf(out_fn, locale_cfg) + + def _write_hostname(self, hostname, filename): + if filename.endswith("/previous-hostname"): + conf = HostnameConf("") + conf.set_hostname(hostname) + util.write_file(filename, str(conf), 0o644) + create_hostname_file = util.get_cfg_option_bool( + self._cfg, "create_hostname_file", True + ) + if create_hostname_file: + subp.subp(["hostnamectl", "set-hostname", str(hostname)]) + else: + subp.subp( + [ + "hostnamectl", + "set-hostname", + "--transient", + str(hostname), + ] + ) + LOG.info("create_hostname_file is False; hostname set transiently") + + def _read_hostname(self, filename, default=None): + if filename.endswith("/previous-hostname"): + return util.load_text_file(filename).strip() + (out, _err) = subp.subp(["hostname"]) + out = out.strip() + if len(out): + return out + else: + return default + + def _read_system_hostname(self): + sys_hostname = self._read_hostname(self.hostname_conf_fn) + return (self.hostname_conf_fn, sys_hostname) + + def set_timezone(self, tz): + tz_file = self._find_tz_file(tz) + util.del_file(self.tz_local_fn) + util.sym_link(tz_file, self.tz_local_fn) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ["oma"] + if command: + cmd.append(command) + cmd.append("-y") + cmd.extend(pkgs) + + subp.subp(cmd, capture=False) + + def install_packages(self, pkglist: PackageList): + self.package_command("install", pkgs=pkglist) + + def update_package_sources(self): + self._runner.run( + "update-sources", + self.package_command, + "refresh", + freq=PER_INSTANCE, + ) + + +def read_locale_conf(sys_path): + exists = False + try: + contents = util.load_text_file(sys_path).splitlines() + exists = True + except IOError: + contents = [] + return (exists, SysConf(contents)) + + +def update_locale_conf(sys_path, locale_cfg): + if not locale_cfg: + return + (exists, contents) = read_locale_conf(sys_path) + updated_am = 0 + for (k, v) in locale_cfg.items(): + if v is None: + continue + v = str(v) + if len(v) == 0: + continue + contents[k] = v + updated_am += 1 + if updated_am: + lines = [ + str(contents), + ] + if not exists: + lines.insert(0, util.make_header()) + util.write_file(sys_path, "\n".join(lines) + "\n", 0o644) diff --git a/cloudinit/util.py b/cloudinit/util.py index 98dd66d59fc..505ae1b8693 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -656,6 +656,7 @@ def _get_variant(info): if linux_dist in ( "almalinux", "alpine", + "aosc", "arch", "azurelinux", "centos", diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 68175cd0ad9..4b1efdbcbf1 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -11,7 +11,7 @@ "netbsd": "NetBSD", "openbsd": "openBSD", "openmandriva": "OpenMandriva admin", "photon": "PhotonOS", "ubuntu": "Ubuntu", "unknown": "Ubuntu"}) %} -{% set groups = ({"alpine": "adm, wheel", "arch": "wheel, users", +{% set groups = ({"alpine": "adm, wheel", "aosc": "wheel", "arch": "wheel, users", "azurelinux": "wheel", "debian": "adm, audio, cdrom, dialout, dip, floppy, netdev, plugdev, sudo, video", "gentoo": "users, wheel", "mariner": "wheel", @@ -220,7 +220,7 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["alpine", "amazon", "arch", "azurelinux", "debian", "fedora", +{% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "freebsd", "gentoo", "mariner", "netbsd", "openbsd", "OpenCloudOS", "openeuler", "openmandriva", "photon", "suse", "TencentOS", "ubuntu"] or is_rhel %} @@ -238,7 +238,7 @@ system_info: {% else %} name: {{ variant }} {% endif %} -{% if variant in ["alpine", "amazon", "arch", "azurelinux", "debian", "fedora", +{% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", "openmandriva", "photon", "suse", "TencentOS", "ubuntu", "unknown"] @@ -320,7 +320,7 @@ system_info: # Automatically discover the best ntp_client ntp_client: auto {% endif %} -{% if variant in ["alpine", "amazon", "arch", "azurelinux", "debian", "fedora", +{% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", "openmandriva", "photon", "suse", "TencentOS", "ubuntu", "unknown"] @@ -368,7 +368,7 @@ system_info: {% endif %} {% if variant in ["debian", "ubuntu", "unknown"] %} ssh_svcname: ssh -{% elif variant in ["alpine", "amazon", "arch", "azurelinux", "fedora", +{% elif variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", "openmandriva", "photon", "suse", "TencentOS"] or is_rhel %} diff --git a/doc/rtd/reference/distros.rst b/doc/rtd/reference/distros.rst index 59309ece211..d54cb889153 100644 --- a/doc/rtd/reference/distros.rst +++ b/doc/rtd/reference/distros.rst @@ -7,6 +7,7 @@ Unix family of operating systems. See the complete list below. * AlmaLinux * Alpine Linux +* AOSC OS * Arch Linux * CentOS * CloudLinux diff --git a/tests/unittests/distros/test_aosc.py b/tests/unittests/distros/test_aosc.py new file mode 100644 index 00000000000..e8a66b7aef2 --- /dev/null +++ b/tests/unittests/distros/test_aosc.py @@ -0,0 +1,10 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from tests.unittests.distros import _get_distro +from tests.unittests.helpers import CiTestCase + + +class TestAOSC(CiTestCase): + def test_get_distro(self): + distro = _get_distro("aosc") + self.assertEqual(distro.osfamily, "aosc") diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 3a92d29e261..6ab6d496b16 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -319,7 +319,8 @@ def test_wb_schema_subcommand_parser(self, m_read_cfg, capsys): ["all"], [ "**Supported distros:** all", - "**Supported distros:** almalinux, alpine, azurelinux, " + "**Supported distros:** " + "almalinux, alpine, aosc, azurelinux, " "centos, cloudlinux, cos, debian, eurolinux, fedora, " "freebsd, mariner, miraclelinux, openbsd, openeuler, " "OpenCloudOS, openmandriva, opensuse, opensuse-microos, " diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index d9accd11460..34dd55c3f6d 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -105,6 +105,7 @@ klausenbusk KsenijaS landon912 ld9379435 +leavelet licebmi linitio LKHN diff --git a/tools/render-template b/tools/render-template index c3af642a08f..78beeecb2cf 100755 --- a/tools/render-template +++ b/tools/render-template @@ -14,6 +14,7 @@ def main(): "almalinux", "alpine", "amazon", + "aosc", "arch", "azurelinux", "benchmark", From 053331e5c58962672693288f681661d644129b9b Mon Sep 17 00:00:00 2001 From: Tobias Urdin Date: Wed, 3 Jul 2024 21:32:14 +0200 Subject: [PATCH 47/75] fix(openbsd): fix mtu on newline in hostname files (#5412) The /etc/hostname.* files should have the mtu on a separate line otherwise it gives error: ifconfig: mtu: bad value The lines are executed in order by ifconfig and mtu should be on it's own line. Fixes: GH-5413 --- cloudinit/net/openbsd.py | 2 +- tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py index 3a4cdf2707c..83b33e0380c 100644 --- a/cloudinit/net/openbsd.py +++ b/cloudinit/net/openbsd.py @@ -27,7 +27,7 @@ def write_config(self): ) mtu = v.get("mtu") if mtu: - content += " mtu %d" % mtu + content += "\nmtu %d" % mtu content += "\n" + self.interface_routes util.write_file(fn, content) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 34dd55c3f6d..30c411e466a 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -179,6 +179,7 @@ TheRealFalcon thetoolsmith timothegenzmer tnt-dev +tobias-urdin tomponline tsanghan tSU-RooT From 2b6fe6403db769de14f7c7b7e4aa65f5bea8f3e0 Mon Sep 17 00:00:00 2001 From: PengpengSun <40026211+PengpengSun@users.noreply.github.com> Date: Thu, 4 Jul 2024 04:06:39 +0800 Subject: [PATCH 48/75] fix(vmware): Set IPv6 to dhcp when there is no IPv6 addr (#5471) When there is no IPv6 addr given in the customization configuration, we shall set IPv6 type to dhcp6, then customized Linux network will be set to dhcp IPv6 explicitly. --- .../sources/helpers/vmware/imc/config_nic.py | 2 +- .../sources/vmware/test_vmware_config_file.py | 68 ++++++++++++++----- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index b07214a228b..254518af9e3 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -207,7 +207,7 @@ def gen_ipv6(self, name, nic): """ if not nic.staticIpv6: - return ([], []) + return ([{"type": "dhcp6"}], []) subnet_list = [] # Static Ipv6 diff --git a/tests/unittests/sources/vmware/test_vmware_config_file.py b/tests/unittests/sources/vmware/test_vmware_config_file.py index fd4bb481e46..c1415934141 100644 --- a/tests/unittests/sources/vmware/test_vmware_config_file.py +++ b/tests/unittests/sources/vmware/test_vmware_config_file.py @@ -241,27 +241,45 @@ def test_get_nics_list_dhcp(self): elif cfg.get("name") == nic2.get("name"): nic2.update(cfg) + # Test NIC1 self.assertEqual("physical", nic1.get("type"), "type of NIC1") self.assertEqual("NIC1", nic1.get("name"), "name of NIC1") self.assertEqual( "00:50:56:a6:8c:08", nic1.get("mac_address"), "mac address of NIC1" ) subnets = nic1.get("subnets") - self.assertEqual(1, len(subnets), "number of subnets for NIC1") - subnet = subnets[0] - self.assertEqual("dhcp", subnet.get("type"), "DHCP type for NIC1") - self.assertEqual("auto", subnet.get("control"), "NIC1 Control type") + self.assertEqual(2, len(subnets), "number of subnets for NIC1") + subnet_ipv4 = subnets[0] + self.assertEqual( + "dhcp", subnet_ipv4.get("type"), "Ipv4 DHCP type for NIC1" + ) + self.assertEqual( + "auto", subnet_ipv4.get("control"), "NIC1 Control type" + ) + subnet_ipv6 = subnets[1] + self.assertEqual( + "dhcp6", subnet_ipv6.get("type"), "Ipv6 DHCP type for NIC1" + ) + # Test NIC2 self.assertEqual("physical", nic2.get("type"), "type of NIC2") self.assertEqual("NIC2", nic2.get("name"), "name of NIC2") self.assertEqual( "00:50:56:a6:5a:de", nic2.get("mac_address"), "mac address of NIC2" ) subnets = nic2.get("subnets") - self.assertEqual(1, len(subnets), "number of subnets for NIC2") - subnet = subnets[0] - self.assertEqual("dhcp", subnet.get("type"), "DHCP type for NIC2") - self.assertEqual("auto", subnet.get("control"), "NIC2 Control type") + self.assertEqual(2, len(subnets), "number of subnets for NIC2") + subnet_ipv4 = subnets[0] + self.assertEqual( + "dhcp", subnet_ipv4.get("type"), "Ipv4 DHCP type for NIC2" + ) + self.assertEqual( + "auto", subnet_ipv4.get("control"), "NIC2 Control type" + ) + subnet_ipv6 = subnets[1] + self.assertEqual( + "dhcp6", subnet_ipv6.get("type"), "Ipv6 DHCP type for NIC2" + ) def test_get_nics_list_static(self): """Tests if NicConfigurator properly calculates network subnets @@ -286,6 +304,7 @@ def test_get_nics_list_static(self): elif cfg.get("name") == nic2.get("name"): nic2.update(cfg) + # Test NIC1 self.assertEqual("physical", nic1.get("type"), "type of NIC1") self.assertEqual("NIC1", nic1.get("name"), "name of NIC1") self.assertEqual( @@ -345,6 +364,7 @@ def test_get_nics_list_static(self): else: self.assertEqual(True, False, "invalid gateway %s" % (gateway)) + # Test NIC2 self.assertEqual("physical", nic2.get("type"), "type of NIC2") self.assertEqual("NIC2", nic2.get("name"), "name of NIC2") self.assertEqual( @@ -352,16 +372,18 @@ def test_get_nics_list_static(self): ) subnets = nic2.get("subnets") - self.assertEqual(1, len(subnets), "Number of subnets for NIC2") + self.assertEqual(2, len(subnets), "Number of subnets for NIC2") - subnet = subnets[0] - self.assertEqual("static", subnet.get("type"), "Subnet type") + subnet_ipv4 = subnets[0] + self.assertEqual("static", subnet_ipv4.get("type"), "Subnet type") self.assertEqual( - "192.168.6.102", subnet.get("address"), "Subnet address" + "192.168.6.102", subnet_ipv4.get("address"), "Subnet address" ) self.assertEqual( - "255.255.0.0", subnet.get("netmask"), "Subnet netmask" + "255.255.0.0", subnet_ipv4.get("netmask"), "Subnet netmask" ) + subnet_ipv6 = subnets[1] + self.assertEqual("dhcp6", subnet_ipv6.get("type"), "Subnet type") def test_custom_script(self): cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") @@ -448,7 +470,10 @@ def test_non_primary_nic_without_gateway(self): "type": "static", "address": "10.20.87.154", "netmask": "255.255.252.0", - } + }, + { + "type": "dhcp6", + }, ], } ], @@ -499,7 +524,10 @@ def test_non_primary_nic_with_gateway(self): "metric": 10000, } ], - } + }, + { + "type": "dhcp6", + }, ], } ], @@ -559,7 +587,10 @@ def test_cust_non_primary_nic_with_gateway_(self): "metric": 10000, } ], - } + }, + { + "type": "dhcp6", + }, ], } ], @@ -604,7 +635,10 @@ def test_a_primary_nic_with_gateway(self): "address": "10.20.87.154", "netmask": "255.255.252.0", "gateway": "10.20.87.253", - } + }, + { + "type": "dhcp6", + }, ], } ], From 0af459eac9c670f2a7215ecfe96f4a4b6444eaba Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 8 Jul 2024 11:03:25 -0500 Subject: [PATCH 49/75] test: pytestify and cleanup test_cc_mounts.py (#5459) * Remove `unittest` constructs and remove base classes. * Replace tests that don't test things with tests that do * Add fstab and mounts combinations test --- tests/unittests/config/test_cc_mounts.py | 389 ++++++++++++++--------- 1 file changed, 234 insertions(+), 155 deletions(-) diff --git a/tests/unittests/config/test_cc_mounts.py b/tests/unittests/config/test_cc_mounts.py index 4795357c039..07ce4b0ba40 100644 --- a/tests/unittests/config/test_cc_mounts.py +++ b/tests/unittests/config/test_cc_mounts.py @@ -1,8 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +# pylint: disable=attribute-defined-outside-init import math import os.path import re +import textwrap from collections import namedtuple from unittest import mock @@ -27,193 +29,176 @@ M_PATH = "cloudinit.config.cc_mounts." -class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): - def setUp(self): - super(TestSanitizeDevname, self).setUp() - self.new_root = self.tmp_dir() - self.patchOS(self.new_root) - - def _touch(self, path): - path = os.path.join(self.new_root, path.lstrip("/")) +class TestSanitizeDevname: + def _touch(self, path, new_root): + path = os.path.join(new_root, path.lstrip("/")) basedir = os.path.dirname(path) if not os.path.exists(basedir): os.makedirs(basedir) open(path, "a").close() - def _makedirs(self, directory): - directory = os.path.join(self.new_root, directory.lstrip("/")) + def _makedirs(self, directory, new_root): + directory = os.path.join(new_root, directory.lstrip("/")) if not os.path.exists(directory): os.makedirs(directory) - def mock_existence_of_disk(self, disk_path): - self._touch(disk_path) - self._makedirs(os.path.join("/sys/block", disk_path.split("/")[-1])) + def mock_existence_of_disk(self, disk_path, new_root): + self._touch(disk_path, new_root) + self._makedirs( + os.path.join("/sys/block", disk_path.split("/")[-1]), new_root + ) - def mock_existence_of_partition(self, disk_path, partition_number): - self.mock_existence_of_disk(disk_path) - self._touch(disk_path + str(partition_number)) + def mock_existence_of_partition( + self, disk_path, partition_number, new_root + ): + self.mock_existence_of_disk(disk_path, new_root) + self._touch(disk_path + str(partition_number), new_root) disk_name = disk_path.split("/")[-1] self._makedirs( os.path.join( "/sys/block", disk_name, disk_name + str(partition_number) - ) + ), + new_root, ) - def test_existent_full_disk_path_is_returned(self): + def test_existent_full_disk_path_is_returned(self, fake_filesystem): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path) - self.assertEqual( - disk_path, - cc_mounts.sanitize_devname(disk_path, lambda x: None), + self.mock_existence_of_disk(disk_path, fake_filesystem) + assert disk_path == cc_mounts.sanitize_devname( + disk_path, lambda x: None ) - def test_existent_disk_name_returns_full_path(self): + def test_existent_disk_name_returns_full_path(self, fake_filesystem): disk_name = "sda" disk_path = "/dev/" + disk_name - self.mock_existence_of_disk(disk_path) - self.assertEqual( - disk_path, - cc_mounts.sanitize_devname(disk_name, lambda x: None), + self.mock_existence_of_disk(disk_path, fake_filesystem) + assert disk_path == cc_mounts.sanitize_devname( + disk_name, lambda x: None ) - def test_existent_meta_disk_is_returned(self): + def test_existent_meta_disk_is_returned(self, fake_filesystem): actual_disk_path = "/dev/sda" - self.mock_existence_of_disk(actual_disk_path) - self.assertEqual( - actual_disk_path, - cc_mounts.sanitize_devname( - "ephemeral0", - lambda x: actual_disk_path, - ), + self.mock_existence_of_disk(actual_disk_path, fake_filesystem) + assert actual_disk_path == cc_mounts.sanitize_devname( + "ephemeral0", + lambda x: actual_disk_path, ) - def test_existent_meta_partition_is_returned(self): + def test_existent_meta_partition_is_returned(self, fake_filesystem): disk_name, partition_part = "/dev/sda", "1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition(disk_name, partition_part) - self.assertEqual( - actual_partition_path, - cc_mounts.sanitize_devname( - "ephemeral0.1", - lambda x: disk_name, - ), + self.mock_existence_of_partition( + disk_name, partition_part, fake_filesystem + ) + assert actual_partition_path == cc_mounts.sanitize_devname( + "ephemeral0.1", + lambda x: disk_name, ) - def test_existent_meta_partition_with_p_is_returned(self): + def test_existent_meta_partition_with_p_is_returned(self, fake_filesystem): disk_name, partition_part = "/dev/sda", "p1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition(disk_name, partition_part) - self.assertEqual( - actual_partition_path, - cc_mounts.sanitize_devname( - "ephemeral0.1", - lambda x: disk_name, - ), + self.mock_existence_of_partition( + disk_name, partition_part, fake_filesystem + ) + assert actual_partition_path == cc_mounts.sanitize_devname( + "ephemeral0.1", + lambda x: disk_name, ) - def test_first_partition_returned_if_existent_disk_is_partitioned(self): + def test_first_partition_returned_if_existent_disk_is_partitioned( + self, fake_filesystem + ): disk_name, partition_part = "/dev/sda", "1" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition(disk_name, partition_part) - self.assertEqual( - actual_partition_path, - cc_mounts.sanitize_devname( - "ephemeral0", - lambda x: disk_name, - ), + self.mock_existence_of_partition( + disk_name, partition_part, fake_filesystem + ) + assert actual_partition_path == cc_mounts.sanitize_devname( + "ephemeral0", + lambda x: disk_name, ) - def test_nth_partition_returned_if_requested(self): + def test_nth_partition_returned_if_requested(self, fake_filesystem): disk_name, partition_part = "/dev/sda", "3" actual_partition_path = disk_name + partition_part - self.mock_existence_of_partition(disk_name, partition_part) - self.assertEqual( - actual_partition_path, - cc_mounts.sanitize_devname( - "ephemeral0.3", - lambda x: disk_name, - ), + self.mock_existence_of_partition( + disk_name, partition_part, fake_filesystem + ) + assert actual_partition_path == cc_mounts.sanitize_devname( + "ephemeral0.3", + lambda x: disk_name, ) - def test_transformer_returning_none_returns_none(self): - self.assertIsNone( + def test_transformer_returning_none_returns_none(self, fake_filesystem): + assert ( cc_mounts.sanitize_devname( "ephemeral0", lambda x: None, ) + is None ) - def test_missing_device_returns_none(self): - self.assertIsNone( + def test_missing_device_returns_none(self, fake_filesystem): + assert ( cc_mounts.sanitize_devname( "/dev/sda", None, ) + is None ) - def test_missing_sys_returns_none(self): + def test_missing_sys_returns_none(self, fake_filesystem): disk_path = "/dev/sda" - self._makedirs(disk_path) - self.assertIsNone( + self._makedirs(disk_path, fake_filesystem) + assert ( cc_mounts.sanitize_devname( disk_path, None, ) + is None ) - def test_existent_disk_but_missing_partition_returns_none(self): + def test_existent_disk_but_missing_partition_returns_none( + self, fake_filesystem + ): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path) - self.assertIsNone( + self.mock_existence_of_disk(disk_path, fake_filesystem) + assert ( cc_mounts.sanitize_devname( "ephemeral0.1", lambda x: disk_path, ) + is None ) - def test_network_device_returns_network_device(self): + def test_network_device_returns_network_device(self, fake_filesystem): disk_path = "netdevice:/path" - self.assertEqual( + assert disk_path == cc_mounts.sanitize_devname( disk_path, - cc_mounts.sanitize_devname( - disk_path, - None, - ), + None, ) - def test_device_aliases_remapping(self): + def test_device_aliases_remapping(self, fake_filesystem): disk_path = "/dev/sda" - self.mock_existence_of_disk(disk_path) - self.assertEqual( - disk_path, - cc_mounts.sanitize_devname( - "mydata", lambda x: None, {"mydata": disk_path} - ), + self.mock_existence_of_disk(disk_path, fake_filesystem) + assert disk_path == cc_mounts.sanitize_devname( + "mydata", lambda x: None, {"mydata": disk_path} ) -class TestSwapFileCreation(test_helpers.FilesystemMockingTestCase): - def setUp(self): - super(TestSwapFileCreation, self).setUp() - self.new_root = self.tmp_dir() - self.patchOS(self.new_root) - - self.fstab_path = os.path.join(self.new_root, "etc/fstab") - self.swap_path = os.path.join(self.new_root, "swap.img") +class TestSwapFileCreation: + @pytest.fixture(autouse=True) + def setup(self, mocker, fake_filesystem: str): + self.new_root = fake_filesystem + self.swap_path = os.path.join(fake_filesystem, "swap.img") + fstab_path = os.path.join(fake_filesystem, "etc/fstab") self._makedirs("/etc") - self.add_patch( - "cloudinit.config.cc_mounts.FSTAB_PATH", - "mock_fstab_path", - self.fstab_path, - autospec=False, - ) - - self.add_patch("cloudinit.config.cc_mounts.subp.subp", "m_subp_subp") - - self.add_patch( - "cloudinit.config.cc_mounts.util.mounts", - "mock_util_mounts", + self.m_fstab = mocker.patch(f"{M_PATH}FSTAB_PATH", fstab_path) + self.m_subp = mocker.patch(f"{M_PATH}subp.subp") + self.m_mounts = mocker.patch( + f"{M_PATH}util.mounts", return_value={ "/dev/sda1": { "fstype": "ext4", @@ -257,7 +242,7 @@ def test_swap_creation_method_fallocate_on_xfs( m_get_mount_info.return_value = ["", "xfs"] cc_mounts.handle(None, self.cc, self.mock_cloud, []) - self.m_subp_subp.assert_has_calls( + self.m_subp.assert_has_calls( [ mock.call( ["fallocate", "-l", "0M", self.swap_path], capture=True @@ -276,7 +261,7 @@ def test_swap_creation_method_xfs( m_get_mount_info.return_value = ["", "xfs"] cc_mounts.handle(None, self.cc, self.mock_cloud, []) - self.m_subp_subp.assert_has_calls( + self.m_subp.assert_has_calls( [ mock.call( [ @@ -302,7 +287,7 @@ def test_swap_creation_method_btrfs( m_get_mount_info.return_value = ["", "btrfs"] cc_mounts.handle(None, self.cc, self.mock_cloud, []) - self.m_subp_subp.assert_has_calls( + self.m_subp.assert_has_calls( [ mock.call(["truncate", "-s", "0", self.swap_path]), mock.call(["chattr", "+C", self.swap_path]), @@ -324,7 +309,7 @@ def test_swap_creation_method_ext4( m_get_mount_info.return_value = ["", "ext4"] cc_mounts.handle(None, self.cc, self.mock_cloud, []) - self.m_subp_subp.assert_has_calls( + self.m_subp.assert_has_calls( [ mock.call( ["fallocate", "-l", "0M", self.swap_path], capture=True @@ -335,35 +320,20 @@ def test_swap_creation_method_ext4( ) -class TestFstabHandling(test_helpers.FilesystemMockingTestCase): +class TestFstabHandling: swap_path = "/dev/sdb1" - def setUp(self): - super(TestFstabHandling, self).setUp() - self.new_root = self.tmp_dir() - self.patchOS(self.new_root) + @pytest.fixture(autouse=True) + def setup(self, mocker, fake_filesystem: str): + self.new_root = fake_filesystem self.fstab_path = os.path.join(self.new_root, "etc/fstab") self._makedirs("/etc") - self.add_patch( - "cloudinit.config.cc_mounts.FSTAB_PATH", - "mock_fstab_path", - self.fstab_path, - autospec=False, - ) - - self.add_patch( - "cloudinit.config.cc_mounts._is_block_device", - "mock_is_block_device", - return_value=True, - ) - - self.add_patch("cloudinit.config.cc_mounts.subp.subp", "m_subp_subp") - - self.add_patch( - "cloudinit.config.cc_mounts.util.mounts", - "mock_util_mounts", + self.m_fstab = mocker.patch(f"{M_PATH}FSTAB_PATH", self.fstab_path) + self.m_subp = mocker.patch(f"{M_PATH}subp.subp") + self.m_mounts = mocker.patch( + f"{M_PATH}util.mounts", return_value={ "/dev/sda1": { "fstype": "ext4", @@ -373,6 +343,10 @@ def setUp(self): }, ) + self.m_is_block_device = mocker.patch( + f"{M_PATH}_is_block_device", return_value=True + ) + self.mock_cloud = mock.Mock() self.mock_log = mock.Mock() self.mock_cloud.device_name_to_device = self.device_name_to_device @@ -392,7 +366,7 @@ def device_name_to_device(self, path): def test_no_fstab(self): """Handle images which do not include an fstab.""" - self.assertFalse(os.path.exists(cc_mounts.FSTAB_PATH)) + assert not os.path.exists(cc_mounts.FSTAB_PATH) fstab_expected_content = ( "%s\tnone\tswap\tsw,comment=cloudconfig\t0\t0\n" % (self.swap_path,) @@ -400,19 +374,70 @@ def test_no_fstab(self): cc_mounts.handle(None, {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() - self.assertEqual(fstab_expected_content, fstab_new_content) + assert fstab_expected_content == fstab_new_content - def test_swap_integrity(self): - """Ensure that the swap file is correctly created and can - swapon successfully. Fixing the corner case of: - kernel: swapon: swapfile has holes""" + @pytest.mark.parametrize( + "fstype, expected", + [ + ( + "btrfs", + [ + mock.call(["truncate", "-s", "0", "/swap.img"]), + mock.call(["chattr", "+C", "/swap.img"]), + mock.call( + ["fallocate", "-l", "0M", "/swap.img"], capture=True + ), + ], + ), + ( + "xfs", + [ + mock.call( + [ + "dd", + "if=/dev/zero", + "of=/swap.img", + "bs=1M", + "count=0", + ], + capture=True, + ) + ], + ), + ( + "ext4", + [ + mock.call( + ["fallocate", "-l", "0M", "/swap.img"], capture=True + ) + ], + ), + ], + ) + def test_swap_creation_command(self, fstype, expected, mocker): + """Ensure that the swap file is correctly created. + + Different filesystems require different methods. + """ + mocker.patch( + "cloudinit.util.get_mount_info", return_value=["", fstype] + ) + mocker.patch("cloudinit.util.kernel_version", return_value=(4, 17)) fstab = "/swap.img swap swap defaults 0 0\n" with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab) - cc = {"swap": ["filename: /swap.img", "size: 512", "maxsize: 512"]} + cc = { + "swap": {"filename": "/swap.img", "size": "512", "maxsize": "512"} + } cc_mounts.handle(None, cc, self.mock_cloud, []) + assert self.m_subp.call_args_list == expected + [ + mock.call(["mkswap", "/swap.img"]), + mock.call(["swapon", "-a"]), + mock.call(["mount", "-a"]), + mock.call(["systemctl", "daemon-reload"]), + ] def test_fstab_no_swap_device(self): """Ensure that cloud-init adds a discovered swap partition @@ -431,7 +456,7 @@ def test_fstab_no_swap_device(self): with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() - self.assertEqual(fstab_expected_content, fstab_new_content) + assert fstab_expected_content == fstab_new_content def test_fstab_same_swap_device_already_configured(self): """Ensure that cloud-init will not add a swap device if the same @@ -449,7 +474,7 @@ def test_fstab_same_swap_device_already_configured(self): with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() - self.assertEqual(fstab_expected_content, fstab_new_content) + assert fstab_expected_content == fstab_new_content def test_fstab_alternate_swap_device_already_configured(self): """Ensure that cloud-init will add a discovered swap device to @@ -470,30 +495,84 @@ def test_fstab_alternate_swap_device_already_configured(self): with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() - self.assertEqual(fstab_expected_content, fstab_new_content) + assert fstab_expected_content == fstab_new_content def test_no_change_fstab_sets_needs_mount_all(self): """verify unchanged fstab entries are mounted if not call mount -a""" - fstab_original_content = ( - "LABEL=cloudimg-rootfs / ext4 defaults 0 0\n" - "LABEL=UEFI /boot/efi vfat defaults 0 0\n" - "/dev/vdb /mnt auto defaults,noexec,comment=cloudconfig 0 2\n" + fstab_original_content = textwrap.dedent( + f""" + LABEL=cloudimg-rootfs / ext4 defaults 0 0 + LABEL=UEFI /boot/efi vfat defaults 0 0 + /dev/vdb /mnt auto defaults,noexec,comment=cloudconfig 0 2 + {self.swap_path} none swap sw,comment=cloudconfig 0 0 + """ # noqa: E501 ) - fstab_expected_content = fstab_original_content cc = {"mounts": [["/dev/vdb", "/mnt", "auto", "defaults,noexec"]]} with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) + cc_mounts.handle(None, cc, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() - self.assertEqual(fstab_expected_content, fstab_new_content) - cc_mounts.handle(None, cc, self.mock_cloud, []) - self.m_subp_subp.assert_has_calls( + assert fstab_original_content == fstab_new_content + self.m_subp.assert_has_calls( [ mock.call(["mount", "-a"]), mock.call(["systemctl", "daemon-reload"]), ] ) + def test_fstab_mounts_combinations(self): + """Verify various combinations of mount entries in /etc/fstab.""" + # First and third lines show that even with errors we keep fstab lines + # unedited unless they contain the cloudconfig comment. + # 2nd line shows we remove a line with a cloudconfig comment that + # can be added back in with the mounts config. + # 4th line shows we remove a line with a cloudconfig comment + # indiscriminately. + fstab_original_content = ( + "LABEL=keepme none ext4 defaults 0 0\n" + "/dev/sda1 /a auto defaults,comment=cloudconfig 0 2\n" + "LABEL=UEFI\n" + "/dev/sda2 /b auto defaults,comment=cloudconfig 0 2\n" + ) + with open(cc_mounts.FSTAB_PATH, "w") as fd: + fd.write(fstab_original_content) + cfg = { + "mounts": [ + # Line that will be overridden due to later None value + ["/dev/sda3", "dontcare", "auto", "defaults", "0", "0"], + # Add the one missing default field to the end + ["/dev/sda4", "/mnt2", "auto", "nofail", "1"], + # Remove all "/dev/sda3"'s here and earlier + ["/dev/sda3", None], + # As long as we have two fields we get the rest of the defaults + ["/dev/sda5", "/mnt3"], + # Takes the place of the line that was removed from fstab + # with the cloudconfig comment + ["/dev/sda1", "/mnt", "xfs"], + # The line that survies after previous Nones + ["/dev/sda3", "/mnt4", "btrfs"], + ] + } + cc_mounts.handle(None, cfg, self.mock_cloud, []) + with open(cc_mounts.FSTAB_PATH, "r") as fd: + fstab_new_content = fd.read() + + assert ( + fstab_new_content.strip() + == textwrap.dedent( + """ + LABEL=keepme none ext4 defaults 0 0 + LABEL=UEFI + /dev/sda4 /mnt2 auto nofail,comment=cloudconfig 1 2 + /dev/sda5 /mnt3 auto defaults,nofail,x-systemd.after=cloud-init.service,_netdev,comment=cloudconfig 0 2 + /dev/sda1 /mnt xfs defaults,nofail,x-systemd.after=cloud-init.service,_netdev,comment=cloudconfig 0 2 + /dev/sda3 /mnt4 btrfs defaults,nofail,x-systemd.after=cloud-init.service,_netdev,comment=cloudconfig 0 2 + /dev/sdb1 none swap sw,comment=cloudconfig 0 0 + """ # noqa: E501 + ).strip() + ) + class TestCreateSwapfile: @pytest.mark.parametrize("fstype", ("xfs", "btrfs", "ext4", "other")) From 8a582709aa189c31094cd2174325b0f9183615d2 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 8 Jul 2024 19:11:42 -0500 Subject: [PATCH 50/75] test: Ensure mkcert executable in ftp tests (#5493) --- tests/integration_tests/datasources/test_nocloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/datasources/test_nocloud.py b/tests/integration_tests/datasources/test_nocloud.py index c6c440840a3..cf11520662c 100644 --- a/tests/integration_tests/datasources/test_nocloud.py +++ b/tests/integration_tests/datasources/test_nocloud.py @@ -326,7 +326,8 @@ def _boot_with_cmdline( 'wget "https://github.com/FiloSottile/mkcert/releases/' "download/${latest_ver}/mkcert-" '${latest_ver}-linux-amd64"' - " -O mkcert" + " -O mkcert && " + "chmod 755 mkcert" ).ok # giddyup From 7130bbbb146a0d6612c11c44b82a65e6c1cd97aa Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 8 Jul 2024 19:26:23 -0500 Subject: [PATCH 51/75] test: Add missing assert to test_status.py (#5494) --- tests/integration_tests/cmd/test_status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index 23509c57cef..50396be709c 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -117,7 +117,7 @@ def test_status_json_errors(client): fi cloud-init status --wait --long > $1 date +%s.%N > $MARKER_FILE -""" # noqa: E501 +""" BEFORE_CLOUD_INIT_LOCAL = """\ @@ -162,7 +162,7 @@ def test_status_block_through_all_boot_status(client): # Assert that before-cloud-init-local.service started before # cloud-init-local.service could create status.json - client.execute("test -f /before-local.start-hasstatusjson").failed + assert client.execute("test -f /before-local.start-hasstatusjson").failed early_unit_timestamp = retry_read_from_file( client, "/before-local.start-nostatusjson" From db828d054b59b98905edf5b58aa7b923a3a8c0e9 Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Fri, 5 Jul 2024 13:58:23 +0200 Subject: [PATCH 52/75] typing: fix check_untyped_defs in cloudinit.util (#5490) GH-5445 --- cloudinit/util.py | 49 +++++++++++++++++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 505ae1b8693..583a658719a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -40,7 +40,9 @@ from errno import ENOENT from functools import lru_cache, total_ordering from pathlib import Path +from types import ModuleType from typing import ( + IO, TYPE_CHECKING, Any, Callable, @@ -54,6 +56,7 @@ Sequence, TypeVar, Union, + cast, ) from urllib import parse @@ -190,6 +193,7 @@ class SeLinuxGuard: def __init__(self, path, recursive=False): # Late import since it might not always # be possible to use this + self.selinux: Optional[ModuleType] try: self.selinux = importer.import_module("selinux") except ImportError: @@ -630,7 +634,7 @@ def get_linux_distro(): dist = ("", "", "") try: # Was removed in 3.8 - dist = platform.dist() # pylint: disable=W1505,E1101 + dist = platform.dist() # type: ignore # pylint: disable=W1505,E1101 except Exception: pass finally: @@ -835,7 +839,9 @@ def set_subprocess_umask_and_gid(): stdin=subprocess.PIPE, preexec_fn=set_subprocess_umask_and_gid, ) - new_fp = proc.stdin + # As stdin is PIPE, then proc.stdin is IO[bytes] + # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.stdin + new_fp = cast(IO[Any], proc.stdin) else: raise TypeError("Invalid type for output format: %s" % outfmt) @@ -862,7 +868,9 @@ def set_subprocess_umask_and_gid(): stdin=subprocess.PIPE, preexec_fn=set_subprocess_umask_and_gid, ) - new_fp = proc.stdin + # As stdin is PIPE, then proc.stdin is IO[bytes] + # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.stdin + new_fp = cast(IO[Any], proc.stdin) else: raise TypeError("Invalid type for error format: %s" % errfmt) @@ -1696,8 +1704,8 @@ def chownbyname(fname, user=None, group=None): # output: "| logger -p" # error: "> /dev/null" # this returns the specific 'mode' entry, cleanly formatted, with value -def get_output_cfg(cfg, mode): - ret = [None, None] +def get_output_cfg(cfg, mode) -> List[Optional[str]]: + ret: List[Optional[str]] = [None, None] if not cfg or "output" not in cfg: return ret @@ -1736,10 +1744,10 @@ def get_output_cfg(cfg, mode): ret[1] = ret[0] swlist = [">>", ">", "|"] - for i in range(len(ret)): - if not ret[i]: + for i, r in enumerate(ret): + if not r: continue - val = ret[i].lstrip() + val = r.lstrip() found = False for s in swlist: if val.startswith(s): @@ -1759,7 +1767,7 @@ def get_config_logfiles(cfg): @param cfg: The cloud-init merged configuration dictionary. """ - logs = [] + logs: List = [] rotated_logs = [] if not cfg or not isinstance(cfg, dict): return logs @@ -1928,10 +1936,11 @@ def mounts(): (dev, mp, fstype, opts, _freq, _passno) = mpline.split() else: m = re.search(mountre, mpline) - dev = m.group(1) - mp = m.group(2) - fstype = m.group(3) - opts = m.group(4) + # safe to type-ignore because of the try-except wrapping + dev = m.group(1) # type: ignore + mp = m.group(2) # type: ignore + fstype = m.group(3) # type: ignore + opts = m.group(4) # type: ignore except Exception: continue # If the name of the mount point contains spaces these @@ -2455,21 +2464,27 @@ def get_proc_env(pid, encoding="utf-8", errors="replace"): @param errors: only used if encoding is true.""" fn = os.path.join("/proc", str(pid), "environ") + contents: Union[str, bytes] try: contents = load_binary_file(fn) except (IOError, OSError): return {} env = {} + null: Union[str, bytes] + equal: Union[str, bytes] null, equal = (b"\x00", b"=") if encoding: null, equal = ("\x00", "=") contents = contents.decode(encoding, errors) - for tok in contents.split(null): + # mypy doesn't know that the types of null, equal and contents are the same + # depending on the previous if branch, see: + # https://github.com/python/mypy/issues/6233 + for tok in contents.split(null): # type: ignore if not tok: continue - (name, val) = tok.split(equal, 1) + (name, val) = tok.split(equal, 1) # type: ignore if name: env[name] = val return env @@ -2529,7 +2544,7 @@ def parse_mount_info(path, mountinfo_lines, log=LOG, get_mnt_opts=False): devpth = None fs_type = None match_mount_point = None - match_mount_point_elements = None + match_mount_point_elements: Optional[List[str]] = None for i, line in enumerate(mountinfo_lines): parts = line.split() @@ -2668,7 +2683,7 @@ def parse_mount(path, get_mnt_opts=False): devpth = None mount_point = None match_mount_point = None - match_mount_point_elements = None + match_mount_point_elements: Optional[List[str]] = None for line in mountoutput.splitlines(): m = re.search(regex, line) if not m: diff --git a/pyproject.toml b/pyproject.toml index 7408488f975..d5578c1379b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ module = [ ignore_missing_imports = true no_implicit_optional = true +# See GH-5445 [[tool.mypy.overrides]] module = [ "cloudinit.analyze", @@ -118,7 +119,6 @@ module = [ "cloudinit.temp_utils", "cloudinit.templater", "cloudinit.user_data", - "cloudinit.util", "tests.integration_tests.instances", "tests.unittests.analyze.test_show", "tests.unittests.cmd.devel.test_hotplug_hook", From 188656b21e53ca2f151b439135b322040560c811 Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Tue, 9 Jul 2024 09:21:59 +0200 Subject: [PATCH 53/75] refactor: util.get_proc_env to work with strs (#5490) There are no call sites requesting not decoding the environment vars. This change decodes then always, simplifying typing and logic. --- cloudinit/util.py | 29 ++++++++++++----------------- tests/unittests/test_util.py | 13 ------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 583a658719a..09d28386646 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2454,14 +2454,16 @@ def is_lxd(): return os.path.exists("/dev/lxd/sock") -def get_proc_env(pid, encoding="utf-8", errors="replace"): +def get_proc_env( + pid, encoding: str = "utf-8", errors: str = "replace" +) -> Dict[str, str]: """ Return the environment in a dict that a given process id was started with. - @param encoding: if true, then decoding will be done with - .decode(encoding, errors) and text will be returned. - if false then binary will be returned. - @param errors: only used if encoding is true.""" + @param encoding: decoding will be done with .decode(encoding, errors) and + text will be returned. + @param errors: passed through .decode(encoding, errors). + """ fn = os.path.join("/proc", str(pid), "environ") contents: Union[str, bytes] @@ -2471,20 +2473,13 @@ def get_proc_env(pid, encoding="utf-8", errors="replace"): return {} env = {} - null: Union[str, bytes] - equal: Union[str, bytes] - null, equal = (b"\x00", b"=") - if encoding: - null, equal = ("\x00", "=") - contents = contents.decode(encoding, errors) - - # mypy doesn't know that the types of null, equal and contents are the same - # depending on the previous if branch, see: - # https://github.com/python/mypy/issues/6233 - for tok in contents.split(null): # type: ignore + null, equal = ("\x00", "=") + contents = contents.decode(encoding, errors) + + for tok in contents.split(null): if not tok: continue - (name, val) = tok.split(equal, 1) # type: ignore + (name, val) = tok.split(equal, 1) if name: env[name] = val return env diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index d790bf4f1ca..8970fb4c863 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -2802,19 +2802,6 @@ def test_non_utf8_in_environment(self, m_load_file): ) self.assertEqual(1, m_load_file.call_count) - @mock.patch(M_PATH + "load_binary_file") - def test_encoding_none_returns_bytes(self, m_load_file): - """encoding none returns bytes.""" - lines = (self.bootflag, self.simple1, self.simple2, self.mixed) - content = self.null.join(lines) - m_load_file.return_value = content - - self.assertEqual( - dict([t.split(b"=") for t in lines]), - util.get_proc_env(1, encoding=None), - ) - self.assertEqual(1, m_load_file.call_count) - @mock.patch(M_PATH + "load_binary_file") def test_all_utf8_encoded(self, m_load_file): """common path where only utf-8 decodable content.""" From 0128716c28eec2c92bd6e76d423c160c91cb3e5f Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Tue, 9 Jul 2024 10:11:09 +0200 Subject: [PATCH 54/75] refactor: util.mounts to handle errors (#5490) Instead of a broad try/except, do properly check for conditions that invalidate a mount location. --- cloudinit/util.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 09d28386646..8d7422aee09 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1927,22 +1927,23 @@ def mounts(): out = subp.subp("mount") mount_locs = out.stdout.splitlines() method = "mount" - mountre = r"^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$" + mountre = re.compile(r"^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$") for mpline in mount_locs: # Linux: /dev/sda1 on /boot type ext4 (rw,relatime,data=ordered) # FreeBSD: /dev/vtbd0p2 on / (ufs, local, journaled soft-updates) - try: - if method == "proc": - (dev, mp, fstype, opts, _freq, _passno) = mpline.split() - else: - m = re.search(mountre, mpline) - # safe to type-ignore because of the try-except wrapping - dev = m.group(1) # type: ignore - mp = m.group(2) # type: ignore - fstype = m.group(3) # type: ignore - opts = m.group(4) # type: ignore - except Exception: - continue + if method == "proc": + words = mpline.split() + if len(words) != 6: + continue + (dev, mp, fstype, opts, _freq, _passno) = words + else: + m = mountre.search(mpline) + if m is None or len(m.groups()) < 4: + continue + dev = m.group(1) + mp = m.group(2) + fstype = m.group(3) + opts = m.group(4) # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. mp = mp.replace("\\040", " ") From 4c0468c5703ada736d4eac96ed20cad12603043b Mon Sep 17 00:00:00 2001 From: Curt Moore Date: Wed, 10 Jul 2024 14:10:46 -0500 Subject: [PATCH 55/75] Set MTU for bond parent interface (#5495) Support for jumbo frames requires that the underlying physical interfaces and the parent bond interface all have the larger MTU configured, not just the physical interfaces. --- cloudinit/net/network_manager.py | 4 ++++ tests/unittests/net/network_configs.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/cloudinit/net/network_manager.py b/cloudinit/net/network_manager.py index a13d4c14f69..41dcd24b27f 100644 --- a/cloudinit/net/network_manager.py +++ b/cloudinit/net/network_manager.py @@ -431,6 +431,10 @@ def render_interface(self, iface, network_state, renderer): self.config["vlan"]["parent"] = renderer.con_ref( iface["vlan-raw-device"] ) + if if_type == "bond" and ipv4_mtu is not None: + if "ethernet" not in self.config: + self.config["ethernet"] = {} + self.config["ethernet"]["mtu"] = str(ipv4_mtu) if if_type == "bridge": # Bridge is ass-backwards compared to bond for port in iface["bridge_ports"]: diff --git a/tests/unittests/net/network_configs.py b/tests/unittests/net/network_configs.py index b68319cc806..2b55bbf421a 100644 --- a/tests/unittests/net/network_configs.py +++ b/tests/unittests/net/network_configs.py @@ -3385,6 +3385,9 @@ route1=2001:67c::/32,2001:67c:1562::1 route2=3001:67c::/32,3001:67c:15::1 + [ethernet] + mtu=9000 + """ ), }, From 7d35664ef8b85840f92f18cc48187f7284d227bc Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Thu, 11 Jul 2024 00:49:58 +0530 Subject: [PATCH 56/75] fix: add schema rules for 'baseurl' and 'metalink' in yum repo config (#5501) At least one of (or both) 'baseurl' or 'metalink' should be provided for yum repository specification. Add schema changes to enforce it. Without this, with just 'metalink' property set, one would get the schema validator error \--- Error: Cloud config schema errors: yum_repos.epel-release: 'baseurl' is a required property \--- Signed-off-by: Ani Sinha --- .../config/schemas/schema-cloud-config-v1.json | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index f5609c539fc..e45504d4113 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -3452,6 +3452,11 @@ "format": "uri", "description": "URL to the directory where the yum repository's 'repodata' directory lives" }, + "metalink": { + "type": "string", + "format": "uri", + "description": "Specifies a URL to a metalink file for the repomd.xml" + }, "name": { "type": "string", "description": "Optional human-readable name of the yum repo." @@ -3479,8 +3484,17 @@ "description": "Any supported yum repository configuration options will be written to the yum repo config file. See: man yum.conf" } }, - "required": [ - "baseurl" + "anyOf": [ + { + "required": [ + "baseurl" + ] + }, + { + "required": [ + "metalink" + ] + } ] } } From 4abdd5a7066c322163579d8d91a44426ac705172 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Wed, 24 Apr 2024 10:51:16 -0600 Subject: [PATCH 57/75] feat(systemd): Warn user of unexpected run mode (#5209) On systemd, services are started by PID 1. When this doesn't happen, cloud-init is in an unknown run state and should warn the user. Reorder pid log to be able to reuse Distro information. Add docstring deprecating util.is_Linux(). --- cloudinit/cmd/main.py | 31 ++++++++++++++++++++++--------- cloudinit/distros/__init__.py | 3 +++ cloudinit/distros/bsd.py | 7 +++++++ cloudinit/util.py | 6 ++++++ 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 4a1c8b2e28c..317f3d0ff59 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -79,17 +79,29 @@ def print_exc(msg=""): sys.stderr.write("\n") -def log_ppid(): - if util.is_Linux(): +def log_ppid(distro, bootstage_name): + if distro.is_linux: ppid = os.getppid() - LOG.info("PID [%s] started cloud-init.", ppid) + log = LOG.info + extra_message = "" + if 1 != ppid and distro.uses_systemd(): + log = LOG.warning + extra_message = ( + " Unsupported configuration: boot stage called " + "outside of systemd" + ) + log( + "PID [%s] started cloud-init '%s'.%s", + ppid, + bootstage_name, + extra_message, + ) def welcome(action, msg=None): if not msg: msg = welcome_format(action) util.multi_log("%s\n" % (msg), console=False, stderr=True, log=LOG) - log_ppid() return msg @@ -333,10 +345,8 @@ def main_init(name, args): # objects config as it may be different from init object # 10. Run the modules for the 'init' stage # 11. Done! - if not args.local: - w_msg = welcome_format(name) - else: - w_msg = welcome_format("%s-local" % (name)) + bootstage_name = "init-local" if args.local else "init" + w_msg = welcome_format(bootstage_name) init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) @@ -364,6 +374,7 @@ def main_init(name, args): # config applied. We send the welcome message now, as stderr/out have # been redirected and log now configured. welcome(name, msg=w_msg) + log_ppid(init.distro, bootstage_name) # re-play early log messages before logging was setup for lvl, msg in early_logs: @@ -591,7 +602,8 @@ def main_modules(action_name, args): # the modules objects configuration # 5. Run the modules for the given stage name # 6. Done! - w_msg = welcome_format("%s:%s" % (action_name, name)) + bootstage_name = "%s:%s" % (action_name, name) + w_msg = welcome_format(bootstage_name) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) @@ -628,6 +640,7 @@ def main_modules(action_name, args): # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) + log_ppid(init.distro, bootstage_name) if name == "init": util.deprecate( diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 73873cebeca..e6bfb1d3b48 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -171,6 +171,7 @@ def __init__(self, name, cfg, paths): self.package_managers: List[PackageManager] = [] self._dhcp_client = None self._fallback_interface = None + self.is_linux = True def _unpickle(self, ci_pkl_version: int) -> None: """Perform deserialization fixes for Distro.""" @@ -187,6 +188,8 @@ def _unpickle(self, ci_pkl_version: int) -> None: self._dhcp_client = None if not hasattr(self, "_fallback_interface"): self._fallback_interface = None + if not hasattr(self, "is_linux"): + self.is_linux = True def _validate_entry(self, entry): if isinstance(entry, str): diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 25b374ba3bc..15be9c36714 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -40,6 +40,13 @@ def __init__(self, name, cfg, paths): cfg["rsyslog_svcname"] = "rsyslogd" self.osfamily = platform.system().lower() self.net_ops = bsd_netops.BsdNetOps + self.is_linux = False + + def _unpickle(self, ci_pkl_version: int) -> None: + super()._unpickle(ci_pkl_version) + + # this needs to be after the super class _unpickle to override it + self.is_linux = False def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/util.py b/cloudinit/util.py index 8d7422aee09..19f1800928d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -499,6 +499,12 @@ def multi_log( @lru_cache() def is_Linux(): + """deprecated: prefer Distro object's `is_linux` property + + Multiple sources of truth is bad, and already know whether we are + working with Linux from the Distro class. Using Distro offers greater code + reusablity, cleaner code, and easier maintenance. + """ return "Linux" in platform.system() From 604d80eb6fc9c78ed5669a58204fa366b4b71bdf Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 3 Jun 2024 18:53:37 -0600 Subject: [PATCH 58/75] test: Don't fail tests which call cloud-init as a command (#5209) Implement verify_clean_boot() to ignore certain expected logs in a platform-specific way. --- .../datasources/test_ec2_ipv6.py | 19 +++-- tests/integration_tests/test_upgrade.py | 12 +-- tests/integration_tests/util.py | 75 ++++++++++++++++++- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/tests/integration_tests/datasources/test_ec2_ipv6.py b/tests/integration_tests/datasources/test_ec2_ipv6.py index 41bb852e627..0f5a2dbf6bb 100644 --- a/tests/integration_tests/datasources/test_ec2_ipv6.py +++ b/tests/integration_tests/datasources/test_ec2_ipv6.py @@ -2,13 +2,22 @@ import pytest +from cloudinit.util import should_log_deprecation from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM - - -def _test_crawl(client, ip): - assert client.execute("cloud-init clean --logs").ok - assert client.execute("cloud-init init --local").ok +from tests.integration_tests.util import get_feature_flag_value + + +def _test_crawl(client: IntegrationInstance, ip: str): + return_code = ( + 2 + if should_log_deprecation( + "24.3", get_feature_flag_value(client, "DEPRECATION_INFO_BOUNDARY") + ) + else 0 + ) + assert client.execute("cloud-init clean --logs") + assert return_code == client.execute("cloud-init init --local").return_code log = client.read_from_file("/var/log/cloud-init.log") assert f"Using metadata source: '{ip}'" in log result = re.findall(r"Crawl of metadata service.* (\d+.\d+) seconds", log) diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py index 970a2406d8a..0a53eabb50e 100644 --- a/tests/integration_tests/test_upgrade.py +++ b/tests/integration_tests/test_upgrade.py @@ -14,7 +14,7 @@ IS_UBUNTU, MANTIC, ) -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log LOG = logging.getLogger("integration_testing.test_upgrade") @@ -81,11 +81,8 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud): pre_cloud_blame = instance.execute("cloud-init analyze blame") # Ensure no issues pre-upgrade - log = instance.read_from_file("/var/log/cloud-init.log") - assert not json.loads(pre_result)["v1"]["errors"] - try: - verify_clean_log(log) + verify_clean_boot(instance) except AssertionError: LOG.warning( "There were errors/warnings/tracebacks pre-upgrade. " @@ -122,10 +119,7 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud): post_cloud_blame = instance.execute("cloud-init analyze blame") # Ensure no issues post-upgrade - assert not json.loads(pre_result)["v1"]["errors"] - - log = instance.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log) + verify_clean_boot(instance) # Ensure important things stayed the same assert pre_hostname == post_hostname diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index d218861f549..1343ac0df73 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -14,6 +14,7 @@ import pytest from cloudinit.subp import subp +from tests.integration_tests.integration_settings import PLATFORM LOG = logging.getLogger("integration_testing.util") @@ -70,7 +71,8 @@ def verify_clean_boot( ): """raise assertions if the client experienced unexpected warnings or errors - fail when an required error isn't found + Fail when a required error isn't found. + Expected warnings and errors are defined in this function. This function is similar to verify_clean_log, hence the similar name. @@ -89,6 +91,77 @@ def verify_clean_boot( require_errors: Optional[list] = None, fail_when_expected_not_found: optional list of expected errors """ + + def append_or_create_list( + maybe_list: Optional[Union[List[str], bool]], value: str + ) -> List[str]: + """handle multiple types""" + if isinstance(maybe_list, list): + maybe_list.append(value) + elif maybe_list is None or isinstance(maybe_list, bool): + maybe_list = [value] + return maybe_list + + # Define exceptions by matrix of platform and Ubuntu release + if "azure" == PLATFORM: + # Consistently on all Azure launches: + ignore_warnings = append_or_create_list( + ignore_warnings, "No lease found; using default endpoint" + ) + elif "lxd_vm" == PLATFORM: + # Ubuntu lxd storage + ignore_warnings = append_or_create_list( + ignore_warnings, "thinpool by default on Ubuntu due to LP #1982780" + ) + ignore_warnings = append_or_create_list( + ignore_warnings, + "Could not match supplied host pattern, ignoring:", + ) + elif "oracle" == PLATFORM: + # LP: #1842752 + ignore_errors = append_or_create_list( + ignore_warnings, "Stderr: RTNETLINK answers: File exists" + ) + # LP: #1833446 + ignore_warnings = append_or_create_list( + ignore_warnings, + "UrlError: 404 Client Error: Not Found for url: " + "http://169.254.169.254/latest/meta-data/", + ) + # Oracle has a file in /etc/cloud/cloud.cfg.d that contains + # users: + # - default + # - name: opc + # ssh_redirect_user: true + # This can trigger a warning about opc having no public key + ignore_warnings = append_or_create_list( + ignore_warnings, + "Unable to disable SSH logins for opc given ssh_redirect_user", + ) + + _verify_clean_boot( + instance, + ignore_warnings=ignore_warnings, + ignore_errors=ignore_errors, + require_warnings=require_warnings, + require_errors=require_errors, + ) + # assert no Tracebacks + assert ( + "0" + == instance.execute( + "grep --count Traceback /var/log/cloud-init.log" + ).stdout.strip() + ), "Unexpected traceback found in /var/log/cloud-init.log" + + +def _verify_clean_boot( + instance: "IntegrationInstance", + ignore_warnings: Optional[Union[List[str], bool]] = None, + ignore_errors: Optional[Union[List[str], bool]] = None, + require_warnings: Optional[list] = None, + require_errors: Optional[list] = None, +): ignore_errors = ignore_errors or [] ignore_warnings = ignore_warnings or [] require_errors = require_errors or [] From 8aa1c30dda7a1bac08219994c446ef1ee23eea36 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 19 Jun 2024 10:37:22 -0600 Subject: [PATCH 59/75] test: allow verify_clean_boot to ignore all or specific tracebacks (#5209) Ensure ignore_warnings=True or ignore_errors=True is honored and not overridden by supplemental warning texts appended. --- .../datasources/test_nocloud.py | 4 ++ tests/integration_tests/util.py | 48 ++++++++++++++----- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/tests/integration_tests/datasources/test_nocloud.py b/tests/integration_tests/datasources/test_nocloud.py index cf11520662c..6cfe037a448 100644 --- a/tests/integration_tests/datasources/test_nocloud.py +++ b/tests/integration_tests/datasources/test_nocloud.py @@ -428,6 +428,10 @@ def test_nocloud_ftps_unencrypted_server_fails( " a scheme of ftps://, which is not allowed. Use ftp:// " "to allow connecting to insecure ftp servers.", ], + ignore_tracebacks=[ + 'ftplib.error_perm: 500 Command "AUTH" not understood.', + "UrlError: Attempted to connect to an insecure ftp server", + ], ) def test_nocloud_ftps_encrypted_server_succeeds( diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 1343ac0df73..4830cf958de 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -66,6 +66,7 @@ def verify_clean_boot( instance: "IntegrationInstance", ignore_warnings: Optional[Union[List[str], bool]] = None, ignore_errors: Optional[Union[List[str], bool]] = None, + ignore_tracebacks: Optional[Union[List[str], bool]] = None, require_warnings: Optional[list] = None, require_errors: Optional[list] = None, ): @@ -94,14 +95,17 @@ def verify_clean_boot( def append_or_create_list( maybe_list: Optional[Union[List[str], bool]], value: str - ) -> List[str]: + ) -> Optional[Union[List[str], bool]]: """handle multiple types""" if isinstance(maybe_list, list): maybe_list.append(value) - elif maybe_list is None or isinstance(maybe_list, bool): + elif maybe_list is True: + return True # Ignoring all texts, so no need to append. + elif maybe_list in (None, False): maybe_list = [value] return maybe_list + traceback_texts = [] # Define exceptions by matrix of platform and Ubuntu release if "azure" == PLATFORM: # Consistently on all Azure launches: @@ -122,12 +126,17 @@ def append_or_create_list( ignore_errors = append_or_create_list( ignore_warnings, "Stderr: RTNETLINK answers: File exists" ) + traceback_texts.append("Stderr: RTNETLINK answers: File exists") # LP: #1833446 ignore_warnings = append_or_create_list( ignore_warnings, "UrlError: 404 Client Error: Not Found for url: " "http://169.254.169.254/latest/meta-data/", ) + traceback_texts.append( + "UrlError: 404 Client Error: Not Found for url: " + "http://169.254.169.254/latest/meta-data/" + ) # Oracle has a file in /etc/cloud/cloud.cfg.d that contains # users: # - default @@ -143,22 +152,17 @@ def append_or_create_list( instance, ignore_warnings=ignore_warnings, ignore_errors=ignore_errors, + ignore_tracebacks=ignore_tracebacks, require_warnings=require_warnings, require_errors=require_errors, ) - # assert no Tracebacks - assert ( - "0" - == instance.execute( - "grep --count Traceback /var/log/cloud-init.log" - ).stdout.strip() - ), "Unexpected traceback found in /var/log/cloud-init.log" def _verify_clean_boot( instance: "IntegrationInstance", ignore_warnings: Optional[Union[List[str], bool]] = None, ignore_errors: Optional[Union[List[str], bool]] = None, + ignore_tracebacks: Optional[Union[List[str], bool]] = None, require_warnings: Optional[list] = None, require_errors: Optional[list] = None, ): @@ -181,9 +185,9 @@ def _verify_clean_boot( if expected in current_error: required_errors_found.add(expected) - # check for unexpected errors if ignore_errors is True: continue + # check for unexpected errors for expected in [*ignore_errors, *require_errors]: if expected in current_error: break @@ -198,9 +202,9 @@ def _verify_clean_boot( if expected in current_warning: required_warnings_found.add(expected) - # check for unexpected warnings if ignore_warnings is True: continue + # check for unexpected warnings for expected in [*ignore_warnings, *require_warnings]: if expected in current_warning: break @@ -241,6 +245,28 @@ def _verify_clean_boot( ) assert not errors, message + if ignore_tracebacks is True: + return + # assert no unexpected Tracebacks + expected_traceback_count = 0 + traceback_count = int( + instance.execute( + "grep --count Traceback /var/log/cloud-init.log" + ).stdout.strip() + ) + if ignore_tracebacks: + for expected_traceback in ignore_tracebacks: + expected_traceback_count += int( + instance.execute( + f"grep --count '{expected_traceback}'" + " /var/log/cloud-init.log" + ).stdout.strip() + ) + assert expected_traceback_count == traceback_count, ( + f"{traceback_count - expected_traceback_count} unexpected traceback(s)" + " found in /var/log/cloud-init.log" + ) + def verify_clean_log(log: str, ignore_deprecations: bool = True): """Assert no unexpected tracebacks or warnings in logs""" From 75add5c72aa575d373825deddcb685f725e290d8 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 19 Jun 2024 11:04:35 -0600 Subject: [PATCH 60/75] feat(systemd): convert warning level message to deprecation (#5209) Avoid using warning level messages as there may be some use-cases in the wild that need to invoke cloud-init boot stages after boot for some reason unknown to upstream. Provide a detailed warning message informing admins to file issues against cloud-init to better represent those feature needs before dropping this feature altogether. --- cloudinit/cmd/main.py | 31 +++++++++++++++++++------------ doc/rtd/reference/cli.rst | 18 +++++++++++------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 317f3d0ff59..590173ae4fc 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -79,23 +79,30 @@ def print_exc(msg=""): sys.stderr.write("\n") +DEPRECATE_BOOT_STAGE_MESSAGE = ( + "Triggering cloud-init boot stages outside of intial system boot is not a" + " fully supported operation which can lead to incomplete or incorrect" + " configuration. As such, cloud-init is deprecating this feature in the" + " future. If you currently use cloud-init in this way," + " please file an issue describing in detail your use case so that" + " cloud-init can better support your needs:" + " https://github.com/canonical/cloud-init/issues/new" +) + + def log_ppid(distro, bootstage_name): if distro.is_linux: ppid = os.getppid() - log = LOG.info - extra_message = "" if 1 != ppid and distro.uses_systemd(): - log = LOG.warning - extra_message = ( - " Unsupported configuration: boot stage called " - "outside of systemd" + util.deprecate( + deprecated=( + "Unsupported configuration: boot stage called " + f"by PID [{ppid}] outside of systemd" + ), + deprecated_version="24.3", + extra_message=DEPRECATE_BOOT_STAGE_MESSAGE, ) - log( - "PID [%s] started cloud-init '%s'.%s", - ppid, - bootstage_name, - extra_message, - ) + LOG.info("PID [%s] started cloud-init '%s'.", ppid, bootstage_name) def welcome(action, msg=None): diff --git a/doc/rtd/reference/cli.rst b/doc/rtd/reference/cli.rst index 0a6bc55ff1f..9c0bbe9c3ee 100644 --- a/doc/rtd/reference/cli.rst +++ b/doc/rtd/reference/cli.rst @@ -212,9 +212,10 @@ Example output: Generally run by OS init systems to execute ``cloud-init``'s stages: *init* and *init-local*. See :ref:`boot_stages` for more info. -Can be run on the command line, but is generally gated to run only once -due to semaphores in :file:`/var/lib/cloud/instance/sem/` and -:file:`/var/lib/cloud/sem`. +Can be run on the command line, but is deprecated, because incomplete +configuration can be applied when run later in boot. The boot stages are +generally gated to run only once due to semaphores in +:file:`/var/lib/cloud/instance/sem/` and :file:`/var/lib/cloud/sem`. * :command:`--local`: Run *init-local* stage instead of *init*. * :command:`--file` : Use additional yaml configuration files. @@ -226,16 +227,19 @@ due to semaphores in :file:`/var/lib/cloud/instance/sem/` and Generally run by OS init systems to execute ``modules:config`` and ``modules:final`` boot stages. This executes cloud config :ref:`modules` -configured to run in the Init, Config and Final stages. The modules are -declared to run in various boot stages in the file +configured to run in the Init, Config and Final stages. Can be run on the +command line, but this is not recommended and will generate a warning because +incomplete configuration can be applied when run later in boot. +The modules are declared to run in various boot stages in the file :file:`/etc/cloud/cloud.cfg` under keys: * ``cloud_init_modules`` * ``cloud_config_modules`` * ``cloud_final_modules`` -Can be run on the command line, but each module is gated to run only once due -to semaphores in :file:`/var/lib/cloud/`. +Can be run on the command line, but is deprecated, because incomplete +configuration can be applied when run later in boot. Each module is gated to +run only once due to semaphores in :file:`/var/lib/cloud/`. * :command:`--mode [init|config|final]`: Run ``modules:init``, ``modules:config`` or ``modules:final`` ``cloud-init`` stages. From a911d07955300b6b07898627c077b13e0ade4e62 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Wed, 10 Jul 2024 16:04:25 -0600 Subject: [PATCH 61/75] fix(test): Fix ip printer for non-lxd (#5488) --- tests/integration_tests/instances.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index 32281756cd1..cfae37c272c 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -294,6 +294,8 @@ def ip(self) -> str: and self.instance.execute_via_ssh ): self._ip = self.instance.ip + elif not isinstance(self.instance, LXDInstance): + self._ip = self.instance.ip except NotImplementedError: self._ip = "Unknown" return self._ip From 18d76ac60d96186a6f89fd8b0b3ace4c70bbd174 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 10 Jul 2024 16:38:58 -0600 Subject: [PATCH 62/75] tests: revert expectation of exit 2 from cloud-init init --local (#5504) Commit 604d80eb introduced assertions expecting exit 2 from the CLI when calling cloud-init init --local. Revert this test assertion as only cloud-init status command exits (2) on deprecations/warnings. Invoking cloud-init's boot stages on the commmand line will only exit 1 if critical errors are encountered to avoid degrading overall systemd health as seen from cloud-init systemd units. When cloud-init boot stages encounter recoverable_errors of any type, there is no need to exit non-zero as those deprecation logs are not-critical to the health of the system as a whole. --- .../datasources/test_ec2_ipv6.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/integration_tests/datasources/test_ec2_ipv6.py b/tests/integration_tests/datasources/test_ec2_ipv6.py index 0f5a2dbf6bb..41bb852e627 100644 --- a/tests/integration_tests/datasources/test_ec2_ipv6.py +++ b/tests/integration_tests/datasources/test_ec2_ipv6.py @@ -2,22 +2,13 @@ import pytest -from cloudinit.util import should_log_deprecation from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.util import get_feature_flag_value - - -def _test_crawl(client: IntegrationInstance, ip: str): - return_code = ( - 2 - if should_log_deprecation( - "24.3", get_feature_flag_value(client, "DEPRECATION_INFO_BOUNDARY") - ) - else 0 - ) - assert client.execute("cloud-init clean --logs") - assert return_code == client.execute("cloud-init init --local").return_code + + +def _test_crawl(client, ip): + assert client.execute("cloud-init clean --logs").ok + assert client.execute("cloud-init init --local").ok log = client.read_from_file("/var/log/cloud-init.log") assert f"Using metadata source: '{ip}'" in log result = re.findall(r"Crawl of metadata service.* (\d+.\d+) seconds", log) From 8dbc5c23b68bf73551fd39c8a05801b86e38519d Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 10 Jul 2024 17:52:25 -0500 Subject: [PATCH 63/75] test: Unconditionally skip test_multi_nic_hotplug_vpc (#5503) It is pretty consistently failing due to #5373 with no fix in sight. --- tests/integration_tests/modules/test_hotplug.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration_tests/modules/test_hotplug.py b/tests/integration_tests/modules/test_hotplug.py index 8c7bc7839d0..c088240de1a 100644 --- a/tests/integration_tests/modules/test_hotplug.py +++ b/tests/integration_tests/modules/test_hotplug.py @@ -301,6 +301,7 @@ def test_multi_nic_hotplug(setup_image, session_cloud: IntegrationCloud): @pytest.mark.skipif(CURRENT_RELEASE <= FOCAL, reason="See LP: #2055397") @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") +@pytest.mark.skip(reason="IMDS race, see GH-5373. Unskip when fixed.") def test_multi_nic_hotplug_vpc(setup_image, session_cloud: IntegrationCloud): """Tests that additional secondary NICs are routable from local networks after the hotplug hook is executed when network updates From e0e6a427fdc6826bf7b11d52157a1c5f9b3dde4d Mon Sep 17 00:00:00 2001 From: Curt Moore Date: Thu, 11 Jul 2024 05:17:25 -0500 Subject: [PATCH 64/75] Fix configuration of DNS servers via OpenStack (#5384) Ensure DNS server addresses are parsed from the proper location of network_data.json Fixes #5386 Co-authored-by: Alberto Contreras --- cloudinit/sources/helpers/openstack.py | 29 +++- .../sources/helpers/test_openstack.py | 126 ++++++++++++++++++ 2 files changed, 153 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 70998dda2ee..9b46a22c37d 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -578,8 +578,8 @@ def convert_net_json(network_json=None, known_macs=None): "scope", "dns_nameservers", "dns_search", - "routes", ], + "routes": ["network", "destination", "netmask", "gateway", "metric"], } links = network_json.get("links", []) @@ -620,6 +620,20 @@ def convert_net_json(network_json=None, known_macs=None): (k, v) for k, v in network.items() if k in valid_keys["subnet"] ) + # Filter the route entries as they may contain extra elements such + # as DNS which are required elsewhere by the cloudinit schema + routes = [ + dict( + (k, v) + for k, v in route.items() + if k in valid_keys["routes"] + ) + for route in network.get("routes", []) + ] + + if routes: + subnet.update({"routes": routes}) + if network["type"] == "ipv4_dhcp": subnet.update({"type": "dhcp4"}) elif network["type"] == "ipv6_dhcp": @@ -646,11 +660,22 @@ def convert_net_json(network_json=None, known_macs=None): } ) + # Look for either subnet or network specific DNS servers + # and add them as subnet level DNS entries. + # Subnet specific nameservers dns_nameservers = [ service["address"] - for service in network.get("services", []) + for route in network.get("routes", []) + for service in route.get("services", []) if service.get("type") == "dns" ] + # Network specific nameservers + for service in network.get("services", []): + if service.get("type") != "dns": + continue + if service["address"] in dns_nameservers: + continue + dns_nameservers.append(service["address"]) if dns_nameservers: subnet["dns_nameservers"] = dns_nameservers diff --git a/tests/unittests/sources/helpers/test_openstack.py b/tests/unittests/sources/helpers/test_openstack.py index 7ae164140a0..6ec0bd75b0d 100644 --- a/tests/unittests/sources/helpers/test_openstack.py +++ b/tests/unittests/sources/helpers/test_openstack.py @@ -231,3 +231,129 @@ def test_bond_mac(self): assert expected == openstack.convert_net_json( network_json=network_json, known_macs=macs ) + + def test_dns_servers(self): + """ + Verify additional properties under subnet.routes are not rendered + """ + network_json = { + "links": [ + { + "id": "ens1f0np0", + "name": "ens1f0np0", + "type": "phy", + "ethernet_mac_address": "xx:xx:xx:xx:xx:00", + "mtu": 9000, + }, + { + "id": "ens1f1np1", + "name": "ens1f1np1", + "type": "phy", + "ethernet_mac_address": "xx:xx:xx:xx:xx:01", + "mtu": 9000, + }, + { + "id": "bond0", + "name": "bond0", + "type": "bond", + "bond_links": ["ens1f0np0", "ens1f1np1"], + "mtu": 9000, + "ethernet_mac_address": "xx:xx:xx:xx:xx:00", + "bond_mode": "802.3ad", + "bond_xmit_hash_policy": "layer3+4", + "bond_miimon": 100, + }, + { + "id": "bond0.123", + "name": "bond0.123", + "type": "vlan", + "vlan_link": "bond0", + "vlan_id": 123, + "vlan_mac_address": "xx:xx:xx:xx:xx:00", + }, + ], + "networks": [ + { + "id": "publicnet-ipv4", + "type": "ipv4", + "link": "bond0.123", + "ip_address": "x.x.x.x", + "netmask": "255.255.255.0", + "routes": [ + { + "network": "0.0.0.0", + "netmask": "0.0.0.0", + "gateway": "x.x.x.1", + "services": [ + {"type": "dns", "address": "1.1.1.1"}, + {"type": "dns", "address": "8.8.8.8"}, + ], + } + ], + "network_id": "00000000-0000-0000-0000-000000000000", + } + ], + "services": [], + } + expected = { + "version": 1, + "config": [ + { + "name": "ens1f0np0", + "type": "physical", + "mtu": 9000, + "subnets": [], + "mac_address": "xx:xx:xx:xx:xx:00", + }, + { + "name": "ens1f1np1", + "type": "physical", + "mtu": 9000, + "subnets": [], + "mac_address": "xx:xx:xx:xx:xx:01", + }, + { + "name": "bond0", + "type": "bond", + "mtu": 9000, + "subnets": [], + "mac_address": "xx:xx:xx:xx:xx:00", + "params": { + "bond-mode": "802.3ad", + "bond-xmit_hash_policy": "layer3+4", + "bond-miimon": 100, + }, + "bond_interfaces": ["ens1f0np0", "ens1f1np1"], + }, + { + "name": "bond0.123", + "type": "vlan", + "subnets": [ + { + "type": "static", + "netmask": "255.255.255.0", + "routes": [ + { + "network": "0.0.0.0", + "netmask": "0.0.0.0", + "gateway": "x.x.x.1", + } + ], + "address": "x.x.x.x", + "dns_nameservers": ["1.1.1.1", "8.8.8.8"], + "ipv4": True, + } + ], + "vlan_id": 123, + "vlan_link": "bond0", + }, + ], + } + macs = { + "xx:xx:xx:xx:xx:00": "ens1f0np0", + "xx:xx:xx:xx:xx:01": "ens1f1np1", + } + netcfg = openstack.convert_net_json( + network_json=network_json, known_macs=macs + ) + assert expected == netcfg From 311f7234765dec7c0f1ede7ecc88b303160bc892 Mon Sep 17 00:00:00 2001 From: Curt Moore Date: Thu, 18 Jul 2024 01:04:53 -0500 Subject: [PATCH 65/75] fix: Update DNS behavior for NetworkManager interfaces (#5496) If DNS information is added to a NetworkManager managed interface where the given protocol family is disabled, NetworkManager will be unable to activate the interface. #5387 --- cloudinit/net/network_manager.py | 10 +- tests/unittests/net/test_network_manager.py | 323 ++++++++++++++++++++ 2 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 tests/unittests/net/test_network_manager.py diff --git a/cloudinit/net/network_manager.py b/cloudinit/net/network_manager.py index 41dcd24b27f..06305668fe4 100644 --- a/cloudinit/net/network_manager.py +++ b/cloudinit/net/network_manager.py @@ -239,7 +239,10 @@ def _add_nameserver(self, dns: str) -> None: Extends the ipv[46].dns property with a name server. """ family = "ipv6" if is_ipv6_address(dns) else "ipv4" - if self.config.has_section(family): + if ( + self.config.has_section(family) + and self._get_config_option(family, "method") != "disabled" + ): self._set_default(family, "dns", "") self.config[family]["dns"] = self.config[family]["dns"] + dns + ";" @@ -248,7 +251,10 @@ def _add_dns_search(self, dns_search: List[str]) -> None: Extends the ipv[46].dns-search property with a name server. """ for family in ["ipv4", "ipv6"]: - if self.config.has_section(family): + if ( + self.config.has_section(family) + and self._get_config_option(family, "method") != "disabled" + ): self._set_default(family, "dns-search", "") self.config[family]["dns-search"] = ( self.config[family]["dns-search"] diff --git a/tests/unittests/net/test_network_manager.py b/tests/unittests/net/test_network_manager.py new file mode 100644 index 00000000000..2aa476d7d15 --- /dev/null +++ b/tests/unittests/net/test_network_manager.py @@ -0,0 +1,323 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import textwrap +from unittest import mock + +import yaml + +from cloudinit.net import network_manager, network_state +from tests.unittests.helpers import dir2dict + + +def assert_equal_dict(expected_d, found_d): + for p, c in expected_d.items(): + if p not in found_d: + continue + assert c == found_d[p] + + +class TestNetworkManagerRenderNetworkState: + def _parse_network_state_from_config(self, config): + with mock.patch("cloudinit.net.network_state.get_interfaces_by_mac"): + config = yaml.safe_load(config) + return network_state.parse_net_config_data(config) + + def test_bond_dns_baseline(self, tmpdir): + + config = textwrap.dedent( + """\ + version: 1 + config: + - mac_address: 'xx:xx:xx:xx:xx:00' + mtu: 9000 + name: ens1f0np0 + subnets: [] + type: physical + - mac_address: 'xx:xx:xx:xx:xx:01' + mtu: 9000 + name: ens1f1np1 + subnets: [] + type: physical + - bond_interfaces: + - ens1f0np0 + - ens1f1np1 + mac_address: 'xx:xx:xx:xx:xx:00' + mtu: 9000 + name: bond0 + params: + bond-miimon: 100 + bond-mode: 802.3ad + bond-xmit_hash_policy: layer3+4 + subnets: [] + type: bond + - name: bond0.123 + subnets: + - address: 0.0.0.0 + ipv4: true + netmask: 255.255.255.0 + prefix: 24 + routes: + - gateway: 0.0.0.1 + netmask: 0.0.0.0 + network: 0.0.0.0 + type: static + type: vlan + vlan_id: 123 + vlan_link: bond0 + - address: 1.1.1.1 + search: hostname1 + type: nameserver + """ + ) + + expected_config = { + "/etc/NetworkManager/system-connections/cloud-init-ens1f0np0.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init ens1f0np0 + uuid=99c4bf6c-1691-53c4-bfe8-abdcb90b278a + autoconnect-priority=120 + type=ethernet + slave-type=bond + master=54317911-f840-516b-a10d-82cb4c1f075c + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + mtu=9000 + mac-address=XX:XX:XX:XX:XX:00 + + """ + ), + "/etc/NetworkManager/system-connections/cloud-init-ens1f1np1.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init ens1f1np1 + uuid=2685ec2b-1c26-583d-a660-0ab24201fef3 + autoconnect-priority=120 + type=ethernet + slave-type=bond + master=54317911-f840-516b-a10d-82cb4c1f075c + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + mtu=9000 + mac-address=XX:XX:XX:XX:XX:01 + + """ + ), + "/etc/NetworkManager/system-connections/cloud-init-bond0.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init bond0 + uuid=54317911-f840-516b-a10d-82cb4c1f075c + autoconnect-priority=120 + type=bond + interface-name=bond0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [bond] + mode=802.3ad + + [ethernet] + mtu=9000 + + """ + ), + "/etc/NetworkManager/system-connections/cloud-init-bond0.123.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init bond0.123 + uuid=7541e7a5-450b-570b-b3e8-a7f9eced114a + autoconnect-priority=120 + type=vlan + interface-name=bond0.123 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [vlan] + id=123 + parent=54317911-f840-516b-a10d-82cb4c1f075c + + [ipv4] + method=manual + may-fail=false + address1=0.0.0.0/24 + route1=0.0.0.0/0,0.0.0.1 + dns=1.1.1.1; + dns-search=hostname1; + + """ + ), + } + with mock.patch("cloudinit.net.get_interfaces_by_mac"): + ns = self._parse_network_state_from_config(config) + target = str(tmpdir) + network_manager.Renderer().render_network_state(ns, target=target) + rendered_content = dir2dict(target) + assert_equal_dict(expected_config, rendered_content) + + def test_bond_dns_redacted_with_method_disabled(self, tmpdir): + + config = textwrap.dedent( + """\ + version: 1 + config: + - mac_address: 'xx:xx:xx:xx:xx:00' + mtu: 9000 + name: ens1f0np0 + subnets: [] + type: physical + - mac_address: 'xx:xx:xx:xx:xx:01' + mtu: 9000 + name: ens1f1np1 + subnets: [] + type: physical + - bond_interfaces: + - ens1f0np0 + - ens1f1np1 + mac_address: 'xx:xx:xx:xx:xx:00' + mtu: 9000 + name: bond0 + params: + bond-miimon: 100 + bond-mode: 802.3ad + bond-xmit_hash_policy: layer3+4 + subnets: [] + type: bond + - name: bond0.123 + subnets: + - address: 0.0.0.0 + ipv4: true + netmask: 255.255.255.0 + prefix: 24 + routes: + - gateway: 0.0.0.1 + netmask: 0.0.0.0 + network: 0.0.0.0 + type: ipv6_slaac # !! to force ipvx.method to be disabled + type: vlan + vlan_id: 123 + vlan_link: bond0 + - address: 1.1.1.1 + search: hostname1 + type: nameserver + """ + ) + + expected_config = { + "/etc/NetworkManager/system-connections/cloud-init-ens1f0np0.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init ens1f0np0 + uuid=99c4bf6c-1691-53c4-bfe8-abdcb90b278a + autoconnect-priority=120 + type=ethernet + slave-type=bond + master=54317911-f840-516b-a10d-82cb4c1f075c + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + mtu=9000 + mac-address=XX:XX:XX:XX:XX:00 + + """ + ), + "/etc/NetworkManager/system-connections/cloud-init-ens1f1np1.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init ens1f1np1 + uuid=2685ec2b-1c26-583d-a660-0ab24201fef3 + autoconnect-priority=120 + type=ethernet + slave-type=bond + master=54317911-f840-516b-a10d-82cb4c1f075c + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + mtu=9000 + mac-address=XX:XX:XX:XX:XX:01 + + """ + ), + "/etc/NetworkManager/system-connections/cloud-init-bond0.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init bond0 + uuid=54317911-f840-516b-a10d-82cb4c1f075c + autoconnect-priority=120 + type=bond + interface-name=bond0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [bond] + mode=802.3ad + + [ethernet] + mtu=9000 + + """ + ), + "/etc/NetworkManager/system-connections/cloud-init-bond0.123.nmconnection": textwrap.dedent( # noqa: E501 + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init bond0.123 + uuid=7541e7a5-450b-570b-b3e8-a7f9eced114a + autoconnect-priority=120 + type=vlan + interface-name=bond0.123 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [vlan] + id=123 + parent=54317911-f840-516b-a10d-82cb4c1f075c + + [ipv6] + method=auto + may-fail=false + address1=0.0.0.0/24 + dns-search=hostname1; + + [ipv4] + method=disabled + route1=0.0.0.0/0,0.0.0.1 + + """ + ), + } + with mock.patch("cloudinit.net.get_interfaces_by_mac"): + ns = self._parse_network_state_from_config(config) + target = str(tmpdir) + network_manager.Renderer().render_network_state(ns, target=target) + rendered_content = dir2dict(target) + assert_equal_dict(expected_config, rendered_content) From 658d1841f7ed6b2397d2d9328c2d143150c8b9f8 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 18 Jul 2024 00:38:45 -0600 Subject: [PATCH 66/75] doc(OFV): Document how to configure cloud-init (#5519) --- doc/rtd/reference/datasources/ovf.rst | 28 ++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/rtd/reference/datasources/ovf.rst b/doc/rtd/reference/datasources/ovf.rst index a233df13a78..0ee33d0b821 100644 --- a/doc/rtd/reference/datasources/ovf.rst +++ b/doc/rtd/reference/datasources/ovf.rst @@ -3,9 +3,35 @@ OVF *** -The OVF datasource provides a datasource for reading data from an +The OVF datasource provides a generic datasource for reading data from an `Open Virtualization Format`_ ISO transport. +What platforms support OVF +-------------------------- + +OFV is an open standard which is supported by various virtualization +platforms, including (but not limited to): + +GCP +OpenShift +Proxmox +vSphere +VirtualBox +Xen + +While these (and many more) platforms support OVF, in some cases cloud-init +has alternative datasources which provide better platform integration. +Make sure to check whether another datasource is exists which is specific to +your platform of choice before trying to use OVF. + +Configuration +------------- + +Cloud-init gets configurations from an OVF XML file. User-data and network +configuration are provided by properties in the XML which contain key / value +pairs. The user-data is provided by a key named ``user-data``, and network +configuration is provided by a key named ``network-config``. + Graceful rpctool fallback ------------------------- From 0b4084374440d2a5a9968129e0460a1a009d9830 Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Thu, 18 Jul 2024 13:36:39 +0530 Subject: [PATCH 67/75] Support setting mirrorlist in yum repository config (#5522) 'mirrorlist' config can be specified instead or along with 'baseurl' in the yum repository config. Add support for specifying mirrorlist instead of 'baseurl'. Fixes GH-5520 Signed-off-by: Ani Sinha --- cloudinit/config/cc_yum_add_repo.py | 2 +- .../schemas/schema-cloud-config-v1.json | 10 +++++ doc/examples/cloud-config-yum-repo.txt | 3 +- .../unittests/config/test_cc_yum_add_repo.py | 40 ++++++++++++++++++- 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 9a717af3d1a..548c83bab6d 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -141,7 +141,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: n_repo_config[k] = v repo_config = n_repo_config missing_required = 0 - req_fields = ["baseurl", "metalink"] + req_fields = ["baseurl", "metalink", "mirrorlist"] for req_field in req_fields: if req_field not in repo_config: missing_required += 1 diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index e45504d4113..253e90b8c55 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -3457,6 +3457,11 @@ "format": "uri", "description": "Specifies a URL to a metalink file for the repomd.xml" }, + "mirrorlist": { + "type": "string", + "format": "uri", + "description": "Specifies a URL to a file containing a baseurls list" + }, "name": { "type": "string", "description": "Optional human-readable name of the yum repo." @@ -3494,6 +3499,11 @@ "required": [ "metalink" ] + }, + { + "required": [ + "mirrorlist" + ] } ] } diff --git a/doc/examples/cloud-config-yum-repo.txt b/doc/examples/cloud-config-yum-repo.txt index 6a4037e2462..cee26677b49 100644 --- a/doc/examples/cloud-config-yum-repo.txt +++ b/doc/examples/cloud-config-yum-repo.txt @@ -11,9 +11,10 @@ yum_repos: # Any repository configuration options # See: man yum.conf # - # At least one of 'baseurl' or 'metalink' is required! + # At least one of 'baseurl' or 'metalink' or 'mirrorlist' is required! baseurl: http://download.fedoraproject.org/pub/epel/testing/5/$basearch metalink: https://mirrors.fedoraproject.org/metalink?repo=epel-$releasever&arch=$basearch&infra=$infra&content=$contentdir + mirrorlist: https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever& enabled: false failovermethod: priority gpgcheck: true diff --git a/tests/unittests/config/test_cc_yum_add_repo.py b/tests/unittests/config/test_cc_yum_add_repo.py index e6a9109ee19..c77262f508f 100644 --- a/tests/unittests/config/test_cc_yum_add_repo.py +++ b/tests/unittests/config/test_cc_yum_add_repo.py @@ -31,7 +31,8 @@ def test_bad_config(self): "yum_repos": { "epel-testing": { "name": "Extra Packages for Enterprise Linux 5 - Testing", - # At least one of baseurl or metalink must be present. + # At least one of baseurl or metalink or mirrorlist + # must be present. # Missing this should cause the repo not to be written # 'baseurl': 'http://blah.org/pub/epel/testing/5/$barch', "enabled": False, @@ -84,6 +85,43 @@ def test_metalink_config(self): for k, v in expected[section].items(): self.assertEqual(parser.get(section, k), v) + def test_mirrorlist_config(self): + cfg = { + "yum_repos": { + "epel-testing": { + "name": "Extra Packages for Enterprise Linux 5 - Testing", + "mirrorlist": "http://mirrors.blah.org/metalink?repo=rhel-$releasever", + "enabled": False, + "gpgcheck": True, + "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL", + "failovermethod": "priority", + }, + }, + } + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + cc_yum_add_repo.handle("yum_add_repo", cfg, None, []) + contents = util.load_text_file("/etc/yum.repos.d/epel-testing.repo") + parser = configparser.ConfigParser() + parser.read_string(contents) + expected = { + "epel-testing": { + "name": "Extra Packages for Enterprise Linux 5 - Testing", + "failovermethod": "priority", + "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL", + "enabled": "0", + "mirrorlist": "http://mirrors.blah.org/metalink?repo=rhel-$releasever", + "gpgcheck": "1", + } + } + for section in expected: + self.assertTrue( + parser.has_section(section), + "Contains section {0}".format(section), + ) + for k, v in expected[section].items(): + self.assertEqual(parser.get(section, k), v) + def test_write_config(self): cfg = { "yum_repos": { From 550c685c98551f65c30832b186fe091721b48477 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 18 Jul 2024 09:04:54 -0400 Subject: [PATCH 68/75] fix: Clean cache if no datasource fallback (#5499) 9929a00 added the ability to used a cached datasource when none is found. This was supposed to be per-datasource, but the lack of cache cleaning got applied universally. This commit makes it so cache will be cleaned as it was before if fallback isn't implemented in datasource. Fixes GH-5486 --- cloudinit/stages.py | 1 + .../assets/DataSourceNoCacheNetworkOnly.py | 23 ++++ .../assets/DataSourceNoCacheWithFallback.py | 29 +++++ .../datasources/test_caching.py | 115 ++++++++++++++++++ tests/integration_tests/instances.py | 4 +- 5 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/assets/DataSourceNoCacheNetworkOnly.py create mode 100644 tests/integration_tests/assets/DataSourceNoCacheWithFallback.py create mode 100644 tests/integration_tests/datasources/test_caching.py diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 52876e72434..872905e39d1 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -406,6 +406,7 @@ def _get_data_source(self, existing) -> sources.DataSource: ds, ) else: + util.del_file(self.paths.instance_link) raise e self.datasource = ds # Ensure we adjust our path members datasource diff --git a/tests/integration_tests/assets/DataSourceNoCacheNetworkOnly.py b/tests/integration_tests/assets/DataSourceNoCacheNetworkOnly.py new file mode 100644 index 00000000000..54a7bab3437 --- /dev/null +++ b/tests/integration_tests/assets/DataSourceNoCacheNetworkOnly.py @@ -0,0 +1,23 @@ +import logging + +from cloudinit import sources + +LOG = logging.getLogger(__name__) + + +class DataSourceNoCacheNetworkOnly(sources.DataSource): + def _get_data(self): + LOG.debug("TEST _get_data called") + return True + + +datasources = [ + ( + DataSourceNoCacheNetworkOnly, + (sources.DEP_FILESYSTEM, sources.DEP_NETWORK), + ), +] + + +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/tests/integration_tests/assets/DataSourceNoCacheWithFallback.py b/tests/integration_tests/assets/DataSourceNoCacheWithFallback.py new file mode 100644 index 00000000000..fdfc473f8a5 --- /dev/null +++ b/tests/integration_tests/assets/DataSourceNoCacheWithFallback.py @@ -0,0 +1,29 @@ +import logging +import os + +from cloudinit import sources + +LOG = logging.getLogger(__name__) + + +class DataSourceNoCacheWithFallback(sources.DataSource): + def _get_data(self): + if os.path.exists("/ci-test-firstboot"): + LOG.debug("TEST _get_data called") + return True + return False + + def check_if_fallback_is_allowed(self): + return True + + +datasources = [ + ( + DataSourceNoCacheWithFallback, + (sources.DEP_FILESYSTEM,), + ), +] + + +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/tests/integration_tests/datasources/test_caching.py b/tests/integration_tests/datasources/test_caching.py new file mode 100644 index 00000000000..33e4b671c28 --- /dev/null +++ b/tests/integration_tests/datasources/test_caching.py @@ -0,0 +1,115 @@ +import pytest + +from tests.integration_tests import releases, util +from tests.integration_tests.instances import IntegrationInstance + + +def setup_custom_datasource(client: IntegrationInstance, datasource_name: str): + client.write_to_file( + "/etc/cloud/cloud.cfg.d/99-imds.cfg", + f"datasource_list: [ {datasource_name}, None ]\n" + "datasource_pkg_list: [ cisources ]", + ) + assert client.execute( + "mkdir -p /usr/lib/python3/dist-packages/cisources" + ) + client.push_file( + util.ASSETS_DIR / f"DataSource{datasource_name}.py", + "/usr/lib/python3/dist-packages/cisources/" + f"DataSource{datasource_name}.py", + ) + + +def verify_no_cache_boot(client: IntegrationInstance): + log = client.read_from_file("/var/log/cloud-init.log") + util.verify_ordered_items_in_text( + [ + "No local datasource found", + "running 'init'", + "no cache found", + "Detected platform", + "TEST _get_data called", + ], + text=log, + ) + util.verify_clean_boot(client) + + +@pytest.mark.skipif( + not releases.IS_UBUNTU, + reason="hardcoded dist-packages directory", +) +def test_no_cache_network_only(client: IntegrationInstance): + """Test cache removal per boot. GH-5486 + + This tests the CloudStack password reset use case. The expectation is: + - Metadata is fetched in network timeframe only + - Because `check_instance_id` is not defined, no cached datasource + is found in the init-local phase, but the cache is used in the + remaining phases due to existance of /run/cloud-init/.instance-id + - Because `check_if_fallback_is_allowed` is not defined, cloud-init + does NOT fall back to the pickled datasource, and will + instead delete the cache during the init-local phase + - Metadata is therefore fetched every boot in the network phase + """ + setup_custom_datasource(client, "NoCacheNetworkOnly") + + # Run cloud-init as if first boot + assert client.execute("cloud-init clean --logs") + client.restart() + + verify_no_cache_boot(client) + + # Clear the log without clean and run cloud-init for subsequent boot + assert client.execute("echo '' > /var/log/cloud-init.log") + client.restart() + + verify_no_cache_boot(client) + + +@pytest.mark.skipif( + not releases.IS_UBUNTU, + reason="hardcoded dist-packages directory", +) +def test_no_cache_with_fallback(client: IntegrationInstance): + """Test we use fallback when defined and no cache available.""" + setup_custom_datasource(client, "NoCacheWithFallback") + + # Run cloud-init as if first boot + assert client.execute("cloud-init clean --logs") + # Used by custom datasource + client.execute("touch /ci-test-firstboot") + client.restart() + + log = client.read_from_file("/var/log/cloud-init.log") + util.verify_ordered_items_in_text( + [ + "no cache found", + "Detected platform", + "TEST _get_data called", + "running 'init'", + "restored from cache with run check", + "running 'modules:config'", + ], + text=log, + ) + util.verify_clean_boot(client) + + # Clear the log without clean and run cloud-init for subsequent boot + assert client.execute("echo '' > /var/log/cloud-init.log") + client.execute("rm /ci-test-firstboot") + client.restart() + + log = client.read_from_file("/var/log/cloud-init.log") + util.verify_ordered_items_in_text( + [ + "cache invalid in datasource", + "Detected platform", + "Restored fallback datasource from checked cache", + "running 'init'", + "restored from cache with run check", + "running 'modules:config'", + ], + text=log, + ) + util.verify_clean_boot(client) diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index cfae37c272c..1c8344ab916 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -106,7 +106,9 @@ def push_file( # First push to a temporary directory because of permissions issues tmp_path = _get_tmp_path() self.instance.push_file(str(local_path), tmp_path) - assert self.execute("mv {} {}".format(tmp_path, str(remote_path))).ok + assert self.execute( + "mv {} {}".format(tmp_path, str(remote_path)) + ), f"Failed to push {tmp_path} to {remote_path}" def read_from_file(self, remote_path) -> str: result = self.execute("cat {}".format(remote_path)) From 57d130eeb75f4652cd92d5605949cdc334113239 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 18 Jul 2024 14:01:25 -0600 Subject: [PATCH 69/75] chore(formatting): fix squashed commit test formatting (#5524) --- tests/integration_tests/datasources/test_caching.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration_tests/datasources/test_caching.py b/tests/integration_tests/datasources/test_caching.py index 33e4b671c28..f043ddbe779 100644 --- a/tests/integration_tests/datasources/test_caching.py +++ b/tests/integration_tests/datasources/test_caching.py @@ -10,9 +10,7 @@ def setup_custom_datasource(client: IntegrationInstance, datasource_name: str): f"datasource_list: [ {datasource_name}, None ]\n" "datasource_pkg_list: [ cisources ]", ) - assert client.execute( - "mkdir -p /usr/lib/python3/dist-packages/cisources" - ) + assert client.execute("mkdir -p /usr/lib/python3/dist-packages/cisources") client.push_file( util.ASSETS_DIR / f"DataSource{datasource_name}.py", "/usr/lib/python3/dist-packages/cisources/" From b0a673a53dd7ea554a9fe197dc4066fec21c53c4 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 19 Jun 2024 14:41:28 -0500 Subject: [PATCH 70/75] feat: Add trace-level logger (#5414) This is useful for logs we want hidden by default but can be turned on via configuration. --- cloudinit/log.py | 37 ++++++++++++++++++++++++++++++------- tests/unittests/test_log.py | 12 +++++++++++- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/cloudinit/log.py b/cloudinit/log.py index 08d0efa3001..61b96262aa1 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -23,6 +23,23 @@ DEFAULT_LOG_FORMAT = "%(asctime)s - %(filename)s[%(levelname)s]: %(message)s" DEPRECATED = 35 +TRACE = logging.DEBUG - 5 + + +class CustomLoggerType(logging.Logger): + """A hack to get mypy to stop complaining about custom logging methods. + + When using deprecated or trace logging, rather than: + LOG = logging.getLogger(__name__) + Instead do: + LOG = cast(CustomLoggerType, logging.getLogger(__name__)) + """ + + def trace(self, *args, **kwargs): + pass + + def deprecated(self, *args, **kwargs): + pass def setup_basic_logging(level=logging.DEBUG, formatter=None): @@ -45,14 +62,20 @@ def flush_loggers(root): flush_loggers(root.parent) -def define_deprecation_logger(lvl=DEPRECATED): - logging.addLevelName(lvl, "DEPRECATED") +def define_extra_loggers() -> None: + """Add DEPRECATED and TRACE log levels to the logging module.""" + + def new_logger(level): + def log_at_level(self, message, *args, **kwargs): + if self.isEnabledFor(level): + self._log(level, message, args, **kwargs) - def deprecated(self, message, *args, **kwargs): - if self.isEnabledFor(lvl): - self._log(lvl, message, args, **kwargs) + return log_at_level - logging.Logger.deprecated = deprecated + logging.addLevelName(DEPRECATED, "DEPRECATED") + logging.addLevelName(TRACE, "TRACE") + setattr(logging.Logger, "deprecated", new_logger(DEPRECATED)) + setattr(logging.Logger, "trace", new_logger(TRACE)) def setup_logging(cfg=None): @@ -183,7 +206,7 @@ def configure_root_logger(): # Always format logging timestamps as UTC time logging.Formatter.converter = time.gmtime - define_deprecation_logger() + define_extra_loggers() setup_backup_logging() reset_logging() diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 87996310349..175afc0eb94 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -6,6 +6,7 @@ import io import logging import time +from typing import cast import pytest @@ -63,10 +64,18 @@ def test_logger_uses_gmtime(self): class TestDeprecatedLogs: def test_deprecated_log_level(self, caplog): - logging.getLogger().deprecated("deprecated message") + logger = cast(log.CustomLoggerType, logging.getLogger()) + logger.deprecated("deprecated message") assert "DEPRECATED" == caplog.records[0].levelname assert "deprecated message" in caplog.text + def test_trace_log_level(self, caplog): + logger = cast(log.CustomLoggerType, logging.getLogger()) + logger.setLevel(logging.NOTSET) + logger.trace("trace message") + assert "TRACE" == caplog.records[0].levelname + assert "trace message" in caplog.text + @pytest.mark.parametrize( "expected_log_level, deprecation_info_boundary", ( @@ -115,6 +124,7 @@ def test_deprecate_log_level_based_on_features( ) def test_log_deduplication(self, caplog): + log.define_extra_loggers() util.deprecate( deprecated="stuff", deprecated_version="19.1", From 8ec2f64ad69439580ed732e6436b6a9e420e00fe Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 19 Jun 2024 15:04:34 -0500 Subject: [PATCH 71/75] refactor: replace verbosity with log levels in logs.py (#5414) --- cloudinit/cmd/devel/logs.py | 116 +++++++++++++------------ tests/unittests/cmd/devel/test_logs.py | 27 ++---- 2 files changed, 66 insertions(+), 77 deletions(-) diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index a1e4eb8dfad..d3600d2ef05 100755 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -7,14 +7,17 @@ """Define 'collect-logs' utility and handler to include in cloud-init cmd.""" import argparse +import logging import os +import pathlib import shutil import subprocess import sys from datetime import datetime, timezone from pathlib import Path -from typing import NamedTuple, Optional +from typing import NamedTuple, Optional, cast +from cloudinit import log from cloudinit.cmd.devel import read_cfg_paths from cloudinit.stages import Init from cloudinit.subp import ProcessExecutionError, subp @@ -27,6 +30,8 @@ write_file, ) +LOG = cast(log.CustomLoggerType, logging.getLogger(__name__)) + class LogPaths(NamedTuple): userdata_raw: str @@ -152,111 +157,103 @@ def _get_copytree_ignore_files(paths: LogPaths): return ignored_files -def _write_command_output_to_file(cmd, filename, msg, verbosity): +def _write_command_output_to_file(cmd, filename, msg): """Helper which runs a command and writes output or error to filename.""" ensure_dir(os.path.dirname(filename)) try: output = subp(cmd).stdout except ProcessExecutionError as e: write_file(filename, str(e)) - _debug("collecting %s failed.\n" % msg, 1, verbosity) + LOG.debug("collecting %s failed.", msg) else: write_file(filename, output) - _debug("collected %s\n" % msg, 1, verbosity) + LOG.debug("collected %s", msg) return output -def _stream_command_output_to_file(cmd, filename, msg, verbosity): - """Helper which runs a command and writes output or error to filename.""" +def _stream_command_output_to_file(cmd, filename, msg): + """Helper which runs a command and writes output or error to filename. + + `subprocess.call` is invoked directly here to stream output to the file. + Otherwise memory usage can be high for large outputs. + """ ensure_dir(os.path.dirname(filename)) try: with open(filename, "w") as f: subprocess.call(cmd, stdout=f, stderr=f) # nosec B603 except OSError as e: write_file(filename, str(e)) - _debug("collecting %s failed.\n" % msg, 1, verbosity) + LOG.debug("collecting %s failed.", msg) else: - _debug("collected %s\n" % msg, 1, verbosity) - - -def _debug(msg, level, verbosity): - if level <= verbosity: - sys.stderr.write(msg) + LOG.debug("collected %s", msg) -def _collect_file(path, out_dir, verbosity): +def _collect_file(path: str, out_dir: str) -> None: if os.path.isfile(path): copy(path, out_dir) - _debug("collected file: %s\n" % path, 1, verbosity) + LOG.debug("collected file: %s", path) else: - _debug("file %s did not exist\n" % path, 2, verbosity) + LOG.trace("file %s did not exist", path) -def _collect_installer_logs( - log_dir: str, include_userdata: bool, verbosity: int -): +def _collect_installer_logs(log_dir: str, include_userdata: bool) -> None: """Obtain subiquity logs and config files.""" for src_file in INSTALLER_APPORT_FILES: destination_dir = Path(log_dir + src_file.path).parent if not destination_dir.exists(): ensure_dir(str(destination_dir)) - _collect_file(src_file.path, str(destination_dir), verbosity) + _collect_file(src_file.path, str(destination_dir)) if include_userdata: for src_file in INSTALLER_APPORT_SENSITIVE_FILES: destination_dir = Path(log_dir + src_file.path).parent if not destination_dir.exists(): ensure_dir(str(destination_dir)) - _collect_file(src_file.path, str(destination_dir), verbosity) + _collect_file(src_file.path, str(destination_dir)) -def _collect_version_info(log_dir: str, verbosity: int): +def _collect_version_info(log_dir: str) -> None: version = _write_command_output_to_file( cmd=["cloud-init", "--version"], filename=os.path.join(log_dir, "version"), msg="cloud-init --version", - verbosity=verbosity, ) dpkg_ver = _write_command_output_to_file( cmd=["dpkg-query", "--show", "-f=${Version}\n", "cloud-init"], filename=os.path.join(log_dir, "dpkg-version"), msg="dpkg version", - verbosity=verbosity, ) if not version: version = dpkg_ver if dpkg_ver else "not-available" - _debug("collected cloud-init version: %s\n" % version, 1, verbosity) + LOG.debug("collected cloud-init version: %s", version) -def _collect_system_logs(log_dir: str, verbosity: int): +def _collect_system_logs(log_dir: str) -> None: _stream_command_output_to_file( cmd=["dmesg"], filename=os.path.join(log_dir, "dmesg.txt"), msg="dmesg output", - verbosity=verbosity, ) _stream_command_output_to_file( cmd=["journalctl", "--boot=0", "-o", "short-precise"], filename=os.path.join(log_dir, "journal.txt"), msg="systemd journal of current boot", - verbosity=verbosity, ) def _collect_cloudinit_logs( log_dir: str, - verbosity: int, init: Init, paths: LogPaths, include_userdata: bool, -): - for log in get_config_logfiles(init.cfg): - _collect_file(log, log_dir, verbosity) +) -> None: + for logfile in get_config_logfiles(init.cfg): + _collect_file(logfile, log_dir) if include_userdata: user_data_file = paths.userdata_raw - _collect_file(user_data_file, log_dir, verbosity) + _collect_file(user_data_file, log_dir) -def _collect_run_dir(log_dir: str, verbosity: int, paths: LogPaths): +def _collect_run_dir(log_dir: str, paths: LogPaths) -> None: run_dir = os.path.join(log_dir, "run") ensure_dir(run_dir) if os.path.exists(paths.run_dir): @@ -267,15 +264,10 @@ def _collect_run_dir(log_dir: str, verbosity: int, paths: LogPaths): ignore=lambda _, __: _get_copytree_ignore_files(paths), ) except shutil.Error as e: - sys.stderr.write("Failed collecting file(s) due to error:\n") - sys.stderr.write(str(e) + "\n") - _debug("collected dir %s\n" % paths.run_dir, 1, verbosity) + LOG.warning("Failed collecting file(s) due to error: %s", e) + LOG.debug("collected directory: %s", paths.run_dir) else: - _debug( - "directory '%s' did not exist\n" % paths.run_dir, - 1, - verbosity, - ) + LOG.debug("directory '%s' did not exist", paths.run_dir) if os.path.exists(os.path.join(paths.run_dir, "disabled")): # Fallback to grab previous cloud/data cloud_data_dir = Path(paths.cloud_data) @@ -286,9 +278,7 @@ def _collect_run_dir(log_dir: str, verbosity: int, paths: LogPaths): ) -def collect_logs( - tarfile: str, include_userdata: bool, verbosity: int = 0 -) -> int: +def collect_logs(tarfile: str, include_userdata: bool) -> int: """Collect all cloud-init logs and tar them up into the provided tarfile. @param tarfile: The path of the tar-gzipped file to create. @@ -296,9 +286,9 @@ def collect_logs( @return: 0 on success, 1 on failure. """ if include_userdata and os.getuid() != 0: - sys.stderr.write( - "To include userdata, root user is required." - " Try sudo cloud-init collect-logs\n" + LOG.error( + "To include userdata, root user is required. " + "Try sudo cloud-init collect-logs" ) return 1 @@ -312,25 +302,37 @@ def collect_logs( init.read_cfg() paths = get_log_paths(init) - _collect_version_info(log_dir, verbosity) - _collect_system_logs(log_dir, verbosity) - _collect_cloudinit_logs( - log_dir, verbosity, init, paths, include_userdata - ) - _collect_installer_logs(log_dir, include_userdata, verbosity) - _collect_run_dir(log_dir, verbosity, paths) + _collect_version_info(log_dir) + _collect_system_logs(log_dir) + _collect_cloudinit_logs(log_dir, init, paths, include_userdata) + _collect_installer_logs(log_dir, include_userdata) + _collect_run_dir(log_dir, paths) with chdir(tmp_dir): subp(["tar", "czvf", tarfile, log_dir.replace(f"{tmp_dir}/", "")]) - sys.stderr.write("Wrote %s\n" % tarfile) + LOG.info("Wrote %s", tarfile) return 0 +def _setup_logger(verbosity: int) -> None: + log.reset_logging() + if verbosity == 0: + level = logging.INFO + elif verbosity == 1: + level = logging.DEBUG + else: + level = log.TRACE + LOG.setLevel(level) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("%(message)s")) + LOG.addHandler(handler) + + def handle_collect_logs_args(name, args): """Handle calls to 'cloud-init collect-logs' as a subcommand.""" + _setup_logger(args.verbosity) return collect_logs( tarfile=args.tarfile, include_userdata=args.userdata, - verbosity=args.verbosity, ) diff --git a/tests/unittests/cmd/devel/test_logs.py b/tests/unittests/cmd/devel/test_logs.py index 7dfdfac6edc..f8385a4c3ef 100644 --- a/tests/unittests/cmd/devel/test_logs.py +++ b/tests/unittests/cmd/devel/test_logs.py @@ -4,7 +4,6 @@ import os import re from datetime import datetime -from io import StringIO import pytest @@ -21,22 +20,19 @@ @mock.patch("cloudinit.cmd.devel.logs.os.getuid") class TestCollectLogs: def test_collect_logs_with_userdata_requires_root_user( - self, m_getuid, tmpdir + self, m_getuid, tmpdir, caplog ): """collect-logs errors when non-root user collects userdata .""" m_getuid.return_value = 100 # non-root output_tarfile = tmpdir.join("logs.tgz") - with mock.patch("sys.stderr", new_callable=StringIO) as m_stderr: - assert 1 == logs.collect_logs( - output_tarfile, include_userdata=True - ) + assert 1 == logs.collect_logs(output_tarfile, include_userdata=True) assert ( "To include userdata, root user is required." - " Try sudo cloud-init collect-logs\n" == m_stderr.getvalue() + " Try sudo cloud-init collect-logs" in caplog.text ) def test_collect_logs_creates_tarfile( - self, m_getuid, m_log_paths, mocker, tmpdir + self, m_getuid, m_log_paths, mocker, tmpdir, caplog ): """collect-logs creates a tarfile with all related cloud-init info.""" m_getuid.return_value = 100 @@ -101,13 +97,10 @@ def fake_subprocess_call(cmd, stdout=None, stderr=None): ) stdout.write(expected_subp[cmd_tuple]) - fake_stderr = mock.MagicMock() - mocker.patch(M_PATH + "subp", side_effect=fake_subp) mocker.patch( M_PATH + "subprocess.call", side_effect=fake_subprocess_call ) - mocker.patch(M_PATH + "sys.stderr", fake_stderr) mocker.patch(M_PATH + "INSTALLER_APPORT_FILES", []) mocker.patch(M_PATH + "INSTALLER_APPORT_SENSITIVE_FILES", []) logs.collect_logs(output_tarfile, include_userdata=False) @@ -151,10 +144,10 @@ def fake_subprocess_call(cmd, stdout=None, stderr=None): assert "results" == load_text_file( os.path.join(out_logdir, "run", "cloud-init", "results.json") ) - fake_stderr.write.assert_any_call("Wrote %s\n" % output_tarfile) + assert f"Wrote {output_tarfile}" in caplog.text def test_collect_logs_includes_optional_userdata( - self, m_getuid, mocker, tmpdir, m_log_paths + self, m_getuid, mocker, tmpdir, m_log_paths, caplog ): """collect-logs include userdata when --include-userdata is set.""" m_getuid.return_value = 0 @@ -215,13 +208,10 @@ def fake_subprocess_call(cmd, stdout=None, stderr=None): ) stdout.write(expected_subp[cmd_tuple]) - fake_stderr = mock.MagicMock() - mocker.patch(M_PATH + "subp", side_effect=fake_subp) mocker.patch( M_PATH + "subprocess.call", side_effect=fake_subprocess_call ) - mocker.patch(M_PATH + "sys.stderr", fake_stderr) mocker.patch(M_PATH + "INSTALLER_APPORT_FILES", []) mocker.patch(M_PATH + "INSTALLER_APPORT_SENSITIVE_FILES", []) logs.collect_logs(output_tarfile, include_userdata=True) @@ -239,7 +229,7 @@ def fake_subprocess_call(cmd, stdout=None, stderr=None): m_log_paths.instance_data_sensitive.name, ) ) - fake_stderr.write.assert_any_call("Wrote %s\n" % output_tarfile) + assert f"Wrote {output_tarfile}" in caplog.text @pytest.mark.parametrize( "cmd, expected_file_contents, expected_return_value", @@ -278,7 +268,6 @@ def test_write_command_output_to_file( filename=output_file, cmd=cmd, msg="", - verbosity=1, ) assert expected_return_value == return_output @@ -301,7 +290,6 @@ def test_stream_command_output_to_file( filename=output_file, cmd=cmd, msg="", - verbosity=1, ) assert expected_file_contents == load_text_file(output_file) @@ -382,7 +370,6 @@ def test_include_installer_logs_when_present( logs._collect_installer_logs( log_dir=tmpdir.strpath, include_userdata=include_userdata, - verbosity=0, ) expect_userdata = bool(include_userdata and apport_sensitive_files) # when subiquity artifacts exist, and userdata set true, expect logs From 19c86ffb45ee3b1aa0034a447fa42028f0945da9 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 19 Jun 2024 15:17:39 -0500 Subject: [PATCH 72/75] refactor: logs.py pathlib changes (#5414) Switch to pathlib where appropriate and call consistently --- cloudinit/cmd/devel/logs.py | 48 +++++++++++++++----------- tests/unittests/cmd/devel/test_logs.py | 12 +++---- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index d3600d2ef05..a6b61fcb720 100755 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -14,8 +14,7 @@ import subprocess import sys from datetime import datetime, timezone -from pathlib import Path -from typing import NamedTuple, Optional, cast +from typing import List, NamedTuple, Optional, cast from cloudinit import log from cloudinit.cmd.devel import read_cfg_paths @@ -157,35 +156,42 @@ def _get_copytree_ignore_files(paths: LogPaths): return ignored_files -def _write_command_output_to_file(cmd, filename, msg): +def _write_command_output_to_file( + cmd: List[str], + file_path: pathlib.Path, + msg: str, +) -> Optional[str]: """Helper which runs a command and writes output or error to filename.""" - ensure_dir(os.path.dirname(filename)) + file_path.parent.mkdir(parents=True, exist_ok=True) try: output = subp(cmd).stdout except ProcessExecutionError as e: - write_file(filename, str(e)) + write_file(file_path, str(e)) LOG.debug("collecting %s failed.", msg) + output = None else: - write_file(filename, output) - LOG.debug("collected %s", msg) - return output + write_file(file_path, output) + LOG.debug("collected %s to file '%s'", msg, file_path.stem) + return output -def _stream_command_output_to_file(cmd, filename, msg): +def _stream_command_output_to_file( + cmd: List[str], file_path: pathlib.Path, msg: str +) -> None: """Helper which runs a command and writes output or error to filename. `subprocess.call` is invoked directly here to stream output to the file. Otherwise memory usage can be high for large outputs. """ - ensure_dir(os.path.dirname(filename)) + file_path.parent.mkdir(parents=True, exist_ok=True) try: - with open(filename, "w") as f: + with file_path.open("w") as f: subprocess.call(cmd, stdout=f, stderr=f) # nosec B603 except OSError as e: - write_file(filename, str(e)) + write_file(file_path, str(e)) LOG.debug("collecting %s failed.", msg) else: - LOG.debug("collected %s", msg) + LOG.debug("collected %s to file '%s'", msg, file_path.stem) def _collect_file(path: str, out_dir: str) -> None: @@ -199,13 +205,13 @@ def _collect_file(path: str, out_dir: str) -> None: def _collect_installer_logs(log_dir: str, include_userdata: bool) -> None: """Obtain subiquity logs and config files.""" for src_file in INSTALLER_APPORT_FILES: - destination_dir = Path(log_dir + src_file.path).parent + destination_dir = pathlib.Path(log_dir + src_file.path).parent if not destination_dir.exists(): ensure_dir(str(destination_dir)) _collect_file(src_file.path, str(destination_dir)) if include_userdata: for src_file in INSTALLER_APPORT_SENSITIVE_FILES: - destination_dir = Path(log_dir + src_file.path).parent + destination_dir = pathlib.Path(log_dir + src_file.path).parent if not destination_dir.exists(): ensure_dir(str(destination_dir)) _collect_file(src_file.path, str(destination_dir)) @@ -214,12 +220,12 @@ def _collect_installer_logs(log_dir: str, include_userdata: bool) -> None: def _collect_version_info(log_dir: str) -> None: version = _write_command_output_to_file( cmd=["cloud-init", "--version"], - filename=os.path.join(log_dir, "version"), + file_path=pathlib.Path(log_dir, "version"), msg="cloud-init --version", ) dpkg_ver = _write_command_output_to_file( cmd=["dpkg-query", "--show", "-f=${Version}\n", "cloud-init"], - filename=os.path.join(log_dir, "dpkg-version"), + file_path=pathlib.Path(log_dir, "dpkg-version"), msg="dpkg version", ) if not version: @@ -230,12 +236,12 @@ def _collect_version_info(log_dir: str) -> None: def _collect_system_logs(log_dir: str) -> None: _stream_command_output_to_file( cmd=["dmesg"], - filename=os.path.join(log_dir, "dmesg.txt"), + file_path=pathlib.Path(log_dir, "dmesg.txt"), msg="dmesg output", ) _stream_command_output_to_file( cmd=["journalctl", "--boot=0", "-o", "short-precise"], - filename=os.path.join(log_dir, "journal.txt"), + file_path=pathlib.Path(log_dir, "journal.txt"), msg="systemd journal of current boot", ) @@ -270,11 +276,11 @@ def _collect_run_dir(log_dir: str, paths: LogPaths) -> None: LOG.debug("directory '%s' did not exist", paths.run_dir) if os.path.exists(os.path.join(paths.run_dir, "disabled")): # Fallback to grab previous cloud/data - cloud_data_dir = Path(paths.cloud_data) + cloud_data_dir = pathlib.Path(paths.cloud_data) if cloud_data_dir.exists(): shutil.copytree( str(cloud_data_dir), - Path(log_dir + str(cloud_data_dir)), + pathlib.Path(log_dir + str(cloud_data_dir)), ) diff --git a/tests/unittests/cmd/devel/test_logs.py b/tests/unittests/cmd/devel/test_logs.py index f8385a4c3ef..b1d9f585d30 100644 --- a/tests/unittests/cmd/devel/test_logs.py +++ b/tests/unittests/cmd/devel/test_logs.py @@ -256,17 +256,17 @@ def fake_subprocess_call(cmd, stdout=None, stderr=None): def test_write_command_output_to_file( self, m_getuid, - tmpdir, + tmp_path, cmd, expected_file_contents, expected_return_value, ): m_getuid.return_value = 100 - output_file = tmpdir.join("test-output-file.txt") + output_file = tmp_path / "test-output-file.txt" return_output = logs._write_command_output_to_file( - filename=output_file, cmd=cmd, + file_path=output_file, msg="", ) @@ -281,14 +281,14 @@ def test_write_command_output_to_file( ], ) def test_stream_command_output_to_file( - self, m_getuid, tmpdir, cmd, expected_file_contents + self, m_getuid, tmp_path, cmd, expected_file_contents ): m_getuid.return_value = 100 - output_file = tmpdir.join("test-output-file.txt") + output_file = tmp_path / "test-output-file.txt" logs._stream_command_output_to_file( - filename=output_file, cmd=cmd, + file_path=output_file, msg="", ) From 6e4153b346bc0d3f3422c01a3f93ecfb28269da2 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 19 Jun 2024 15:18:12 -0500 Subject: [PATCH 73/75] refactor: logs.py add typing and small misc refactors (#5414) --- cloudinit/cmd/devel/logs.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index a6b61fcb720..f5ae53ce26a 100755 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -100,7 +100,9 @@ class ApportFile(NamedTuple): ] -def get_parser(parser=None): +def get_parser( + parser: Optional[argparse.ArgumentParser] = None, +) -> argparse.ArgumentParser: """Build or extend and arg parser for collect-logs utility. @param parser: Optional existing ArgumentParser instance representing the @@ -145,7 +147,7 @@ def get_parser(parser=None): return parser -def _get_copytree_ignore_files(paths: LogPaths): +def _get_copytree_ignore_files(paths: LogPaths) -> List[str]: """Return a list of files to ignore for /run/cloud-init directory""" ignored_files = [ "hook-hotplug-cmd", # named pipe for hotplug @@ -229,8 +231,7 @@ def _collect_version_info(log_dir: str) -> None: msg="dpkg version", ) if not version: - version = dpkg_ver if dpkg_ver else "not-available" - LOG.debug("collected cloud-init version: %s", version) + version = dpkg_ver or "not-available" def _collect_system_logs(log_dir: str) -> None: @@ -333,7 +334,7 @@ def _setup_logger(verbosity: int) -> None: LOG.addHandler(handler) -def handle_collect_logs_args(name, args): +def handle_collect_logs_args(_name: str, args: argparse.Namespace) -> int: """Handle calls to 'cloud-init collect-logs' as a subcommand.""" _setup_logger(args.verbosity) return collect_logs( @@ -342,11 +343,5 @@ def handle_collect_logs_args(name, args): ) -def main(): - """Tool to collect and tar all cloud-init related logs.""" - parser = get_parser() - return handle_collect_logs_args("collect-logs", parser.parse_args()) - - if __name__ == "__main__": - sys.exit(main()) + sys.exit(handle_collect_logs_args("", get_parser().parse_args())) From 669f8f400f8be83f25522d9256838ba9d376fb61 Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Fri, 19 Jul 2024 17:09:32 +0200 Subject: [PATCH 74/75] update changelog (new upstream snapshot) --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 278ba7ec2f5..f7a2ac28f2f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +cloud-init (24.3~1g6e4153b3-0ubuntu1) UNRELEASED; urgency=medium + + * Upstream snapshot based on upstream/main at 6e4153b3. + - Bugs fixed in this snapshot: (LP: #2065787) + + -- Alberto Contreras Fri, 19 Jul 2024 17:09:32 +0200 + cloud-init (24.2~5g8c396a4b-0ubuntu1) oracular; urgency=medium * Upstream snapshot based on upstream/main at 8c396a4b. From 87c915f7901f16b719d6c5b92b6beedce6224245 Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Fri, 19 Jul 2024 17:10:02 +0200 Subject: [PATCH 75/75] releasing cloud-init version 24.3~1g6e4153b3-0ubuntu1 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index f7a2ac28f2f..77cf2331990 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ -cloud-init (24.3~1g6e4153b3-0ubuntu1) UNRELEASED; urgency=medium +cloud-init (24.3~1g6e4153b3-0ubuntu1) oracular; urgency=medium * Upstream snapshot based on upstream/main at 6e4153b3. - Bugs fixed in this snapshot: (LP: #2065787) - -- Alberto Contreras Fri, 19 Jul 2024 17:09:32 +0200 + -- Alberto Contreras Fri, 19 Jul 2024 17:09:53 +0200 cloud-init (24.2~5g8c396a4b-0ubuntu1) oracular; urgency=medium