From 58f4ec13f476546a5b69ef201c1ac80fdb22abbf Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Mon, 10 Oct 2022 17:31:10 +0800 Subject: [PATCH 01/11] feat: linux control group version 2 API support cgroup v2 --- .../ipc/IPCHibernateTest.java | 3 +- .../GenericExternalServiceIntegTest.java | 117 ++++++++++++++- .../unix/linux/CGroupSubSystemPath.java | 73 ++++++++++ .../util/platforms/unix/linux/Cgroup.java | 92 ++++++------ .../platforms/unix/linux/CgroupSubSystem.java | 93 ++++++++++++ .../unix/linux/CgroupSubSystemV2.java | 72 ++++++++++ .../unix/linux/CgroupV2FreezerState.java | 26 ++++ .../platforms/unix/linux/LinuxPlatform.java | 23 ++- .../linux/LinuxSystemResourceController.java | 117 +++++++++------ .../LinuxSystemResourceControllerV2.java | 133 ++++++++++++++++++ .../LinuxSystemResourceControllerV2Test.java | 111 +++++++++++++++ 11 files changed, 772 insertions(+), 88 deletions(-) create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java create mode 100644 src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java index 1a38192c9f..597749aabe 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java @@ -10,6 +10,7 @@ import com.aws.greengrass.testcommons.testutilities.GGExtension; import com.aws.greengrass.testcommons.testutilities.TestUtils; import com.aws.greengrass.util.platforms.unix.linux.Cgroup; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -115,7 +116,7 @@ void GIVEN_LifeCycleEventStreamClient_WHEN_pause_resume_component_THEN_target_se private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState.valueOf( - new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName)), + new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)), StandardCharsets.UTF_8).trim()); } } diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index f6b37ace13..497437e801 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -21,7 +21,11 @@ import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler; import com.aws.greengrass.util.Pair; import com.aws.greengrass.util.platforms.unix.linux.Cgroup; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2; +import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; +import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceControllerV2; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -32,14 +36,18 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; @@ -75,6 +83,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -84,6 +93,9 @@ class GenericExternalServiceIntegTest extends BaseITCase { private Kernel kernel; + @Mock + LinuxPlatform platform; + static Stream posixTestUserConfig() { return Stream.of( arguments("config_run_with_user.yaml", "nobody", "nobody"), @@ -582,6 +594,87 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re assertResourceLimits(componentName, 10240l * 1024, 1.5); } + @EnabledOnOs({OS.LINUX}) + @Test + void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_resource_limits_V2() throws Exception { + String componentName = "echo_service"; + // Run with no resource limit + ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel, + getClass().getResource("config_run_with_user.yaml")); + CountDownLatch service = new CountDownLatch(1); + kernel.getContext().addGlobalStateChangeListener((s, oldState, newState) -> { + if (s.getName().equals(componentName) && newState.equals(State.RUNNING)) { + service.countDown(); + } + }); + + String componentPathString = "/sys/fs/cgroup/greengrass/" + componentName; + String cgroupv2RootPathString = "/sys/fs/cgroup"; + String cgroupv2GGPathString = "/sys/fs/cgroup/greengrass"; + Path cgroupv2GGPath = Paths.get(cgroupv2GGPathString); + Path path = Paths.get(componentPathString); + Path cgroupv2RootPath = Paths.get(cgroupv2RootPathString); + + Files.createDirectories(path); + + if (!Files.exists(path.resolve("memory.max"))) { + Files.createFile(path.resolve("memory.max")); + } + if (!Files.exists(path.resolve("cpu.max"))) { + Files.createFile(path.resolve("cpu.max")); + } + Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8)); + + if (!Files.exists(cgroupv2RootPath.resolve("cgroup.subtree_control"))) { + Files.createFile(cgroupv2RootPath.resolve("cgroup.subtree_control")); + } + if (!Files.exists(cgroupv2GGPath.resolve("cgroup.subtree_control"))) { + Files.createFile(cgroupv2GGPath.resolve("cgroup.subtree_control")); + } + + kernel.launch(); + assertTrue(service.await(20, TimeUnit.SECONDS), "service running"); + + LinuxSystemResourceControllerV2 linuxSystemResourceControllerV2 = new LinuxSystemResourceControllerV2(platform); + LinuxSystemResourceControllerV2 spyV2 = spy(linuxSystemResourceControllerV2); + Set rootPath = new HashSet<>(); + rootPath.add(cgroupv2RootPathString); + + lenient().doReturn(spyV2).when(platform).getSystemResourceController(); + lenient().doReturn(rootPath).when(spyV2).getMountedPaths(); + // Run with nucleus default resource limit + assertResourceLimits_V2(componentName, 10240l * 1024, 1.5); + + // Run with component resource limit + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(102400l); + + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.5); + // Block until events are completed + kernel.getContext().waitForPublishQueueToClear(); + + assertResourceLimits_V2(componentName, 102400l * 1024, 0.5); + + // Run with updated component resource limit + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(51200l); + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.35); + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, VERSION_CONFIG_KEY).withValue("2.0.0"); + // Block until events are completed + kernel.getContext().waitForPublishQueueToClear(); + + assertResourceLimits_V2(componentName, 51200l * 1024, 0.35); + + //Remove component resource limit, should fall back to default + kernel.getConfig().lookupTopics(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + SYSTEM_RESOURCE_LIMITS_TOPICS).remove(); + kernel.getContext().waitForPublishQueueToClear(); + + assertResourceLimits_V2(componentName, 10240l * 1024, 1.5); + } + @Test void GIVEN_service_starts_up_WHEN_service_breaks_THEN_status_details_persisted_for_errored_and_broken_states() throws Exception { @@ -713,11 +806,11 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers } private void assertResourceLimits(String componentName, long memory, double cpus) throws Exception { - byte[] buf1 = Files.readAllBytes(Cgroup.Memory.getComponentMemoryLimitPath(componentName)); + byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystem.Memory).getComponentMemoryLimitPath(componentName)); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); - byte[] buf2 = Files.readAllBytes(Cgroup.CPU.getComponentCpuQuotaPath(componentName)); - byte[] buf3 = Files.readAllBytes(Cgroup.CPU.getComponentCpuPeriodPath(componentName)); + byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuQuotaPath(componentName)); + byte[] buf3 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuPeriodPath(componentName)); int quota = Integer.parseInt(new String(buf2, StandardCharsets.UTF_8).trim()); int period = Integer.parseInt(new String(buf3, StandardCharsets.UTF_8).trim()); @@ -725,6 +818,22 @@ private void assertResourceLimits(String componentName, long memory, double cpus assertThat(expectedQuota, equalTo(quota)); } + private void assertResourceLimits_V2(String componentName, long memory, double cpus) throws Exception { + byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.Memory).getComponentMemoryLimitPath(componentName)); + assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); + + byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.CPU).getComponentCpuMaxPath(componentName)); + + String cpuMaxContent = new String(buf2, StandardCharsets.UTF_8).trim(); + String[] cpuMaxContentArr = cpuMaxContent.split(" "); + + String cpuMaxStr = cpuMaxContentArr[0]; + String cpuPeriodStr = cpuMaxContentArr[1]; + int quota = Integer.parseInt(cpuMaxStr); + int expectedQuota = (int) (cpus * Integer.parseInt(cpuPeriodStr)); + assertThat(expectedQuota, equalTo(quota)); + } + void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service_and_freeze_thaw_cgroup( ExtensionContext context) throws Exception { ignoreExceptionOfType(context, FileSystemException.class); @@ -761,7 +870,7 @@ void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState - .valueOf(new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName)) + .valueOf(new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)) , StandardCharsets.UTF_8).trim()); } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java new file mode 100644 index 0000000000..500fdfd03e --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java @@ -0,0 +1,73 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import java.nio.file.Path; +import java.nio.file.Paths; + +@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "CGroupSubSystemPath virtual filesystem path cannot be relative") +public interface CGroupSubSystemPath { + String CGROUP_ROOT = "/sys/fs/cgroup"; + String GG_NAMESPACE = "greengrass"; + String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes"; + String CPU_CFS_PERIOD_US = "cpu.cfs_period_us"; + String CPU_CFS_QUOTA_US = "cpu.cfs_quota_us"; + String CGROUP_PROCS = "cgroup.procs"; + String FREEZER_STATE_FILE = "freezer.state"; + String CPU_MAX = "cpu.max"; + String MEMORY_MAX = "memory.max"; + String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control"; + String CGROUP_FREEZE = "cgroup.freeze"; + + default Path getRootPath() { + return Paths.get(CGROUP_ROOT); + } + + String rootMountCmd(); + + default String subsystemMountCmd() { + return null; + } + + Path getSubsystemRootPath(); + + Path getSubsystemGGPath(); + + Path getSubsystemComponentPath(String componentName); + + Path getComponentMemoryLimitPath(String componentName); + + default Path getComponentCpuPeriodPath(String componentName) { + return null; + } + + default Path getComponentCpuQuotaPath(String componentName) { + return null; + } + + Path getCgroupProcsPath(String componentName); + + Path getCgroupFreezerStateFilePath(String componentName); + + default Path getRootSubTreeControlPath() { + return null; + } + + default Path getGGSubTreeControlPath() { + return null; + } + + default Path getComponentCpuMaxPath(String componentName) { + return null; + } + + default Path getCgroupFreezePath(String componentName) { + return null; + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java index ed4b814ed7..43b877f506 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java @@ -5,81 +5,93 @@ package com.aws.greengrass.util.platforms.unix.linux; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - import java.nio.file.Path; -import java.nio.file.Paths; /** - * Represents Linux cgroup v1 subsystems. + * Represents Linux cgroup subsystems. */ -@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Cgroup virtual filesystem path " - + "cannot be relative") -public enum Cgroup { - Memory("memory"), CPU("cpu,cpuacct"), Freezer("freezer", "freezer"); - - private static final String CGROUP_ROOT = "/sys/fs/cgroup"; - private static final String GG_NAMESPACE = "greengrass"; - private static final String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes"; - private static final String CPU_CFS_PERIOD_US = "cpu.cfs_period_us"; - private static final String CPU_CFS_QUOTA_US = "cpu.cfs_quota_us"; - private static final String CGROUP_PROCS = "cgroup.procs"; - private static final String FREEZER_STATE_FILE = "freezer.state"; +public class Cgroup { + private final CGroupSubSystemPath subSystem; - private final String osString; - private final String mountSrc; - - Cgroup(String str) { - osString = str; - mountSrc = "cgroup"; + public Cgroup(CGroupSubSystemPath subSystem) { + this.subSystem = subSystem; } - Cgroup(String str, String mountSrc) { - this.osString = str; - this.mountSrc = mountSrc; + public Path getRootPath() { + return subSystem.getRootPath(); } - public static Path getRootPath() { - return Paths.get(CGROUP_ROOT); - } - - public static String rootMountCmd() { - return String.format("mount -t tmpfs cgroup %s", CGROUP_ROOT); + /** + * root mount cmd. + * + * @return mount command string + */ + public String rootMountCmd() { + return subSystem.rootMountCmd(); } public String subsystemMountCmd() { - return String.format("mount -t cgroup -o %s %s %s", osString, mountSrc, getSubsystemRootPath()); + return subSystem.subsystemMountCmd(); } public Path getSubsystemRootPath() { - return Paths.get(CGROUP_ROOT).resolve(osString); + return subSystem.getSubsystemRootPath(); } public Path getSubsystemGGPath() { - return getSubsystemRootPath().resolve(GG_NAMESPACE); + return subSystem.getSubsystemGGPath(); } public Path getSubsystemComponentPath(String componentName) { - return getSubsystemGGPath().resolve(componentName); + return subSystem.getSubsystemComponentPath(componentName); } + /** + * get component memory limit path. + * + * @param componentName componentName + * @return memory limit Path + */ public Path getComponentMemoryLimitPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); + return subSystem.getComponentMemoryLimitPath(componentName); } public Path getComponentCpuPeriodPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CPU_CFS_PERIOD_US); + return subSystem.getComponentCpuPeriodPath(componentName); } public Path getComponentCpuQuotaPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CPU_CFS_QUOTA_US); + return subSystem.getComponentCpuQuotaPath(componentName); } public Path getCgroupProcsPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); + return subSystem.getCgroupProcsPath(componentName); } + /** + * get cgroup freezer path. + * + * @param componentName componentName + * @return cgroup freezer path + */ public Path getCgroupFreezerStateFilePath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); + return subSystem.getCgroupFreezerStateFilePath(componentName); + } + + public Path getRootSubTreeControlPath() { + return subSystem.getRootSubTreeControlPath(); } + + public Path getGGSubTreeControlPath() { + return subSystem.getGGSubTreeControlPath(); + } + + public Path getComponentCpuMaxPath(String componentName) { + return subSystem.getComponentCpuMaxPath(componentName); + } + + public Path getCgroupFreezePath(String componentName) { + return subSystem.getCgroupFreezePath(componentName); + } + } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java new file mode 100644 index 0000000000..6b36dd7c3a --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import java.nio.file.Path; +import java.nio.file.Paths; + +@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "CgroupSubSystem virtual filesystem path cannot be relative") +public enum CgroupSubSystem implements CGroupSubSystemPath { + Memory("memory", ""), CPU("cpu,cpuacct", ""), Freezer("freezer", "freezer"); + + private String osString; + private String mountSrc; + + CgroupSubSystem(String osString, String mountSrc) { + this.osString = osString; + this.mountSrc = mountSrc; + } + + /** + * Get the osString associated with this CgroupSubController. + * + * @return the osString associated with this CgroupSubController. + */ + public String getOsString() { + return osString; + } + + /** + * Get the mountSrc associated with this CgroupSubController. + * + * @return the mountSrc associated with this CgroupSubController. + */ + public String getMountSrc() { + return mountSrc; + } + + @Override + public String rootMountCmd() { + return String.format("mount -t tmpfs cgroup %s", CGROUP_ROOT); + } + + @Override + public String subsystemMountCmd() { + return String.format("mount -t cgroup -o %s %s %s", osString, mountSrc, getSubsystemRootPath()); + } + + @Override + public Path getSubsystemRootPath() { + return Paths.get(CGROUP_ROOT).resolve(osString); + } + + @Override + public Path getSubsystemGGPath() { + return getSubsystemRootPath().resolve(GG_NAMESPACE); + } + + @Override + public Path getSubsystemComponentPath(String componentName) { + return getSubsystemGGPath().resolve(componentName); + } + + @Override + public Path getComponentMemoryLimitPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); + } + + @Override + public Path getComponentCpuPeriodPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CPU_CFS_PERIOD_US); + } + + @Override + public Path getComponentCpuQuotaPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CPU_CFS_QUOTA_US); + } + + @Override + public Path getCgroupProcsPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); + } + + @Override + public Path getCgroupFreezerStateFilePath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java new file mode 100644 index 0000000000..7a3cf1b31e --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java @@ -0,0 +1,72 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import java.nio.file.Path; +import java.nio.file.Paths; + +@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "CgroupSubSystemV2 virtual filesystem path cannot be relative") +public enum CgroupSubSystemV2 implements CGroupSubSystemPath { + Memory, CPU, Freezer, Unified; + + @Override + public String rootMountCmd() { + return String.format("mount -t cgroup2 none %s", CGROUP_ROOT); + } + + @Override + public Path getSubsystemRootPath() { + return Paths.get(CGROUP_ROOT); + } + + @Override + public Path getSubsystemGGPath() { + return getSubsystemRootPath().resolve(GG_NAMESPACE); + } + + @Override + public Path getSubsystemComponentPath(String componentName) { + return getSubsystemGGPath().resolve(componentName); + } + + @Override + public Path getComponentMemoryLimitPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX); + } + + @Override + public Path getCgroupProcsPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); + } + + @Override + public Path getCgroupFreezerStateFilePath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); + } + + @Override + public Path getRootSubTreeControlPath() { + return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL); + } + + @Override + public Path getGGSubTreeControlPath() { + return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL); + } + + @Override + public Path getComponentCpuMaxPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CPU_MAX); + } + + @Override + public Path getCgroupFreezePath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java new file mode 100644 index 0000000000..09ebdb79b1 --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +public enum CgroupV2FreezerState { + THAWED(0), + FROZEN(1); + + private int index; + + CgroupV2FreezerState(int index) { + this.index = index; + } + + /** + * Get the index value associated with this CgroupV2FreezerState. + * + * @return the integer index value associated with this CgroupV2FreezerState. + */ + public int getIndex() { + return index; + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java index 83dcd8c9af..6bb3944688 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java @@ -7,12 +7,33 @@ import com.aws.greengrass.util.platforms.SystemResourceController; import com.aws.greengrass.util.platforms.unix.UnixPlatform; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Cgroup Controller virtual filesystem path cannot be relative") public class LinuxPlatform extends UnixPlatform { - SystemResourceController systemResourceController = new LinuxSystemResourceController(this); + private static final String CGROUP_ROOT = "/sys/fs/cgroup"; + private static final String CGROUP_CONTROLLERS = "cgroup.controllers"; + + SystemResourceController systemResourceController; @Override public SystemResourceController getSystemResourceController() { + //if the path exists, identify it as cgroupv1, otherwise identify it as cgroupv2 + if (Files.exists(getControllersRootPath())) { + systemResourceController = new LinuxSystemResourceControllerV2(this); + } else { + systemResourceController = new LinuxSystemResourceController(this); + } + return systemResourceController; } + + private Path getControllersRootPath() { + return Paths.get(CGROUP_ROOT).resolve(CGROUP_CONTROLLERS); + } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java index a2bd992321..f926b80d84 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java @@ -34,20 +34,34 @@ public class LinuxSystemResourceController implements SystemResourceController { private static final Logger logger = LogManager.getLogger(LinuxSystemResourceController.class); - private static final String COMPONENT_NAME = "componentName"; - private static final String MEMORY_KEY = "memory"; - private static final String CPUS_KEY = "cpus"; + + protected static final String COMPONENT_NAME = "componentName"; + protected static final String MEMORY_KEY = "memory"; + protected static final String CPUS_KEY = "cpus"; private static final String UNICODE_SPACE = "\\040"; - private static final List RESOURCE_LIMIT_CGROUPS = Arrays.asList(Cgroup.Memory, Cgroup.CPU); + protected Cgroup memoryCgroup; + protected Cgroup cpuCgroup; + protected Cgroup freezerCgroup; + protected Cgroup unifiedCgroup; + protected List resourceLimitCgroups; + protected CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); + + protected LinuxPlatform platform; - private final CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); + public LinuxSystemResourceController() { - protected final LinuxPlatform platform; + } - public LinuxSystemResourceController(LinuxPlatform platform) { + LinuxSystemResourceController(LinuxPlatform platform) { this.platform = platform; + this.memoryCgroup = new Cgroup(CgroupSubSystem.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystem.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystem.Freezer); + resourceLimitCgroups = Arrays.asList( + memoryCgroup, cpuCgroup); } + @Override public void removeResourceController(GreengrassService component) { usedCgroups.forEach(cg -> { @@ -64,36 +78,24 @@ public void removeResourceController(GreengrassService component) { @Override public void updateResourceLimits(GreengrassService component, Map resourceLimit) { try { - if (!Files.exists(Cgroup.Memory.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component, Cgroup.Memory); - } - if (resourceLimit.containsKey(MEMORY_KEY)) { - long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY)); - if (memoryLimitInKB > 0) { - String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); - Files.write(Cgroup.Memory.getComponentMemoryLimitPath(component.getServiceName()), - memoryLimit.getBytes(StandardCharsets.UTF_8)); - } else { - logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) - .log("The provided memory limit is invalid"); - } - } + updateMemoryResourceLimits(component, resourceLimit); - if (!Files.exists(Cgroup.CPU.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component, Cgroup.CPU); + if (!Files.exists(cpuCgroup.getSubsystemComponentPath(component.getServiceName()))) { + initializeCgroup(component, cpuCgroup); } if (resourceLimit.containsKey(CPUS_KEY)) { double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY)); if (cpu > 0) { byte[] content = Files.readAllBytes( - Cgroup.CPU.getComponentCpuPeriodPath(component.getServiceName())); + cpuCgroup.getComponentCpuPeriodPath(component.getServiceName())); int cpuPeriodUs = Integer.parseInt(new String(content, StandardCharsets.UTF_8).trim()); int cpuQuotaUs = (int) (cpuPeriodUs * cpu); String cpuQuotaUsStr = Integer.toString(cpuQuotaUs); - Files.write(Cgroup.CPU.getComponentCpuQuotaPath(component.getServiceName()), + Files.write(cpuCgroup.getComponentCpuQuotaPath(component.getServiceName()), cpuQuotaUsStr.getBytes(StandardCharsets.UTF_8)); + } else { logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) .log("The provided cpu limit is invalid"); @@ -105,9 +107,27 @@ public void updateResourceLimits(GreengrassService component, Map resourceLimit) throws IOException { + if (!Files.exists(memoryCgroup.getSubsystemComponentPath(component.getServiceName()))) { + initializeCgroup(component, memoryCgroup); + } + if (resourceLimit.containsKey(MEMORY_KEY)) { + long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY)); + if (memoryLimitInKB > 0) { + String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); + Files.write(memoryCgroup.getComponentMemoryLimitPath(component.getServiceName()), + memoryLimit.getBytes(StandardCharsets.UTF_8)); + } else { + logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) + .log("The provided memory limit is invalid"); + } + } + } + @Override public void resetResourceLimits(GreengrassService component) { - for (Cgroup cg : RESOURCE_LIMIT_CGROUPS) { + for (Cgroup cg : resourceLimitCgroups) { try { if (Files.exists(cg.getSubsystemComponentPath(component.getServiceName()))) { Files.delete(cg.getSubsystemComponentPath(component.getServiceName())); @@ -122,7 +142,7 @@ public void resetResourceLimits(GreengrassService component) { @Override public void addComponentProcess(GreengrassService component, Process process) { - RESOURCE_LIMIT_CGROUPS.forEach(cg -> { + resourceLimitCgroups.forEach(cg -> { try { addComponentProcessToCgroup(component.getServiceName(), process, cg); @@ -139,19 +159,14 @@ public void addComponentProcess(GreengrassService component, Process process) { }, 1, TimeUnit.SECONDS); } catch (IOException e) { - handleErrorAddingPidToCgroup(e, component.getServiceName()); + handleErrorAddingPidToCgroup(e, component.getServiceName()); } }); } @Override public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { - initializeCgroup(component, Cgroup.Freezer); - - for (Process process: processes) { - addComponentProcessToCgroup(component.getServiceName(), process, Cgroup.Freezer); - } - + prePauseComponentProcesses(component, processes); if (CgroupFreezerState.FROZEN.equals(currentFreezerCgroupState(component.getServiceName()))) { return; } @@ -160,6 +175,7 @@ public void pauseComponentProcesses(GreengrassService component, List p StandardOpenOption.TRUNCATE_EXISTING); } + @Override public void resumeComponentProcesses(GreengrassService component) throws IOException { if (CgroupFreezerState.THAWED.equals(currentFreezerCgroupState(component.getServiceName()))) { @@ -170,7 +186,7 @@ public void resumeComponentProcesses(GreengrassService component) throws IOExcep StandardOpenOption.TRUNCATE_EXISTING); } - private void addComponentProcessToCgroup(String component, Process process, Cgroup cg) + protected void addComponentProcessToCgroup(String component, Process process, Cgroup cg) throws IOException { if (!Files.exists(cg.getSubsystemComponentPath(component))) { @@ -219,7 +235,12 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) { } } - private Set getMountedPaths() throws IOException { + /** + * Get mounted path for Cgroup. + * @return mounted path set + * @throws IOException IOException + */ + public Set getMountedPaths() throws IOException { Set mountedPaths = new HashSet<>(); Path procMountsPath = Paths.get("/proc/self/mounts"); @@ -244,15 +265,18 @@ private Set getMountedPaths() throws IOException { return mountedPaths; } - private void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { + protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { Set mounts = getMountedPaths(); - if (!mounts.contains(Cgroup.getRootPath().toString())) { - platform.runCmd(Cgroup.rootMountCmd(), o -> {}, "Failed to mount cgroup root"); + + if (!mounts.contains(cgroup.getRootPath().toString())) { + platform.runCmd(cgroup.rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); Files.createDirectory(cgroup.getSubsystemRootPath()); } if (!mounts.contains(cgroup.getSubsystemRootPath().toString())) { - platform.runCmd(cgroup.subsystemMountCmd(), o -> {}, "Failed to mount cgroup subsystem"); + platform.runCmd(cgroup.subsystemMountCmd(), o -> { + }, "Failed to mount cgroup subsystem"); } if (!Files.exists(cgroup.getSubsystemGGPath())) { Files.createDirectory(cgroup.getSubsystemGGPath()); @@ -260,6 +284,7 @@ private void initializeCgroup(GreengrassService component, Cgroup cgroup) throws if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); } + usedCgroups.add(cgroup); } @@ -268,8 +293,8 @@ private Set pidsInComponentCgroup(Cgroup cgroup, String component) thro .stream().map(Integer::parseInt).collect(Collectors.toSet()); } - private Path freezerCgroupStateFile(String component) { - return Cgroup.Freezer.getCgroupFreezerStateFilePath(component); + protected Path freezerCgroupStateFile(String component) { + return freezerCgroup.getCgroupFreezerStateFilePath(component); } private CgroupFreezerState currentFreezerCgroupState(String component) throws IOException { @@ -281,6 +306,14 @@ private CgroupFreezerState currentFreezerCgroupState(String component) throws IO return CgroupFreezerState.valueOf(stateFileContent.get(0).trim()); } + protected void prePauseComponentProcesses(GreengrassService component, List processes) throws IOException { + initializeCgroup(component, freezerCgroup); + + for (Process process : processes) { + addComponentProcessToCgroup(component.getServiceName(), process, freezerCgroup); + } + } + public enum CgroupFreezerState { THAWED, FREEZING, FROZEN } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java new file mode 100644 index 0000000000..57a7f52c96 --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java @@ -0,0 +1,133 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import com.aws.greengrass.lifecyclemanager.GreengrassService; +import com.aws.greengrass.logging.api.Logger; +import com.aws.greengrass.logging.impl.LogManager; +import com.aws.greengrass.util.Coerce; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * https://www.kernel.org/doc/Documentation/cgroup-v2.txt + */ +public class LinuxSystemResourceControllerV2 extends LinuxSystemResourceController { + private static final Logger logger = LogManager.getLogger(LinuxSystemResourceControllerV2.class); + private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids"; + + /** + * Linux system resource controller V2 constrcutors. + * + * @param platform linux platform + */ + public LinuxSystemResourceControllerV2(LinuxPlatform platform) { + super(); + this.platform = platform; + this.unifiedCgroup = new Cgroup(CgroupSubSystemV2.Unified); + this.memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystemV2.Freezer); + resourceLimitCgroups = Arrays.asList(unifiedCgroup); + } + + + @Override + public void updateResourceLimits(GreengrassService component, Map resourceLimit) { + try { + super.updateMemoryResourceLimits(component, resourceLimit); + + if (resourceLimit.containsKey(CPUS_KEY)) { + double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY)); + if (cpu > 0) { + handleCpuLimits(component, cpu); + } else { + logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) + .log("The provided cpu limit is invalid"); + } + } + } catch (IOException e) { + logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName()) + .log("Failed to apply resource limits"); + } + } + + @Override + public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { + prePauseComponentProcesses(component, processes); + + Files.write(freezerCgroupStateFile(component.getServiceName()), + String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } + + @Override + public void resumeComponentProcesses(GreengrassService component) throws IOException { + Files.write(freezerCgroupStateFile(component.getServiceName()), + String.valueOf(CgroupV2FreezerState.THAWED.getIndex()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } + + @Override + protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { + Set mounts = getMountedPaths(); + + if (!mounts.contains(cgroup.getRootPath().toString())) { + platform.runCmd(cgroup.rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); + Files.createDirectory(cgroup.getSubsystemRootPath()); + } + + if (!Files.exists(cgroup.getSubsystemGGPath())) { + Files.createDirectory(cgroup.getSubsystemGGPath()); + } + if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { + Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); + } + + //Enable controllers for root group + Files.write(cgroup.getRootSubTreeControlPath(), + CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); + //Enable controllers for gg group + Files.write(cgroup.getGGSubTreeControlPath(), + CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); + + usedCgroups.add(cgroup); + } + + private void handleCpuLimits(GreengrassService component, double cpu) throws IOException { + byte[] content = Files.readAllBytes( + cpuCgroup.getComponentCpuMaxPath(component.getServiceName())); + String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim(); + String[] cpuMaxContentArr = cpuMaxContent.split(" "); + String cpuMaxStr = "max"; + String cpuPeriodStr = "100000"; + + if (cpuMaxContentArr.length >= 2) { + cpuMaxStr = cpuMaxContentArr[0]; + cpuPeriodStr = cpuMaxContentArr[1]; + + if (!StringUtils.isEmpty(cpuPeriodStr)) { + int period = Integer.parseInt(cpuPeriodStr.trim()); + int max = (int) (period * cpu); + cpuMaxStr = Integer.toString(max); + } + } + + String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr); + Files.write(cpuCgroup.getComponentCpuMaxPath(component.getServiceName()), + latestCpuMaxContent.getBytes(StandardCharsets.UTF_8)); + } + +} diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java new file mode 100644 index 0000000000..b6f43377a7 --- /dev/null +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -0,0 +1,111 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import com.aws.greengrass.lifecyclemanager.GreengrassService; +import com.aws.greengrass.testcommons.testutilities.GGExtension; +import com.aws.greengrass.util.Utils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith({MockitoExtension.class, GGExtension.class}) +@DisabledOnOs(OS.WINDOWS) +class LinuxSystemResourceControllerV2Test { + @Mock + GreengrassService component; + @Mock + Cgroup cpuCgroup; + @Mock + Cgroup memoryCgroup; + @Mock + LinuxPlatform platform; + @InjectMocks + @Spy + LinuxSystemResourceControllerV2 linuxSystemResourceControllerV2 = new LinuxSystemResourceControllerV2(platform); + private static final String FILE_PATH = "/cgroupv2test"; + private static final String CGROUP_MEMORY_LIMIT_FILE_NAME = "memory.txt"; + private static final String CGROUP_CPU_LIMIT_FILE_NAME = "cpu.txt"; + private static final long MEMORY_IN_KB = 2048000; + private static final double CPU_TIME = 0.5; + private static final String COMPONENT_NAME = "testComponentName"; + + @Test + void GIVEN_cgroupv2_WHEN_memory_limit_updated_THEN_memory_limit_file_updated() throws IOException { + Map resourceLimit = new HashMap<>(); + resourceLimit.put("memory", String.valueOf(MEMORY_IN_KB)); + doReturn("testComponentName").when(component).getServiceName(); + + Path path = Paths.get(FILE_PATH + "/" + CGROUP_MEMORY_LIMIT_FILE_NAME); + + Path componentNameFolderPath = Paths.get(FILE_PATH); + Utils.createPaths(componentNameFolderPath); + File file = new File(FILE_PATH + "/" + CGROUP_MEMORY_LIMIT_FILE_NAME); + if (!Files.exists(path)) { + file.createNewFile(); + } + + when(memoryCgroup.getComponentMemoryLimitPath(COMPONENT_NAME)).thenReturn(path); + lenient().when(memoryCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); + linuxSystemResourceControllerV2.updateResourceLimits(component, resourceLimit); + + List mounts = Files.readAllLines(path); + assertEquals(String.valueOf(MEMORY_IN_KB * 1024), mounts.get(0)); + + Files.deleteIfExists(path); + Files.deleteIfExists(componentNameFolderPath); + } + + @Test + void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws IOException { + Map resourceLimit = new HashMap<>(); + resourceLimit.put("cpus", String.valueOf(CPU_TIME)); + doReturn("testComponentName").when(component).getServiceName(); + + Path path = Paths.get(FILE_PATH + "/" + CGROUP_CPU_LIMIT_FILE_NAME); + + Path componentNameFolderPath = Paths.get(FILE_PATH); + Utils.createPaths(componentNameFolderPath); + File file = new File(FILE_PATH + "/" + CGROUP_CPU_LIMIT_FILE_NAME); + if (!Files.exists(path)) { + file.createNewFile(); + } + + Files.write(path, "max 100000".getBytes(StandardCharsets.UTF_8)); + + doNothing().when(linuxSystemResourceControllerV2).updateMemoryResourceLimits(component, resourceLimit); + when(cpuCgroup.getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); + lenient().when(cpuCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); + linuxSystemResourceControllerV2.updateResourceLimits(component, resourceLimit); + + List mounts = Files.readAllLines(path); + assertEquals((int) (CPU_TIME * 100000) + " 100000", mounts.get(0)); + + Files.deleteIfExists(path); + Files.deleteIfExists(componentNameFolderPath); + } +} \ No newline at end of file From 1abc7ccf32b6edb138c0c402beb0da0a7e4d935a Mon Sep 17 00:00:00 2001 From: "yiwen.chen" Date: Tue, 25 Oct 2022 00:34:46 -0700 Subject: [PATCH 02/11] feat: linux control group version 2 API support cgroup v2 --- .../GenericExternalServiceIntegTest.java | 147 ++++++++++-------- .../unix/linux/CGroupSubSystemPath.java | 4 +- .../platforms/unix/linux/CgroupSubSystem.java | 3 +- .../unix/linux/CgroupSubSystemV2.java | 3 +- .../platforms/unix/linux/LinuxPlatform.java | 8 +- .../linux/LinuxSystemResourceController.java | 7 +- .../LinuxSystemResourceControllerV2.java | 3 +- 7 files changed, 93 insertions(+), 82 deletions(-) diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index 860f4381b8..4c3816e1e2 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -18,36 +18,35 @@ import com.aws.greengrass.logging.impl.GreengrassLogMessage; import com.aws.greengrass.logging.impl.Slf4jLogAdapter; import com.aws.greengrass.status.model.ComponentStatusDetails; +import com.aws.greengrass.testcommons.testutilities.GGExtension; import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler; import com.aws.greengrass.util.Pair; -import com.aws.greengrass.util.platforms.unix.linux.Cgroup; -import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; -import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2; -import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform; -import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; -import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceControllerV2; +import com.aws.greengrass.util.platforms.SystemResourceController; +import com.aws.greengrass.util.platforms.unix.linux.*; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.SystemUtils; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mock; +import org.mockito.*; +import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; @@ -83,18 +82,22 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; -import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; - +@ExtendWith({GGExtension.class, MockitoExtension.class}) class GenericExternalServiceIntegTest extends BaseITCase { private Kernel kernel; - @Mock - LinuxPlatform platform; + LinuxPlatform linuxPlatform; + + private final static String ROOT_PATH_STRING = "/systest21/fs/cgroup"; + private final static String GG_PATH_STRING = "greengrass"; + + @Spy + SystemResourceController spySystemResourceController; static Stream posixTestUserConfig() { return Stream.of( @@ -635,82 +638,78 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re @EnabledOnOs({OS.LINUX}) @Test void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_resource_limits_V2() throws Exception { - String componentName = "echo_service"; + String echoComponentName = "echo_service"; + String mainComponentName = "main"; + String rootGGPathString = ROOT_PATH_STRING + "/" + GG_PATH_STRING; + String componentPathString = rootGGPathString + "/" + echoComponentName; // Run with no resource limit ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel, getClass().getResource("config_run_with_user.yaml")); CountDownLatch service = new CountDownLatch(1); kernel.getContext().addGlobalStateChangeListener((s, oldState, newState) -> { - if (s.getName().equals(componentName) && newState.equals(State.RUNNING)) { + if (s.getName().equals(echoComponentName) && newState.equals(State.RUNNING)) { service.countDown(); } }); - String componentPathString = "/sys/fs/cgroup/greengrass/" + componentName; - String cgroupv2RootPathString = "/sys/fs/cgroup"; - String cgroupv2GGPathString = "/sys/fs/cgroup/greengrass"; - Path cgroupv2GGPath = Paths.get(cgroupv2GGPathString); - Path path = Paths.get(componentPathString); - Path cgroupv2RootPath = Paths.get(cgroupv2RootPathString); - - Files.createDirectories(path); - - if (!Files.exists(path.resolve("memory.max"))) { - Files.createFile(path.resolve("memory.max")); - } - if (!Files.exists(path.resolve("cpu.max"))) { - Files.createFile(path.resolve("cpu.max")); - } - Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8)); + linuxPlatform = kernel.getContext().get(LinuxPlatform.class); - if (!Files.exists(cgroupv2RootPath.resolve("cgroup.subtree_control"))) { - Files.createFile(cgroupv2RootPath.resolve("cgroup.subtree_control")); - } - if (!Files.exists(cgroupv2GGPath.resolve("cgroup.subtree_control"))) { - Files.createFile(cgroupv2GGPath.resolve("cgroup.subtree_control")); - } + createComponentData(echoComponentName); + createComponentData(mainComponentName); - kernel.launch(); - assertTrue(service.await(20, TimeUnit.SECONDS), "service running"); + setFinalStatic(linuxPlatform, LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS"), Paths.get(componentPathString + "/memory.max")); - LinuxSystemResourceControllerV2 linuxSystemResourceControllerV2 = new LinuxSystemResourceControllerV2(platform); - LinuxSystemResourceControllerV2 spyV2 = spy(linuxSystemResourceControllerV2); - Set rootPath = new HashSet<>(); - rootPath.add(cgroupv2RootPathString); + spySystemResourceController = linuxPlatform.getSystemResourceController(); + LinuxSystemResourceControllerV2 controllerV2 = (LinuxSystemResourceControllerV2) spySystemResourceController; - lenient().doReturn(spyV2).when(platform).getSystemResourceController(); - lenient().doReturn(rootPath).when(spyV2).getMountedPaths(); - // Run with nucleus default resource limit - assertResourceLimits_V2(componentName, 10240l * 1024, 1.5); + Field memoryCgroupField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("memoryCgroup"); + memoryCgroupField.setAccessible(true); + Cgroup memoryCgroup = (Cgroup) memoryCgroupField.get(controllerV2); + Field subsystem = memoryCgroup.getClass().getDeclaredField("subSystem"); + subsystem.setAccessible(true); + CgroupSubSystemV2 cg = (CgroupSubSystemV2) subsystem.get(memoryCgroup); + Field f = cg.getClass().getInterfaces()[0].getDeclaredField("CGROUP_ROOT"); + setFinalStatic(cg, f, Paths.get(ROOT_PATH_STRING)); - // Run with component resource limit - kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, - SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(102400l); + Field mountsField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("MOUNT_PATH"); + mountsField.setAccessible(true); + String mountPathFile = rootGGPathString + "/mountPath.txt"; + final Path mountPathFilePath = Paths.get(mountPathFile); + if (!Files.exists(mountPathFilePath)) { + Files.createFile(mountPathFilePath); + Files.write(mountPathFilePath, String.format("test1 %s test2 test3 test4 test5", ROOT_PATH_STRING).getBytes(StandardCharsets.UTF_8)); - kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, - SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.5); - // Block until events are completed - kernel.getContext().waitForPublishQueueToClear(); + } + setFinalStatic(controllerV2, mountsField, mountPathFile); - assertResourceLimits_V2(componentName, 102400l * 1024, 0.5); + kernel.launch(); + assertResourceLimits_V2(10240l * 1024, 1.5); // Run with updated component resource limit - kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC, SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(51200l); - kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC, SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.35); - kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, VERSION_CONFIG_KEY).withValue("2.0.0"); + kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, echoComponentName, VERSION_CONFIG_KEY).withValue("2.0.0"); // Block until events are completed kernel.getContext().waitForPublishQueueToClear(); - assertResourceLimits_V2(componentName, 51200l * 1024, 0.35); + assertResourceLimits_V2(51200l * 1024, 0.35); //Remove component resource limit, should fall back to default - kernel.getConfig().lookupTopics(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC, + kernel.getConfig().lookupTopics(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC, SYSTEM_RESOURCE_LIMITS_TOPICS).remove(); kernel.getContext().waitForPublishQueueToClear(); - assertResourceLimits_V2(componentName, 10240l * 1024, 1.5); + assertResourceLimits_V2(10240l * 1024, 1.5); + + FileUtils.deleteDirectory(Paths.get("/systest21").toFile()); + } + + private void setFinalStatic(Object obj, Field field, Object newValue) throws Exception { + field.setAccessible(true); + FieldUtils.removeFinalModifier(field, true); + field.set(obj, newValue); } @Test @@ -856,11 +855,27 @@ private void assertResourceLimits(String componentName, long memory, double cpus assertThat(expectedQuota, equalTo(quota)); } - private void assertResourceLimits_V2(String componentName, long memory, double cpus) throws Exception { - byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.Memory).getComponentMemoryLimitPath(componentName)); + private void createComponentData(String componentName) throws IOException { + Path path = Paths.get(ROOT_PATH_STRING).resolve(GG_PATH_STRING).resolve(componentName); + Files.createDirectories(path); + + if (!Files.exists(path.resolve("memory.max"))) { + Files.createFile(path.resolve("memory.max")); + } + if (!Files.exists(path.resolve("cpu.max"))) { + Files.createFile(path.resolve("cpu.max")); + } + Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8)); + if (!Files.exists(path.resolve("cgroup.proc"))) { + Files.createFile(path.resolve("cgroup.proc")); + } + } + + private void assertResourceLimits_V2(long memory, double cpus) throws Exception { + byte[] buf1 = Files.readAllBytes(Paths.get(String.format("%s/%s/echo_service/memory.max", ROOT_PATH_STRING, GG_PATH_STRING))); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); - byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.CPU).getComponentCpuMaxPath(componentName)); + byte[] buf2 = Files.readAllBytes(Paths.get(String.format("%s/%s/echo_service/cpu.max", ROOT_PATH_STRING, GG_PATH_STRING))); String cpuMaxContent = new String(buf2, StandardCharsets.UTF_8).trim(); String[] cpuMaxContentArr = cpuMaxContent.split(" "); diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java index 500fdfd03e..beef8b1b01 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java @@ -13,7 +13,7 @@ @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CGroupSubSystemPath virtual filesystem path cannot be relative") public interface CGroupSubSystemPath { - String CGROUP_ROOT = "/sys/fs/cgroup"; + Path CGROUP_ROOT = Paths.get("/sys/fs/cgroup"); String GG_NAMESPACE = "greengrass"; String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes"; String CPU_CFS_PERIOD_US = "cpu.cfs_period_us"; @@ -26,7 +26,7 @@ public interface CGroupSubSystemPath { String CGROUP_FREEZE = "cgroup.freeze"; default Path getRootPath() { - return Paths.get(CGROUP_ROOT); + return CGROUP_ROOT; } String rootMountCmd(); diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java index 6b36dd7c3a..9e3aab7855 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java @@ -8,7 +8,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.nio.file.Path; -import java.nio.file.Paths; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CgroupSubSystem virtual filesystem path cannot be relative") @@ -53,7 +52,7 @@ public String subsystemMountCmd() { @Override public Path getSubsystemRootPath() { - return Paths.get(CGROUP_ROOT).resolve(osString); + return CGROUP_ROOT.resolve(osString); } @Override diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java index 7a3cf1b31e..c064572cc2 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java @@ -8,7 +8,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.nio.file.Path; -import java.nio.file.Paths; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CgroupSubSystemV2 virtual filesystem path cannot be relative") @@ -22,7 +21,7 @@ public String rootMountCmd() { @Override public Path getSubsystemRootPath() { - return Paths.get(CGROUP_ROOT); + return CGROUP_ROOT; } @Override diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java index 6bb3944688..bac64342fa 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java @@ -16,15 +16,14 @@ @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Cgroup Controller virtual filesystem path cannot be relative") public class LinuxPlatform extends UnixPlatform { - private static final String CGROUP_ROOT = "/sys/fs/cgroup"; - private static final String CGROUP_CONTROLLERS = "cgroup.controllers"; + private static final Path CGROUP_CONTROLLERS = Paths.get("/sys/fs/cgroup/cgroup.controllers"); SystemResourceController systemResourceController; @Override public SystemResourceController getSystemResourceController() { //if the path exists, identify it as cgroupv1, otherwise identify it as cgroupv2 - if (Files.exists(getControllersRootPath())) { + if (Files.exists(CGROUP_CONTROLLERS)) { systemResourceController = new LinuxSystemResourceControllerV2(this); } else { systemResourceController = new LinuxSystemResourceController(this); @@ -33,7 +32,4 @@ public SystemResourceController getSystemResourceController() { return systemResourceController; } - private Path getControllersRootPath() { - return Paths.get(CGROUP_ROOT).resolve(CGROUP_CONTROLLERS); - } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java index f926b80d84..ab150f395f 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java @@ -39,6 +39,7 @@ public class LinuxSystemResourceController implements SystemResourceController { protected static final String MEMORY_KEY = "memory"; protected static final String CPUS_KEY = "cpus"; private static final String UNICODE_SPACE = "\\040"; + protected static final String MOUNT_PATH = "/proc/self/mounts"; protected Cgroup memoryCgroup; protected Cgroup cpuCgroup; protected Cgroup freezerCgroup; @@ -48,6 +49,8 @@ public class LinuxSystemResourceController implements SystemResourceController { protected LinuxPlatform platform; + protected Set mounts; + public LinuxSystemResourceController() { } @@ -243,7 +246,7 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) { public Set getMountedPaths() throws IOException { Set mountedPaths = new HashSet<>(); - Path procMountsPath = Paths.get("/proc/self/mounts"); + Path procMountsPath = Paths.get(MOUNT_PATH); List mounts = Files.readAllLines(procMountsPath); for (String mount : mounts) { String[] split = mount.split(" "); @@ -266,7 +269,7 @@ public Set getMountedPaths() throws IOException { } protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { - Set mounts = getMountedPaths(); + mounts = getMountedPaths(); if (!mounts.contains(cgroup.getRootPath().toString())) { platform.runCmd(cgroup.rootMountCmd(), o -> { diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java index 57a7f52c96..6e10e34e62 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java @@ -18,7 +18,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Set; /** * https://www.kernel.org/doc/Documentation/cgroup-v2.txt @@ -81,7 +80,7 @@ public void resumeComponentProcesses(GreengrassService component) throws IOExcep @Override protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { - Set mounts = getMountedPaths(); + mounts = getMountedPaths(); if (!mounts.contains(cgroup.getRootPath().toString())) { platform.runCmd(cgroup.rootMountCmd(), o -> { From 5832fb22d5bf0e1bfb1a5219c584d716e294a2f3 Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Tue, 25 Oct 2022 15:42:55 +0800 Subject: [PATCH 03/11] feat: linux control group version 2 API support cgroup v2 --- .../GenericExternalServiceIntegTest.java | 42 +++++++++++-------- .../platforms/unix/linux/LinuxPlatform.java | 3 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index 4c3816e1e2..f07204d3ac 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -22,7 +22,12 @@ import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler; import com.aws.greengrass.util.Pair; import com.aws.greengrass.util.platforms.SystemResourceController; -import com.aws.greengrass.util.platforms.unix.linux.*; +import com.aws.greengrass.util.platforms.unix.linux.Cgroup; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2; +import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform; +import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; +import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceControllerV2; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -36,7 +41,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.*; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; @@ -46,7 +51,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; @@ -88,16 +94,15 @@ @ExtendWith({GGExtension.class, MockitoExtension.class}) class GenericExternalServiceIntegTest extends BaseITCase { - private Kernel kernel; + @Spy LinuxPlatform linuxPlatform; private final static String ROOT_PATH_STRING = "/systest21/fs/cgroup"; private final static String GG_PATH_STRING = "greengrass"; - @Spy - SystemResourceController spySystemResourceController; + SystemResourceController systemResourceController; static Stream posixTestUserConfig() { return Stream.of( @@ -652,16 +657,17 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re } }); - linuxPlatform = kernel.getContext().get(LinuxPlatform.class); + linuxPlatform = spy(kernel.getContext().get(LinuxPlatform.class)); createComponentData(echoComponentName); createComponentData(mainComponentName); - setFinalStatic(linuxPlatform, LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS"), Paths.get(componentPathString + "/memory.max")); - - spySystemResourceController = linuxPlatform.getSystemResourceController(); - LinuxSystemResourceControllerV2 controllerV2 = (LinuxSystemResourceControllerV2) spySystemResourceController; - + // Due to cgroup v1 is active by default (in test platform), and the directories of cgroup v1 are read-only + // therefore, here create some directories and files as fake cgroup v2 files to support testing + Field controllerField = LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS"); + setFinalStatic(controllerField, Paths.get(componentPathString + "/memory.max")); + systemResourceController = linuxPlatform.getSystemResourceController(); + LinuxSystemResourceControllerV2 controllerV2 = (LinuxSystemResourceControllerV2) systemResourceController; Field memoryCgroupField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("memoryCgroup"); memoryCgroupField.setAccessible(true); Cgroup memoryCgroup = (Cgroup) memoryCgroupField.get(controllerV2); @@ -669,7 +675,7 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re subsystem.setAccessible(true); CgroupSubSystemV2 cg = (CgroupSubSystemV2) subsystem.get(memoryCgroup); Field f = cg.getClass().getInterfaces()[0].getDeclaredField("CGROUP_ROOT"); - setFinalStatic(cg, f, Paths.get(ROOT_PATH_STRING)); + setFinalStatic(f, Paths.get(ROOT_PATH_STRING)); Field mountsField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("MOUNT_PATH"); mountsField.setAccessible(true); @@ -680,7 +686,7 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re Files.write(mountPathFilePath, String.format("test1 %s test2 test3 test4 test5", ROOT_PATH_STRING).getBytes(StandardCharsets.UTF_8)); } - setFinalStatic(controllerV2, mountsField, mountPathFile); + setFinalStatic(mountsField, mountPathFile); kernel.launch(); assertResourceLimits_V2(10240l * 1024, 1.5); @@ -706,10 +712,10 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re FileUtils.deleteDirectory(Paths.get("/systest21").toFile()); } - private void setFinalStatic(Object obj, Field field, Object newValue) throws Exception { + private void setFinalStatic(Field field, Object newValue) throws Exception { field.setAccessible(true); FieldUtils.removeFinalModifier(field, true); - field.set(obj, newValue); + field.set(null, newValue); } @Test @@ -866,8 +872,8 @@ private void createComponentData(String componentName) throws IOException { Files.createFile(path.resolve("cpu.max")); } Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8)); - if (!Files.exists(path.resolve("cgroup.proc"))) { - Files.createFile(path.resolve("cgroup.proc")); + if (!Files.exists(path.resolve("cgroup.procs"))) { + Files.createFile(path.resolve("cgroup.procs")); } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java index bac64342fa..13b7c9119e 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java @@ -17,12 +17,11 @@ justification = "Cgroup Controller virtual filesystem path cannot be relative") public class LinuxPlatform extends UnixPlatform { private static final Path CGROUP_CONTROLLERS = Paths.get("/sys/fs/cgroup/cgroup.controllers"); - SystemResourceController systemResourceController; @Override public SystemResourceController getSystemResourceController() { - //if the path exists, identify it as cgroupv1, otherwise identify it as cgroupv2 + //if the path exists, identify it as cgroupv2, otherwise identify it as cgroupv1 if (Files.exists(CGROUP_CONTROLLERS)) { systemResourceController = new LinuxSystemResourceControllerV2(this); } else { From 1b95b7d70f5c46ecd1d2a18c9dc97e28360be76e Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Thu, 3 Nov 2022 17:03:51 +0800 Subject: [PATCH 04/11] feat: linux control group version 2 API support cgroup v2 --- .../GenericExternalServiceIntegTest.java | 7 +- .../unix/linux/CGroupSubSystemPath.java | 47 ++++++ .../util/platforms/unix/linux/Cgroup.java | 24 +++ .../platforms/unix/linux/CgroupSubSystem.java | 77 ++++++++++ .../unix/linux/CgroupSubSystemV2.java | 74 +++++++++ .../platforms/unix/linux/LinuxPlatform.java | 7 +- .../linux/LinuxSystemResourceController.java | 145 +++++------------- .../LinuxSystemResourceControllerV2.java | 132 ---------------- .../LinuxSystemResourceControllerV2Test.java | 30 ++-- 9 files changed, 281 insertions(+), 262 deletions(-) delete mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index f07204d3ac..7ad911722a 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -27,7 +27,6 @@ import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2; import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; -import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceControllerV2; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -667,8 +666,8 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re Field controllerField = LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS"); setFinalStatic(controllerField, Paths.get(componentPathString + "/memory.max")); systemResourceController = linuxPlatform.getSystemResourceController(); - LinuxSystemResourceControllerV2 controllerV2 = (LinuxSystemResourceControllerV2) systemResourceController; - Field memoryCgroupField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("memoryCgroup"); + LinuxSystemResourceController controllerV2 = (LinuxSystemResourceController) systemResourceController; + Field memoryCgroupField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("memoryCgroup"); memoryCgroupField.setAccessible(true); Cgroup memoryCgroup = (Cgroup) memoryCgroupField.get(controllerV2); Field subsystem = memoryCgroup.getClass().getDeclaredField("subSystem"); @@ -677,7 +676,7 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re Field f = cg.getClass().getInterfaces()[0].getDeclaredField("CGROUP_ROOT"); setFinalStatic(f, Paths.get(ROOT_PATH_STRING)); - Field mountsField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("MOUNT_PATH"); + Field mountsField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("MOUNT_PATH"); mountsField.setAccessible(true); String mountPathFile = rootGGPathString + "/mountPath.txt"; final Path mountPathFilePath = Paths.get(mountPathFile); diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java index beef8b1b01..e99c3130a0 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java @@ -5,10 +5,16 @@ package com.aws.greengrass.util.platforms.unix.linux; +import com.aws.greengrass.lifecyclemanager.GreengrassService; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Set; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CGroupSubSystemPath virtual filesystem path cannot be relative") @@ -24,6 +30,8 @@ public interface CGroupSubSystemPath { String MEMORY_MAX = "memory.max"; String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control"; String CGROUP_FREEZE = "cgroup.freeze"; + String MOUNT_PATH = "/proc/self/mounts"; + String UNICODE_SPACE = "\\040"; default Path getRootPath() { return CGROUP_ROOT; @@ -70,4 +78,43 @@ default Path getComponentCpuMaxPath(String componentName) { default Path getCgroupFreezePath(String componentName) { return null; } + + void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException; + + void handleCpuLimits(GreengrassService component, double cpu) throws IOException; + + void pauseComponentProcessesCore(GreengrassService component, List processes) throws IOException; + + void resumeComponentProcesses(GreengrassService component) throws IOException; + + /** + * Get mounted paths. + * + * @return A set of String + * @throws IOException IOException + */ + default Set getMountedPaths() throws IOException { + Set mountedPaths = new HashSet<>(); + + Path procMountsPath = Paths.get(MOUNT_PATH); + List mounts = Files.readAllLines(procMountsPath); + for (String mount : mounts) { + String[] split = mount.split(" "); + // As reported in fstab(5) manpage, struct is: + // 1st field is volume name + // 2nd field is path with spaces escaped as \040 + // 3rd field is fs type + // 4th field is mount options + // 5th field is used by dump(8) (ignored) + // 6th field is fsck order (ignored) + if (split.length < 6) { + continue; + } + + // We only need the path of the mounts to verify whether cgroup is mounted + String path = split[1].replace(UNICODE_SPACE, " "); + mountedPaths.add(path); + } + return mountedPaths; + } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java index 43b877f506..d28dece13b 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java @@ -5,7 +5,11 @@ package com.aws.greengrass.util.platforms.unix.linux; +import com.aws.greengrass.lifecyclemanager.GreengrassService; + +import java.io.IOException; import java.nio.file.Path; +import java.util.List; /** * Represents Linux cgroup subsystems. @@ -94,4 +98,24 @@ public Path getCgroupFreezePath(String componentName) { return subSystem.getCgroupFreezePath(componentName); } + public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException { + subSystem.initializeCgroup(component, platform); + } + + public void handleCpuLimits(GreengrassService component, double cpu) throws IOException { + subSystem.handleCpuLimits(component, cpu); + } + + public void pauseComponentProcessesCore(GreengrassService component, List processes) + throws IOException { + subSystem.pauseComponentProcessesCore(component, processes); + } + + public void resumeComponentProcesses(GreengrassService component) throws IOException { + subSystem.resumeComponentProcesses(component); + } + + protected Path freezerCgroupStateFile(String component) { + return getCgroupFreezerStateFilePath(component); + } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java index 9e3aab7855..3dbaac81f9 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java @@ -5,9 +5,17 @@ package com.aws.greengrass.util.platforms.unix.linux; +import com.aws.greengrass.lifecyclemanager.GreengrassService; +import com.aws.greengrass.util.Utils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Set; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CgroupSubSystem virtual filesystem path cannot be relative") @@ -89,4 +97,73 @@ public Path getCgroupProcsPath(String componentName) { public Path getCgroupFreezerStateFilePath(String componentName) { return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); } + + @Override + public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException { + Set mounts = getMountedPaths(); + + if (!mounts.contains(getRootPath().toString())) { + platform.runCmd(rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); + Files.createDirectory(getSubsystemRootPath()); + } + + if (!mounts.contains(getSubsystemRootPath().toString())) { + platform.runCmd(subsystemMountCmd(), o -> { + }, "Failed to mount cgroup subsystem"); + } + if (!Files.exists(getSubsystemGGPath())) { + Files.createDirectory(getSubsystemGGPath()); + } + if (!Files.exists(getSubsystemComponentPath(component.getServiceName()))) { + Files.createDirectory(getSubsystemComponentPath(component.getServiceName())); + } + } + + @Override + public void handleCpuLimits(GreengrassService component, double cpu) throws IOException { + byte[] content = Files.readAllBytes( + getComponentCpuPeriodPath(component.getServiceName())); + int cpuPeriodUs = Integer.parseInt(new String(content, StandardCharsets.UTF_8).trim()); + + int cpuQuotaUs = (int) (cpuPeriodUs * cpu); + String cpuQuotaUsStr = Integer.toString(cpuQuotaUs); + + Files.write(getComponentCpuQuotaPath(component.getServiceName()), + cpuQuotaUsStr.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public void pauseComponentProcessesCore(GreengrassService component, List processes) + throws IOException { + if (LinuxSystemResourceController.CgroupFreezerState.FROZEN.equals( + currentFreezerCgroupState(component.getServiceName()))) { + return; + } + Files.write(getCgroupFreezerStateFilePath(component.getServiceName()), + LinuxSystemResourceController.CgroupFreezerState.FROZEN.toString().getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } + + @Override + public void resumeComponentProcesses(GreengrassService component) throws IOException { + if (LinuxSystemResourceController.CgroupFreezerState.THAWED.equals( + currentFreezerCgroupState(component.getServiceName()))) { + return; + } + + Files.write(getCgroupFreezerStateFilePath(component.getServiceName()), + LinuxSystemResourceController.CgroupFreezerState.THAWED.toString().getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } + + private LinuxSystemResourceController.CgroupFreezerState currentFreezerCgroupState(String component) + throws IOException { + List stateFileContent = + Files.readAllLines(getCgroupFreezerStateFilePath(component)); + if (Utils.isEmpty(stateFileContent) || stateFileContent.size() != 1) { + throw new IOException("Unexpected error reading freezer cgroup state"); + } + return LinuxSystemResourceController.CgroupFreezerState.valueOf(stateFileContent.get(0).trim()); + } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java index c064572cc2..3aef0931cb 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java @@ -5,14 +5,23 @@ package com.aws.greengrass.util.platforms.unix.linux; +import com.aws.greengrass.lifecyclemanager.GreengrassService; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.StringUtils; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Set; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CgroupSubSystemV2 virtual filesystem path cannot be relative") public enum CgroupSubSystemV2 implements CGroupSubSystemPath { Memory, CPU, Freezer, Unified; + private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids"; @Override public String rootMountCmd() { @@ -68,4 +77,69 @@ public Path getComponentCpuMaxPath(String componentName) { public Path getCgroupFreezePath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); } + + @Override + public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException { + Set mounts = getMountedPaths(); + + if (!mounts.contains(getRootPath().toString())) { + platform.runCmd(rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); + Files.createDirectory(getSubsystemRootPath()); + } + + if (!Files.exists(getSubsystemGGPath())) { + Files.createDirectory(getSubsystemGGPath()); + } + if (!Files.exists(getSubsystemComponentPath(component.getServiceName()))) { + Files.createDirectory(getSubsystemComponentPath(component.getServiceName())); + } + + //Enable controllers for root group + Files.write(getRootSubTreeControlPath(), + CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); + //Enable controllers for gg group + Files.write(getGGSubTreeControlPath(), + CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public void handleCpuLimits(GreengrassService component, double cpu) throws IOException { + byte[] content = Files.readAllBytes( + getComponentCpuMaxPath(component.getServiceName())); + String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim(); + String[] cpuMaxContentArr = cpuMaxContent.split(" "); + String cpuMaxStr = "max"; + String cpuPeriodStr = "100000"; + + if (cpuMaxContentArr.length >= 2) { + cpuMaxStr = cpuMaxContentArr[0]; + cpuPeriodStr = cpuMaxContentArr[1]; + + if (!StringUtils.isEmpty(cpuPeriodStr)) { + int period = Integer.parseInt(cpuPeriodStr.trim()); + int max = (int) (period * cpu); + cpuMaxStr = Integer.toString(max); + } + } + + String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr); + Files.write(getComponentCpuMaxPath(component.getServiceName()), + latestCpuMaxContent.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public void pauseComponentProcessesCore(GreengrassService component, List processes) + throws IOException { + Files.write(getCgroupFreezerStateFilePath(component.getServiceName()), + String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } + + @Override + public void resumeComponentProcesses(GreengrassService component) throws IOException { + Files.write(getCgroupFreezerStateFilePath(component.getServiceName()), + String.valueOf(CgroupV2FreezerState.THAWED.getIndex()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java index 13b7c9119e..5084789edf 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java @@ -17,18 +17,15 @@ justification = "Cgroup Controller virtual filesystem path cannot be relative") public class LinuxPlatform extends UnixPlatform { private static final Path CGROUP_CONTROLLERS = Paths.get("/sys/fs/cgroup/cgroup.controllers"); - SystemResourceController systemResourceController; @Override public SystemResourceController getSystemResourceController() { //if the path exists, identify it as cgroupv2, otherwise identify it as cgroupv1 if (Files.exists(CGROUP_CONTROLLERS)) { - systemResourceController = new LinuxSystemResourceControllerV2(this); + return new LinuxSystemResourceController(this, false); } else { - systemResourceController = new LinuxSystemResourceController(this); + return new LinuxSystemResourceController(this, true); } - - return systemResourceController; } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java index ab150f395f..bf49b53367 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java @@ -11,16 +11,13 @@ import com.aws.greengrass.util.Coerce; import com.aws.greengrass.util.Utils; import com.aws.greengrass.util.platforms.SystemResourceController; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.zeroturnaround.process.PidUtil; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -32,37 +29,47 @@ import static org.apache.commons.io.FileUtils.ONE_KB; +@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Cgroup Controller virtual filesystem path cannot be relative") public class LinuxSystemResourceController implements SystemResourceController { private static final Logger logger = LogManager.getLogger(LinuxSystemResourceController.class); - - protected static final String COMPONENT_NAME = "componentName"; - protected static final String MEMORY_KEY = "memory"; - protected static final String CPUS_KEY = "cpus"; - private static final String UNICODE_SPACE = "\\040"; - protected static final String MOUNT_PATH = "/proc/self/mounts"; - protected Cgroup memoryCgroup; - protected Cgroup cpuCgroup; - protected Cgroup freezerCgroup; - protected Cgroup unifiedCgroup; - protected List resourceLimitCgroups; + private static final String COMPONENT_NAME = "componentName"; + private static final String MEMORY_KEY = "memory"; + private static final String CPUS_KEY = "cpus"; + private Cgroup memoryCgroup; + private Cgroup cpuCgroup; + private Cgroup freezerCgroup; + private Cgroup unifiedCgroup; + private List resourceLimitCgroups; protected CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); protected LinuxPlatform platform; - protected Set mounts; - - public LinuxSystemResourceController() { + /** + * LinuxSystemResourceController Constructor. + * + * @param platform platform + * @param isV1Used if you use v1 + */ + public LinuxSystemResourceController(LinuxPlatform platform, boolean isV1Used) { + this.platform = platform; + if (isV1Used) { + this.memoryCgroup = new Cgroup(CgroupSubSystem.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystem.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystem.Freezer); + resourceLimitCgroups = Arrays.asList( + memoryCgroup, cpuCgroup); + } else { + this.unifiedCgroup = new Cgroup(CgroupSubSystemV2.Unified); + this.memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystemV2.Freezer); + resourceLimitCgroups = Arrays.asList(unifiedCgroup); + } } - LinuxSystemResourceController(LinuxPlatform platform) { - this.platform = platform; - this.memoryCgroup = new Cgroup(CgroupSubSystem.Memory); - this.cpuCgroup = new Cgroup(CgroupSubSystem.CPU); - this.freezerCgroup = new Cgroup(CgroupSubSystem.Freezer); - resourceLimitCgroups = Arrays.asList( - memoryCgroup, cpuCgroup); - } + @Override @@ -89,15 +96,7 @@ public void updateResourceLimits(GreengrassService component, Map 0) { - byte[] content = Files.readAllBytes( - cpuCgroup.getComponentCpuPeriodPath(component.getServiceName())); - int cpuPeriodUs = Integer.parseInt(new String(content, StandardCharsets.UTF_8).trim()); - - int cpuQuotaUs = (int) (cpuPeriodUs * cpu); - String cpuQuotaUsStr = Integer.toString(cpuQuotaUs); - - Files.write(cpuCgroup.getComponentCpuQuotaPath(component.getServiceName()), - cpuQuotaUsStr.getBytes(StandardCharsets.UTF_8)); + cpuCgroup.handleCpuLimits(component, cpu); } else { logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) @@ -170,23 +169,13 @@ public void addComponentProcess(GreengrassService component, Process process) { @Override public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { prePauseComponentProcesses(component, processes); - if (CgroupFreezerState.FROZEN.equals(currentFreezerCgroupState(component.getServiceName()))) { - return; - } - Files.write(freezerCgroupStateFile(component.getServiceName()), - CgroupFreezerState.FROZEN.toString().getBytes(StandardCharsets.UTF_8), - StandardOpenOption.TRUNCATE_EXISTING); + freezerCgroup.pauseComponentProcessesCore(component, processes); } @Override public void resumeComponentProcesses(GreengrassService component) throws IOException { - if (CgroupFreezerState.THAWED.equals(currentFreezerCgroupState(component.getServiceName()))) { - return; - } - Files.write(freezerCgroupStateFile(component.getServiceName()), - CgroupFreezerState.THAWED.toString().getBytes(StandardCharsets.UTF_8), - StandardOpenOption.TRUNCATE_EXISTING); + freezerCgroup.resumeComponentProcesses(component); } protected void addComponentProcessToCgroup(String component, Process process, Cgroup cg) @@ -238,56 +227,8 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) { } } - /** - * Get mounted path for Cgroup. - * @return mounted path set - * @throws IOException IOException - */ - public Set getMountedPaths() throws IOException { - Set mountedPaths = new HashSet<>(); - - Path procMountsPath = Paths.get(MOUNT_PATH); - List mounts = Files.readAllLines(procMountsPath); - for (String mount : mounts) { - String[] split = mount.split(" "); - // As reported in fstab(5) manpage, struct is: - // 1st field is volume name - // 2nd field is path with spaces escaped as \040 - // 3rd field is fs type - // 4th field is mount options - // 5th field is used by dump(8) (ignored) - // 6th field is fsck order (ignored) - if (split.length < 6) { - continue; - } - - // We only need the path of the mounts to verify whether cgroup is mounted - String path = split[1].replace(UNICODE_SPACE, " "); - mountedPaths.add(path); - } - return mountedPaths; - } - protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { - mounts = getMountedPaths(); - - if (!mounts.contains(cgroup.getRootPath().toString())) { - platform.runCmd(cgroup.rootMountCmd(), o -> { - }, "Failed to mount cgroup root"); - Files.createDirectory(cgroup.getSubsystemRootPath()); - } - - if (!mounts.contains(cgroup.getSubsystemRootPath().toString())) { - platform.runCmd(cgroup.subsystemMountCmd(), o -> { - }, "Failed to mount cgroup subsystem"); - } - if (!Files.exists(cgroup.getSubsystemGGPath())) { - Files.createDirectory(cgroup.getSubsystemGGPath()); - } - if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { - Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); - } - + cgroup.initializeCgroup(component, platform); usedCgroups.add(cgroup); } @@ -296,18 +237,6 @@ private Set pidsInComponentCgroup(Cgroup cgroup, String component) thro .stream().map(Integer::parseInt).collect(Collectors.toSet()); } - protected Path freezerCgroupStateFile(String component) { - return freezerCgroup.getCgroupFreezerStateFilePath(component); - } - - private CgroupFreezerState currentFreezerCgroupState(String component) throws IOException { - List stateFileContent = - Files.readAllLines(freezerCgroupStateFile(component)); - if (Utils.isEmpty(stateFileContent) || stateFileContent.size() != 1) { - throw new IOException("Unexpected error reading freezer cgroup state"); - } - return CgroupFreezerState.valueOf(stateFileContent.get(0).trim()); - } protected void prePauseComponentProcesses(GreengrassService component, List processes) throws IOException { initializeCgroup(component, freezerCgroup); diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java deleted file mode 100644 index 6e10e34e62..0000000000 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.aws.greengrass.util.platforms.unix.linux; - -import com.aws.greengrass.lifecyclemanager.GreengrassService; -import com.aws.greengrass.logging.api.Logger; -import com.aws.greengrass.logging.impl.LogManager; -import com.aws.greengrass.util.Coerce; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -/** - * https://www.kernel.org/doc/Documentation/cgroup-v2.txt - */ -public class LinuxSystemResourceControllerV2 extends LinuxSystemResourceController { - private static final Logger logger = LogManager.getLogger(LinuxSystemResourceControllerV2.class); - private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids"; - - /** - * Linux system resource controller V2 constrcutors. - * - * @param platform linux platform - */ - public LinuxSystemResourceControllerV2(LinuxPlatform platform) { - super(); - this.platform = platform; - this.unifiedCgroup = new Cgroup(CgroupSubSystemV2.Unified); - this.memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); - this.cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); - this.freezerCgroup = new Cgroup(CgroupSubSystemV2.Freezer); - resourceLimitCgroups = Arrays.asList(unifiedCgroup); - } - - - @Override - public void updateResourceLimits(GreengrassService component, Map resourceLimit) { - try { - super.updateMemoryResourceLimits(component, resourceLimit); - - if (resourceLimit.containsKey(CPUS_KEY)) { - double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY)); - if (cpu > 0) { - handleCpuLimits(component, cpu); - } else { - logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) - .log("The provided cpu limit is invalid"); - } - } - } catch (IOException e) { - logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName()) - .log("Failed to apply resource limits"); - } - } - - @Override - public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { - prePauseComponentProcesses(component, processes); - - Files.write(freezerCgroupStateFile(component.getServiceName()), - String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8), - StandardOpenOption.TRUNCATE_EXISTING); - } - - @Override - public void resumeComponentProcesses(GreengrassService component) throws IOException { - Files.write(freezerCgroupStateFile(component.getServiceName()), - String.valueOf(CgroupV2FreezerState.THAWED.getIndex()).getBytes(StandardCharsets.UTF_8), - StandardOpenOption.TRUNCATE_EXISTING); - } - - @Override - protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { - mounts = getMountedPaths(); - - if (!mounts.contains(cgroup.getRootPath().toString())) { - platform.runCmd(cgroup.rootMountCmd(), o -> { - }, "Failed to mount cgroup root"); - Files.createDirectory(cgroup.getSubsystemRootPath()); - } - - if (!Files.exists(cgroup.getSubsystemGGPath())) { - Files.createDirectory(cgroup.getSubsystemGGPath()); - } - if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { - Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); - } - - //Enable controllers for root group - Files.write(cgroup.getRootSubTreeControlPath(), - CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); - //Enable controllers for gg group - Files.write(cgroup.getGGSubTreeControlPath(), - CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); - - usedCgroups.add(cgroup); - } - - private void handleCpuLimits(GreengrassService component, double cpu) throws IOException { - byte[] content = Files.readAllBytes( - cpuCgroup.getComponentCpuMaxPath(component.getServiceName())); - String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim(); - String[] cpuMaxContentArr = cpuMaxContent.split(" "); - String cpuMaxStr = "max"; - String cpuPeriodStr = "100000"; - - if (cpuMaxContentArr.length >= 2) { - cpuMaxStr = cpuMaxContentArr[0]; - cpuPeriodStr = cpuMaxContentArr[1]; - - if (!StringUtils.isEmpty(cpuPeriodStr)) { - int period = Integer.parseInt(cpuPeriodStr.trim()); - int max = (int) (period * cpu); - cpuMaxStr = Integer.toString(max); - } - } - - String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr); - Files.write(cpuCgroup.getComponentCpuMaxPath(component.getServiceName()), - latestCpuMaxContent.getBytes(StandardCharsets.UTF_8)); - } - -} diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java index b6f43377a7..a9e02e016b 100644 --- a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -8,6 +8,7 @@ import com.aws.greengrass.lifecyclemanager.GreengrassService; import com.aws.greengrass.testcommons.testutilities.GGExtension; import com.aws.greengrass.util.Utils; +import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; @@ -28,25 +29,28 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; + @ExtendWith({MockitoExtension.class, GGExtension.class}) @DisabledOnOs(OS.WINDOWS) class LinuxSystemResourceControllerV2Test { @Mock GreengrassService component; @Mock - Cgroup cpuCgroup; - @Mock - Cgroup memoryCgroup; - @Mock LinuxPlatform platform; + @Spy + Cgroup cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); + @Spy + Cgroup memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); @InjectMocks @Spy - LinuxSystemResourceControllerV2 linuxSystemResourceControllerV2 = new LinuxSystemResourceControllerV2(platform); + LinuxSystemResourceController linuxSystemResourceControllerV2 = new LinuxSystemResourceController( + platform, false); + private static final String FILE_PATH = "/cgroupv2test"; private static final String CGROUP_MEMORY_LIMIT_FILE_NAME = "memory.txt"; private static final String CGROUP_CPU_LIMIT_FILE_NAME = "cpu.txt"; @@ -71,13 +75,13 @@ void GIVEN_cgroupv2_WHEN_memory_limit_updated_THEN_memory_limit_file_updated() t when(memoryCgroup.getComponentMemoryLimitPath(COMPONENT_NAME)).thenReturn(path); lenient().when(memoryCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); + when(cpuCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(path); + linuxSystemResourceControllerV2.updateResourceLimits(component, resourceLimit); List mounts = Files.readAllLines(path); assertEquals(String.valueOf(MEMORY_IN_KB * 1024), mounts.get(0)); - - Files.deleteIfExists(path); - Files.deleteIfExists(componentNameFolderPath); + FileUtils.deleteDirectory(componentNameFolderPath.toFile()); } @Test @@ -97,10 +101,10 @@ void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws Files.write(path, "max 100000".getBytes(StandardCharsets.UTF_8)); - doNothing().when(linuxSystemResourceControllerV2).updateMemoryResourceLimits(component, resourceLimit); - when(cpuCgroup.getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); - lenient().when(cpuCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); - linuxSystemResourceControllerV2.updateResourceLimits(component, resourceLimit); + CGroupSubSystemPath cpuSystemV2 = spy(CgroupSubSystemV2.CPU); + lenient().when(cpuSystemV2.getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); + + cpuSystemV2.handleCpuLimits(component, 0.5); List mounts = Files.readAllLines(path); assertEquals((int) (CPU_TIME * 100000) + " 100000", mounts.get(0)); From b8cd78190f95b485503d820ecdb1384608e7cfbc Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Fri, 4 Nov 2022 16:13:36 +0800 Subject: [PATCH 05/11] feat: linux control group version 2 API support cgroup v2 --- .../ipc/IPCHibernateTest.java | 5 +- .../GenericExternalServiceIntegTest.java | 87 ++----------- ...temPath.java => CGroupSubSystemPaths.java} | 16 ++- .../{CgroupSubSystem.java => CGroupV1.java} | 23 +--- .../{CgroupSubSystemV2.java => CGroupV2.java} | 22 +--- .../util/platforms/unix/linux/Cgroup.java | 121 ------------------ .../linux/LinuxSystemResourceController.java | 36 +++--- .../LinuxSystemResourceControllerV2Test.java | 6 +- 8 files changed, 52 insertions(+), 264 deletions(-) rename src/main/java/com/aws/greengrass/util/platforms/unix/linux/{CGroupSubSystemPath.java => CGroupSubSystemPaths.java} (88%) rename src/main/java/com/aws/greengrass/util/platforms/unix/linux/{CgroupSubSystem.java => CGroupV1.java} (89%) rename src/main/java/com/aws/greengrass/util/platforms/unix/linux/{CgroupSubSystemV2.java => CGroupV2.java} (88%) delete mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java index 597749aabe..c6ea202df4 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java @@ -9,8 +9,7 @@ import com.aws.greengrass.lifecyclemanager.Kernel; import com.aws.greengrass.testcommons.testutilities.GGExtension; import com.aws.greengrass.testcommons.testutilities.TestUtils; -import com.aws.greengrass.util.platforms.unix.linux.Cgroup; -import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; +import com.aws.greengrass.util.platforms.unix.linux.CGroupV1; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -116,7 +115,7 @@ void GIVEN_LifeCycleEventStreamClient_WHEN_pause_resume_component_THEN_target_se private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState.valueOf( - new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)), + new String(Files.readAllBytes(CGroupV1.Freezer.getCgroupFreezerStateFilePath(serviceName)), StandardCharsets.UTF_8).trim()); } } diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index 7ad911722a..ad0f8bde3c 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -21,15 +21,9 @@ import com.aws.greengrass.testcommons.testutilities.GGExtension; import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler; import com.aws.greengrass.util.Pair; -import com.aws.greengrass.util.platforms.SystemResourceController; -import com.aws.greengrass.util.platforms.unix.linux.Cgroup; -import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; -import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2; -import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform; +import com.aws.greengrass.util.platforms.unix.linux.CGroupV1; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.SystemUtils; -import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,11 +34,9 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; -import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystemException; import java.nio.file.Files; @@ -95,14 +87,9 @@ class GenericExternalServiceIntegTest extends BaseITCase { private Kernel kernel; - @Spy - LinuxPlatform linuxPlatform; - - private final static String ROOT_PATH_STRING = "/systest21/fs/cgroup"; + private final static String ROOT_PATH_STRING = "/sys/fs/cgroup"; private final static String GG_PATH_STRING = "greengrass"; - SystemResourceController systemResourceController; - static Stream posixTestUserConfig() { return Stream.of( arguments("config_run_with_user.yaml", "nobody", "nobody"), @@ -551,7 +538,7 @@ void GIVEN_posix_default_user_WHEN_runs_THEN_runs_with_default_user(String file, String messageOnStdout = m.getMessage(); if (STDOUT.equals(m.getEventType()) && messageOnStdout != null && (messageOnStdout.contains("run as") - || messageOnStdout.contains("install as") )) { + || messageOnStdout.contains("install as") )) { stdouts.add(messageOnStdout); countDownLatch.countDown(); } @@ -642,10 +629,9 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re @EnabledOnOs({OS.LINUX}) @Test void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_resource_limits_V2() throws Exception { + assumeTrue(ifCgroupV2(), "skip this test case if v1 is enabled."); + String echoComponentName = "echo_service"; - String mainComponentName = "main"; - String rootGGPathString = ROOT_PATH_STRING + "/" + GG_PATH_STRING; - String componentPathString = rootGGPathString + "/" + echoComponentName; // Run with no resource limit ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel, getClass().getResource("config_run_with_user.yaml")); @@ -656,37 +642,6 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re } }); - linuxPlatform = spy(kernel.getContext().get(LinuxPlatform.class)); - - createComponentData(echoComponentName); - createComponentData(mainComponentName); - - // Due to cgroup v1 is active by default (in test platform), and the directories of cgroup v1 are read-only - // therefore, here create some directories and files as fake cgroup v2 files to support testing - Field controllerField = LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS"); - setFinalStatic(controllerField, Paths.get(componentPathString + "/memory.max")); - systemResourceController = linuxPlatform.getSystemResourceController(); - LinuxSystemResourceController controllerV2 = (LinuxSystemResourceController) systemResourceController; - Field memoryCgroupField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("memoryCgroup"); - memoryCgroupField.setAccessible(true); - Cgroup memoryCgroup = (Cgroup) memoryCgroupField.get(controllerV2); - Field subsystem = memoryCgroup.getClass().getDeclaredField("subSystem"); - subsystem.setAccessible(true); - CgroupSubSystemV2 cg = (CgroupSubSystemV2) subsystem.get(memoryCgroup); - Field f = cg.getClass().getInterfaces()[0].getDeclaredField("CGROUP_ROOT"); - setFinalStatic(f, Paths.get(ROOT_PATH_STRING)); - - Field mountsField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("MOUNT_PATH"); - mountsField.setAccessible(true); - String mountPathFile = rootGGPathString + "/mountPath.txt"; - final Path mountPathFilePath = Paths.get(mountPathFile); - if (!Files.exists(mountPathFilePath)) { - Files.createFile(mountPathFilePath); - Files.write(mountPathFilePath, String.format("test1 %s test2 test3 test4 test5", ROOT_PATH_STRING).getBytes(StandardCharsets.UTF_8)); - - } - setFinalStatic(mountsField, mountPathFile); - kernel.launch(); assertResourceLimits_V2(10240l * 1024, 1.5); @@ -707,14 +662,10 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re kernel.getContext().waitForPublishQueueToClear(); assertResourceLimits_V2(10240l * 1024, 1.5); - - FileUtils.deleteDirectory(Paths.get("/systest21").toFile()); } - private void setFinalStatic(Field field, Object newValue) throws Exception { - field.setAccessible(true); - FieldUtils.removeFinalModifier(field, true); - field.set(null, newValue); + private boolean ifCgroupV2() { + return Files.exists(Paths.get("/sys/fs/cgroup/cgroup.controllers")); } @Test @@ -848,11 +799,11 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers } private void assertResourceLimits(String componentName, long memory, double cpus) throws Exception { - byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystem.Memory).getComponentMemoryLimitPath(componentName)); + byte[] buf1 = Files.readAllBytes(CGroupV1.Memory.getComponentMemoryLimitPath(componentName)); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); - byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuQuotaPath(componentName)); - byte[] buf3 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuPeriodPath(componentName)); + byte[] buf2 = Files.readAllBytes(CGroupV1.CPU.getComponentCpuQuotaPath(componentName)); + byte[] buf3 = Files.readAllBytes(CGroupV1.CPU.getComponentCpuPeriodPath(componentName)); int quota = Integer.parseInt(new String(buf2, StandardCharsets.UTF_8).trim()); int period = Integer.parseInt(new String(buf3, StandardCharsets.UTF_8).trim()); @@ -860,22 +811,6 @@ private void assertResourceLimits(String componentName, long memory, double cpus assertThat(expectedQuota, equalTo(quota)); } - private void createComponentData(String componentName) throws IOException { - Path path = Paths.get(ROOT_PATH_STRING).resolve(GG_PATH_STRING).resolve(componentName); - Files.createDirectories(path); - - if (!Files.exists(path.resolve("memory.max"))) { - Files.createFile(path.resolve("memory.max")); - } - if (!Files.exists(path.resolve("cpu.max"))) { - Files.createFile(path.resolve("cpu.max")); - } - Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8)); - if (!Files.exists(path.resolve("cgroup.procs"))) { - Files.createFile(path.resolve("cgroup.procs")); - } - } - private void assertResourceLimits_V2(long memory, double cpus) throws Exception { byte[] buf1 = Files.readAllBytes(Paths.get(String.format("%s/%s/echo_service/memory.max", ROOT_PATH_STRING, GG_PATH_STRING))); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); @@ -928,7 +863,7 @@ void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState - .valueOf(new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)) + .valueOf(new String(Files.readAllBytes(CGroupV1.Freezer.getCgroupFreezerStateFilePath(serviceName)) , StandardCharsets.UTF_8).trim()); } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPaths.java similarity index 88% rename from src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java rename to src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPaths.java index e99c3130a0..61ffd1dd21 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPaths.java @@ -18,7 +18,7 @@ @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CGroupSubSystemPath virtual filesystem path cannot be relative") -public interface CGroupSubSystemPath { +public interface CGroupSubSystemPaths { Path CGROUP_ROOT = Paths.get("/sys/fs/cgroup"); String GG_NAMESPACE = "greengrass"; String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes"; @@ -45,9 +45,13 @@ default String subsystemMountCmd() { Path getSubsystemRootPath(); - Path getSubsystemGGPath(); + default Path getSubsystemGGPath() { + return getSubsystemRootPath().resolve(GG_NAMESPACE); + } - Path getSubsystemComponentPath(String componentName); + default Path getSubsystemComponentPath(String componentName) { + return getSubsystemGGPath().resolve(componentName); + } Path getComponentMemoryLimitPath(String componentName); @@ -59,7 +63,9 @@ default Path getComponentCpuQuotaPath(String componentName) { return null; } - Path getCgroupProcsPath(String componentName); + default Path getCgroupProcsPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); + } Path getCgroupFreezerStateFilePath(String componentName); @@ -83,7 +89,7 @@ default Path getCgroupFreezePath(String componentName) { void handleCpuLimits(GreengrassService component, double cpu) throws IOException; - void pauseComponentProcessesCore(GreengrassService component, List processes) throws IOException; + void pauseComponentProcessesCore(GreengrassService component) throws IOException; void resumeComponentProcesses(GreengrassService component) throws IOException; diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV1.java similarity index 89% rename from src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java rename to src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV1.java index 3dbaac81f9..73d2a3cf78 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV1.java @@ -18,14 +18,14 @@ import java.util.Set; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", - justification = "CgroupSubSystem virtual filesystem path cannot be relative") -public enum CgroupSubSystem implements CGroupSubSystemPath { + justification = "CGroupV1 virtual filesystem path cannot be relative") +public enum CGroupV1 implements CGroupSubSystemPaths { Memory("memory", ""), CPU("cpu,cpuacct", ""), Freezer("freezer", "freezer"); private String osString; private String mountSrc; - CgroupSubSystem(String osString, String mountSrc) { + CGroupV1(String osString, String mountSrc) { this.osString = osString; this.mountSrc = mountSrc; } @@ -63,16 +63,6 @@ public Path getSubsystemRootPath() { return CGROUP_ROOT.resolve(osString); } - @Override - public Path getSubsystemGGPath() { - return getSubsystemRootPath().resolve(GG_NAMESPACE); - } - - @Override - public Path getSubsystemComponentPath(String componentName) { - return getSubsystemGGPath().resolve(componentName); - } - @Override public Path getComponentMemoryLimitPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); @@ -88,11 +78,6 @@ public Path getComponentCpuQuotaPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CPU_CFS_QUOTA_US); } - @Override - public Path getCgroupProcsPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); - } - @Override public Path getCgroupFreezerStateFilePath(String componentName) { return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); @@ -134,7 +119,7 @@ public void handleCpuLimits(GreengrassService component, double cpu) throws IOEx } @Override - public void pauseComponentProcessesCore(GreengrassService component, List processes) + public void pauseComponentProcessesCore(GreengrassService component) throws IOException { if (LinuxSystemResourceController.CgroupFreezerState.FROZEN.equals( currentFreezerCgroupState(component.getServiceName()))) { diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV2.java similarity index 88% rename from src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java rename to src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV2.java index 3aef0931cb..d1efd2872b 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV2.java @@ -14,12 +14,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.List; import java.util.Set; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", - justification = "CgroupSubSystemV2 virtual filesystem path cannot be relative") -public enum CgroupSubSystemV2 implements CGroupSubSystemPath { + justification = "CGroupV2 virtual filesystem path cannot be relative") +public enum CGroupV2 implements CGroupSubSystemPaths { Memory, CPU, Freezer, Unified; private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids"; @@ -33,26 +32,11 @@ public Path getSubsystemRootPath() { return CGROUP_ROOT; } - @Override - public Path getSubsystemGGPath() { - return getSubsystemRootPath().resolve(GG_NAMESPACE); - } - - @Override - public Path getSubsystemComponentPath(String componentName) { - return getSubsystemGGPath().resolve(componentName); - } - @Override public Path getComponentMemoryLimitPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX); } - @Override - public Path getCgroupProcsPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); - } - @Override public Path getCgroupFreezerStateFilePath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); @@ -129,7 +113,7 @@ public void handleCpuLimits(GreengrassService component, double cpu) throws IOEx } @Override - public void pauseComponentProcessesCore(GreengrassService component, List processes) + public void pauseComponentProcessesCore(GreengrassService component) throws IOException { Files.write(getCgroupFreezerStateFilePath(component.getServiceName()), String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8), diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java deleted file mode 100644 index d28dece13b..0000000000 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.aws.greengrass.util.platforms.unix.linux; - -import com.aws.greengrass.lifecyclemanager.GreengrassService; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; - -/** - * Represents Linux cgroup subsystems. - */ -public class Cgroup { - private final CGroupSubSystemPath subSystem; - - public Cgroup(CGroupSubSystemPath subSystem) { - this.subSystem = subSystem; - } - - public Path getRootPath() { - return subSystem.getRootPath(); - } - - /** - * root mount cmd. - * - * @return mount command string - */ - public String rootMountCmd() { - return subSystem.rootMountCmd(); - } - - public String subsystemMountCmd() { - return subSystem.subsystemMountCmd(); - } - - public Path getSubsystemRootPath() { - return subSystem.getSubsystemRootPath(); - } - - public Path getSubsystemGGPath() { - return subSystem.getSubsystemGGPath(); - } - - public Path getSubsystemComponentPath(String componentName) { - return subSystem.getSubsystemComponentPath(componentName); - } - - /** - * get component memory limit path. - * - * @param componentName componentName - * @return memory limit Path - */ - public Path getComponentMemoryLimitPath(String componentName) { - return subSystem.getComponentMemoryLimitPath(componentName); - } - - public Path getComponentCpuPeriodPath(String componentName) { - return subSystem.getComponentCpuPeriodPath(componentName); - } - - public Path getComponentCpuQuotaPath(String componentName) { - return subSystem.getComponentCpuQuotaPath(componentName); - } - - public Path getCgroupProcsPath(String componentName) { - return subSystem.getCgroupProcsPath(componentName); - } - - /** - * get cgroup freezer path. - * - * @param componentName componentName - * @return cgroup freezer path - */ - public Path getCgroupFreezerStateFilePath(String componentName) { - return subSystem.getCgroupFreezerStateFilePath(componentName); - } - - public Path getRootSubTreeControlPath() { - return subSystem.getRootSubTreeControlPath(); - } - - public Path getGGSubTreeControlPath() { - return subSystem.getGGSubTreeControlPath(); - } - - public Path getComponentCpuMaxPath(String componentName) { - return subSystem.getComponentCpuMaxPath(componentName); - } - - public Path getCgroupFreezePath(String componentName) { - return subSystem.getCgroupFreezePath(componentName); - } - - public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException { - subSystem.initializeCgroup(component, platform); - } - - public void handleCpuLimits(GreengrassService component, double cpu) throws IOException { - subSystem.handleCpuLimits(component, cpu); - } - - public void pauseComponentProcessesCore(GreengrassService component, List processes) - throws IOException { - subSystem.pauseComponentProcessesCore(component, processes); - } - - public void resumeComponentProcesses(GreengrassService component) throws IOException { - subSystem.resumeComponentProcesses(component); - } - - protected Path freezerCgroupStateFile(String component) { - return getCgroupFreezerStateFilePath(component); - } -} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java index bf49b53367..3252a58827 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java @@ -36,12 +36,12 @@ public class LinuxSystemResourceController implements SystemResourceController { private static final String COMPONENT_NAME = "componentName"; private static final String MEMORY_KEY = "memory"; private static final String CPUS_KEY = "cpus"; - private Cgroup memoryCgroup; - private Cgroup cpuCgroup; - private Cgroup freezerCgroup; - private Cgroup unifiedCgroup; - private List resourceLimitCgroups; - protected CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); + private CGroupSubSystemPaths memoryCgroup; + private CGroupSubSystemPaths cpuCgroup; + private CGroupSubSystemPaths freezerCgroup; + private CGroupSubSystemPaths unifiedCgroup; + private List resourceLimitCgroups; + protected CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); protected LinuxPlatform platform; @@ -55,16 +55,16 @@ public LinuxSystemResourceController(LinuxPlatform platform, boolean isV1Used) { this.platform = platform; if (isV1Used) { - this.memoryCgroup = new Cgroup(CgroupSubSystem.Memory); - this.cpuCgroup = new Cgroup(CgroupSubSystem.CPU); - this.freezerCgroup = new Cgroup(CgroupSubSystem.Freezer); + this.memoryCgroup = CGroupV1.Memory; + this.cpuCgroup = CGroupV1.CPU; + this.freezerCgroup = CGroupV1.Freezer; resourceLimitCgroups = Arrays.asList( memoryCgroup, cpuCgroup); } else { - this.unifiedCgroup = new Cgroup(CgroupSubSystemV2.Unified); - this.memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); - this.cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); - this.freezerCgroup = new Cgroup(CgroupSubSystemV2.Freezer); + this.unifiedCgroup = CGroupV2.Unified; + this.memoryCgroup = CGroupV2.Memory; + this.cpuCgroup = CGroupV2.CPU; + this.freezerCgroup = CGroupV2.Freezer; resourceLimitCgroups = Arrays.asList(unifiedCgroup); } } @@ -129,7 +129,7 @@ protected void updateMemoryResourceLimits(GreengrassService component, @Override public void resetResourceLimits(GreengrassService component) { - for (Cgroup cg : resourceLimitCgroups) { + for (CGroupSubSystemPaths cg : resourceLimitCgroups) { try { if (Files.exists(cg.getSubsystemComponentPath(component.getServiceName()))) { Files.delete(cg.getSubsystemComponentPath(component.getServiceName())); @@ -169,7 +169,7 @@ public void addComponentProcess(GreengrassService component, Process process) { @Override public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { prePauseComponentProcesses(component, processes); - freezerCgroup.pauseComponentProcessesCore(component, processes); + freezerCgroup.pauseComponentProcessesCore(component); } @@ -178,7 +178,7 @@ public void resumeComponentProcesses(GreengrassService component) throws IOExcep freezerCgroup.resumeComponentProcesses(component); } - protected void addComponentProcessToCgroup(String component, Process process, Cgroup cg) + protected void addComponentProcessToCgroup(String component, Process process, CGroupSubSystemPaths cg) throws IOException { if (!Files.exists(cg.getSubsystemComponentPath(component))) { @@ -227,12 +227,12 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) { } } - protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { + protected void initializeCgroup(GreengrassService component, CGroupSubSystemPaths cgroup) throws IOException { cgroup.initializeCgroup(component, platform); usedCgroups.add(cgroup); } - private Set pidsInComponentCgroup(Cgroup cgroup, String component) throws IOException { + private Set pidsInComponentCgroup(CGroupSubSystemPaths cgroup, String component) throws IOException { return Files.readAllLines(cgroup.getCgroupProcsPath(component)) .stream().map(Integer::parseInt).collect(Collectors.toSet()); } diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java index a9e02e016b..58dff9ed12 100644 --- a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -43,9 +43,9 @@ class LinuxSystemResourceControllerV2Test { @Mock LinuxPlatform platform; @Spy - Cgroup cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); + CGroupSubSystemPaths cpuCgroup = CGroupV2.CPU; @Spy - Cgroup memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); + CGroupSubSystemPaths memoryCgroup = CGroupV2.Memory; @InjectMocks @Spy LinuxSystemResourceController linuxSystemResourceControllerV2 = new LinuxSystemResourceController( @@ -101,7 +101,7 @@ void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws Files.write(path, "max 100000".getBytes(StandardCharsets.UTF_8)); - CGroupSubSystemPath cpuSystemV2 = spy(CgroupSubSystemV2.CPU); + CGroupSubSystemPaths cpuSystemV2 = spy(CGroupV2.CPU); lenient().when(cpuSystemV2.getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); cpuSystemV2.handleCpuLimits(component, 0.5); From 58441a7931777567157391605fcb37e875088658 Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Tue, 8 Nov 2022 13:14:27 +0800 Subject: [PATCH 06/11] feat: linux control group version 2 API support cgroup v2 --- .../unix/linux/LinuxSystemResourceControllerV2Test.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java index 58dff9ed12..9788500a32 100644 --- a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -86,8 +86,6 @@ void GIVEN_cgroupv2_WHEN_memory_limit_updated_THEN_memory_limit_file_updated() t @Test void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws IOException { - Map resourceLimit = new HashMap<>(); - resourceLimit.put("cpus", String.valueOf(CPU_TIME)); doReturn("testComponentName").when(component).getServiceName(); Path path = Paths.get(FILE_PATH + "/" + CGROUP_CPU_LIMIT_FILE_NAME); From 8ef01430473beb92117c03bfffa929ed5c52fbed Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Fri, 11 Nov 2022 15:28:55 +0800 Subject: [PATCH 07/11] feat: linux control group version 2 API support cgroup v2 --- .../GenericExternalServiceIntegTest.java | 8 +-- .../unix/linux/CGroupSubSystemPaths.java | 53 ++++++++++--------- .../util/platforms/unix/linux/CGroupV1.java | 30 +++-------- .../util/platforms/unix/linux/CGroupV2.java | 26 +-------- .../unix/linux/InitializeCgroup.java | 13 +++++ .../LinuxSystemResourceControllerV2Test.java | 2 +- 6 files changed, 55 insertions(+), 77 deletions(-) create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/InitializeCgroup.java diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index ad0f8bde3c..c544ff79bc 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -643,7 +643,7 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re }); kernel.launch(); - assertResourceLimits_V2(10240l * 1024, 1.5); + assertResourceLimitsCgroupV2(10240l * 1024, 1.5); // Run with updated component resource limit kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC, @@ -654,14 +654,14 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re // Block until events are completed kernel.getContext().waitForPublishQueueToClear(); - assertResourceLimits_V2(51200l * 1024, 0.35); + assertResourceLimitsCgroupV2(51200l * 1024, 0.35); //Remove component resource limit, should fall back to default kernel.getConfig().lookupTopics(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC, SYSTEM_RESOURCE_LIMITS_TOPICS).remove(); kernel.getContext().waitForPublishQueueToClear(); - assertResourceLimits_V2(10240l * 1024, 1.5); + assertResourceLimitsCgroupV2(10240l * 1024, 1.5); } private boolean ifCgroupV2() { @@ -811,7 +811,7 @@ private void assertResourceLimits(String componentName, long memory, double cpus assertThat(expectedQuota, equalTo(quota)); } - private void assertResourceLimits_V2(long memory, double cpus) throws Exception { + private void assertResourceLimitsCgroupV2(long memory, double cpus) throws Exception { byte[] buf1 = Files.readAllBytes(Paths.get(String.format("%s/%s/echo_service/memory.max", ROOT_PATH_STRING, GG_PATH_STRING))); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPaths.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPaths.java index 61ffd1dd21..32bec8b12e 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPaths.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPaths.java @@ -39,10 +39,6 @@ default Path getRootPath() { String rootMountCmd(); - default String subsystemMountCmd() { - return null; - } - Path getSubsystemRootPath(); default Path getSubsystemGGPath() { @@ -55,38 +51,45 @@ default Path getSubsystemComponentPath(String componentName) { Path getComponentMemoryLimitPath(String componentName); - default Path getComponentCpuPeriodPath(String componentName) { - return null; - } - - default Path getComponentCpuQuotaPath(String componentName) { - return null; - } - default Path getCgroupProcsPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); } Path getCgroupFreezerStateFilePath(String componentName); - default Path getRootSubTreeControlPath() { - return null; - } + void initializeCgroup(GreengrassService component, LinuxPlatform platform) + throws IOException; - default Path getGGSubTreeControlPath() { - return null; - } + /** + * Initialize cgroup core method. + * + * @param component component + * @param platform platform + * @param mountSubSystem mount subsystem method + * @throws IOException IOException + */ + default void initializeCgroupCore(GreengrassService component, LinuxPlatform platform, + InitializeCgroup mountSubSystem) throws IOException { + Set mounts = getMountedPaths(); + + if (!mounts.contains(getRootPath().toString())) { + platform.runCmd(rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); + Files.createDirectory(getSubsystemRootPath()); + } - default Path getComponentCpuMaxPath(String componentName) { - return null; - } + if (!mounts.contains(getSubsystemRootPath().toString())) { + mountSubSystem.add(); + } - default Path getCgroupFreezePath(String componentName) { - return null; + if (!Files.exists(getSubsystemGGPath())) { + Files.createDirectory(getSubsystemGGPath()); + } + if (!Files.exists(getSubsystemComponentPath(component.getServiceName()))) { + Files.createDirectory(getSubsystemComponentPath(component.getServiceName())); + } } - void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException; - void handleCpuLimits(GreengrassService component, double cpu) throws IOException; void pauseComponentProcessesCore(GreengrassService component) throws IOException; diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV1.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV1.java index 73d2a3cf78..a51285ea12 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV1.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV1.java @@ -15,7 +15,6 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.List; -import java.util.Set; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CGroupV1 virtual filesystem path cannot be relative") @@ -53,7 +52,6 @@ public String rootMountCmd() { return String.format("mount -t tmpfs cgroup %s", CGROUP_ROOT); } - @Override public String subsystemMountCmd() { return String.format("mount -t cgroup -o %s %s %s", osString, mountSrc, getSubsystemRootPath()); } @@ -68,12 +66,10 @@ public Path getComponentMemoryLimitPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); } - @Override public Path getComponentCpuPeriodPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CPU_CFS_PERIOD_US); } - @Override public Path getComponentCpuQuotaPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CPU_CFS_QUOTA_US); } @@ -84,25 +80,13 @@ public Path getCgroupFreezerStateFilePath(String componentName) { } @Override - public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException { - Set mounts = getMountedPaths(); - - if (!mounts.contains(getRootPath().toString())) { - platform.runCmd(rootMountCmd(), o -> { - }, "Failed to mount cgroup root"); - Files.createDirectory(getSubsystemRootPath()); - } - - if (!mounts.contains(getSubsystemRootPath().toString())) { - platform.runCmd(subsystemMountCmd(), o -> { - }, "Failed to mount cgroup subsystem"); - } - if (!Files.exists(getSubsystemGGPath())) { - Files.createDirectory(getSubsystemGGPath()); - } - if (!Files.exists(getSubsystemComponentPath(component.getServiceName()))) { - Files.createDirectory(getSubsystemComponentPath(component.getServiceName())); - } + public void initializeCgroup(GreengrassService component, LinuxPlatform platform) + throws IOException { + initializeCgroupCore(component, platform, () -> { + platform.runCmd(subsystemMountCmd(), o -> { + }, "Failed to mount cgroup subsystem"); + } + ); } @Override diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV2.java index d1efd2872b..766d0123b9 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupV2.java @@ -14,7 +14,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Set; @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "CGroupV2 virtual filesystem path cannot be relative") @@ -42,43 +41,22 @@ public Path getCgroupFreezerStateFilePath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); } - @Override public Path getRootSubTreeControlPath() { return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL); } - @Override public Path getGGSubTreeControlPath() { return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL); } - @Override public Path getComponentCpuMaxPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CPU_MAX); } - @Override - public Path getCgroupFreezePath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); - } - @Override public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException { - Set mounts = getMountedPaths(); - - if (!mounts.contains(getRootPath().toString())) { - platform.runCmd(rootMountCmd(), o -> { - }, "Failed to mount cgroup root"); - Files.createDirectory(getSubsystemRootPath()); - } - - if (!Files.exists(getSubsystemGGPath())) { - Files.createDirectory(getSubsystemGGPath()); - } - if (!Files.exists(getSubsystemComponentPath(component.getServiceName()))) { - Files.createDirectory(getSubsystemComponentPath(component.getServiceName())); - } - + initializeCgroupCore(component, platform, () -> { + }); //Enable controllers for root group Files.write(getRootSubTreeControlPath(), CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/InitializeCgroup.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/InitializeCgroup.java new file mode 100644 index 0000000000..34a2ffcc9b --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/InitializeCgroup.java @@ -0,0 +1,13 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import java.io.IOException; + +@FunctionalInterface +public interface InitializeCgroup { + void add() throws IOException; +} diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java index 9788500a32..aa7187eae4 100644 --- a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -100,7 +100,7 @@ void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws Files.write(path, "max 100000".getBytes(StandardCharsets.UTF_8)); CGroupSubSystemPaths cpuSystemV2 = spy(CGroupV2.CPU); - lenient().when(cpuSystemV2.getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); + lenient().when(((CGroupV2) cpuSystemV2).getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); cpuSystemV2.handleCpuLimits(component, 0.5); From ec2348b1057a3dfb6da986aaa31066f5fa8a620e Mon Sep 17 00:00:00 2001 From: "yiwen.chen" Date: Fri, 11 Nov 2022 01:23:54 -0800 Subject: [PATCH 08/11] feat: linux control group version 2 API support cgroup v2 --- .../GenericExternalServiceIntegTest.java | 43 ++++++ .../LinuxSystemResourceControllerV2Test.java | 136 +++++++++++------- 2 files changed, 130 insertions(+), 49 deletions(-) diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index c544ff79bc..382b87c057 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -22,6 +22,7 @@ import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler; import com.aws.greengrass.util.Pair; import com.aws.greengrass.util.platforms.unix.linux.CGroupV1; +import com.aws.greengrass.util.platforms.unix.linux.CGroupV2; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.AfterEach; @@ -580,6 +581,7 @@ void GIVEN_posix_default_user_WHEN_runs_THEN_runs_with_default_user(String file, @EnabledOnOs({OS.LINUX}) @Test void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_resource_limits() throws Exception { + assumeTrue(!ifCgroupV2(), "skip this test case if v2 is enabled."); String componentName = "echo_service"; // Run with no resource limit ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel, @@ -798,6 +800,45 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers assertThat(statusB.get().getStatusReason(), containsString(ComponentStatusCode.RUN_TIMEOUT.getDescription())); } + @Test + void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service_and_freeze_thaw_cgroup_V2( + ExtensionContext context) throws Exception { + assumeTrue(ifCgroupV2(), "skip this test case if v1 is enabled."); + ignoreExceptionOfType(context, FileSystemException.class); + ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel, + getClass().getResource("long_running_services.yaml")); + kernel.launch(); + + CountDownLatch mainRunningLatch = new CountDownLatch(1); + kernel.getContext().addGlobalStateChangeListener((service, oldState, newState) -> { + if (kernel.getMain().equals(service) && newState.isRunning()) { + mainRunningLatch.countDown(); + } + }); + + // wait for main to run + assertTrue(mainRunningLatch.await(60, TimeUnit.SECONDS), "main running"); + + GenericExternalService component = (GenericExternalService) kernel.locate("sleeperA"); + assertThat(component.getState(), is(State.RUNNING)); + + component.pause(); + assertTrue(component.isPaused()); + assertEquals(getCgroupFreezerStateV2(component.getServiceName()), + "1"); + + component.resume(); + assertFalse(component.isPaused()); + assertEquals(getCgroupFreezerStateV2(component.getServiceName()), + "0"); + } + + private String getCgroupFreezerStateV2(String serviceName) + throws IOException { + return new String(Files.readAllBytes(CGroupV2.Freezer.getCgroupFreezerStateFilePath(serviceName)) + , StandardCharsets.UTF_8).trim(); + } + private void assertResourceLimits(String componentName, long memory, double cpus) throws Exception { byte[] buf1 = Files.readAllBytes(CGroupV1.Memory.getComponentMemoryLimitPath(componentName)); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); @@ -827,8 +868,10 @@ private void assertResourceLimitsCgroupV2(long memory, double cpus) throws Excep assertThat(expectedQuota, equalTo(quota)); } + @Test void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service_and_freeze_thaw_cgroup( ExtensionContext context) throws Exception { + assumeTrue(!ifCgroupV2(), "skip this test case if v2 is enabled."); ignoreExceptionOfType(context, FileSystemException.class); ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel, getClass().getResource("long_running_services.yaml")); diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java index aa7187eae4..75c61fbfee 100644 --- a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -7,8 +7,6 @@ import com.aws.greengrass.lifecyclemanager.GreengrassService; import com.aws.greengrass.testcommons.testutilities.GGExtension; -import com.aws.greengrass.util.Utils; -import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; @@ -18,21 +16,21 @@ import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; @ExtendWith({MockitoExtension.class, GGExtension.class}) @@ -42,72 +40,112 @@ class LinuxSystemResourceControllerV2Test { GreengrassService component; @Mock LinuxPlatform platform; - @Spy - CGroupSubSystemPaths cpuCgroup = CGroupV2.CPU; - @Spy - CGroupSubSystemPaths memoryCgroup = CGroupV2.Memory; @InjectMocks @Spy LinuxSystemResourceController linuxSystemResourceControllerV2 = new LinuxSystemResourceController( platform, false); - private static final String FILE_PATH = "/cgroupv2test"; - private static final String CGROUP_MEMORY_LIMIT_FILE_NAME = "memory.txt"; - private static final String CGROUP_CPU_LIMIT_FILE_NAME = "cpu.txt"; private static final long MEMORY_IN_KB = 2048000; - private static final double CPU_TIME = 0.5; - private static final String COMPONENT_NAME = "testComponentName"; + private static final double CPU_TIME = 0.2; @Test - void GIVEN_cgroupv2_WHEN_memory_limit_updated_THEN_memory_limit_file_updated() throws IOException { + void GIVEN_cgroupv2_WHEN_memory_limit_updated_THEN_memory_limit_file_updated_V2() throws IOException { + assumeTrue(ifCgroupV2(), "skip this test case if v1 is enabled."); Map resourceLimit = new HashMap<>(); resourceLimit.put("memory", String.valueOf(MEMORY_IN_KB)); doReturn("testComponentName").when(component).getServiceName(); - Path path = Paths.get(FILE_PATH + "/" + CGROUP_MEMORY_LIMIT_FILE_NAME); - - Path componentNameFolderPath = Paths.get(FILE_PATH); - Utils.createPaths(componentNameFolderPath); - File file = new File(FILE_PATH + "/" + CGROUP_MEMORY_LIMIT_FILE_NAME); - if (!Files.exists(path)) { - file.createNewFile(); - } + linuxSystemResourceControllerV2.updateResourceLimits(component, resourceLimit); + String memoryValue = new String(Files.readAllBytes( + CGroupV2.Memory.getComponentMemoryLimitPath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + assertEquals(String.valueOf(MEMORY_IN_KB * 1024), memoryValue); + } - when(memoryCgroup.getComponentMemoryLimitPath(COMPONENT_NAME)).thenReturn(path); - lenient().when(memoryCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); - when(cpuCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(path); + @Test + void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated_V2() throws IOException { + assumeTrue(ifCgroupV2(), "skip this test case if v1 is enabled."); + Map resourceLimit = new HashMap<>(); + resourceLimit.put("cpus", String.valueOf(CPU_TIME)); + doReturn("testComponentName").when(component).getServiceName(); linuxSystemResourceControllerV2.updateResourceLimits(component, resourceLimit); - - List mounts = Files.readAllLines(path); - assertEquals(String.valueOf(MEMORY_IN_KB * 1024), mounts.get(0)); - FileUtils.deleteDirectory(componentNameFolderPath.toFile()); + String rawCpuValue = new String(Files.readAllBytes( + CGroupV2.CPU.getComponentCpuMaxPath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + assertEquals(String.valueOf((int) (CPU_TIME * 100000)), rawCpuValue.split(" ")[0]); } @Test - void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws IOException { + void GIVEN_cgroupv2_WHEN_pause_resume_THEN_freeze_state_1_0_V2() throws IOException { + assumeTrue(ifCgroupV2(), "skip this test case if v1 is enabled."); + LinuxSystemResourceController controller = new LinuxSystemResourceController(platform, false); doReturn("testComponentName").when(component).getServiceName(); + List processes = new ArrayList<>(); + controller.pauseComponentProcesses(component, processes); + String freezerStateValue = new String(Files.readAllBytes( + CGroupV2.Freezer.getCgroupFreezerStateFilePath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + assertEquals(freezerStateValue, "1"); + + controller.resumeComponentProcesses(component); + freezerStateValue = new String(Files.readAllBytes( + CGroupV2.Freezer.getCgroupFreezerStateFilePath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + assertEquals(freezerStateValue, "0"); + } - Path path = Paths.get(FILE_PATH + "/" + CGROUP_CPU_LIMIT_FILE_NAME); - - Path componentNameFolderPath = Paths.get(FILE_PATH); - Utils.createPaths(componentNameFolderPath); - File file = new File(FILE_PATH + "/" + CGROUP_CPU_LIMIT_FILE_NAME); - if (!Files.exists(path)) { - file.createNewFile(); - } + @Test + void GIVEN_cgroupv1_WHEN_memory_limit_updated_THEN_memory_limit_file_updated_V1() throws IOException { + assumeTrue(!ifCgroupV2(), "skip this test case if v2 is enabled."); + Map resourceLimit = new HashMap<>(); + resourceLimit.put("memory", String.valueOf(MEMORY_IN_KB)); + doReturn("testComponentName").when(component).getServiceName(); - Files.write(path, "max 100000".getBytes(StandardCharsets.UTF_8)); + new LinuxSystemResourceController(platform, true).updateResourceLimits(component, resourceLimit); + String memoryValue = new String(Files.readAllBytes( + CGroupV1.Memory.getComponentMemoryLimitPath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + assertEquals(String.valueOf(MEMORY_IN_KB * 1024), memoryValue); + } - CGroupSubSystemPaths cpuSystemV2 = spy(CGroupV2.CPU); - lenient().when(((CGroupV2) cpuSystemV2).getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); + @Test + void GIVEN_cgroupv1_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated_V1() throws IOException { + assumeTrue(!ifCgroupV2(), "skip this test case if v2 is enabled."); + Map resourceLimit = new HashMap<>(); + resourceLimit.put("cpus", String.valueOf(CPU_TIME)); + doReturn("testComponentName").when(component).getServiceName(); - cpuSystemV2.handleCpuLimits(component, 0.5); + new LinuxSystemResourceController(platform, true).updateResourceLimits(component, resourceLimit); + String rawCpuValue = new String(Files.readAllBytes( + CGroupV1.CPU.getComponentCpuQuotaPath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + assertEquals(String.valueOf((int) (CPU_TIME * 100000)), rawCpuValue); + } - List mounts = Files.readAllLines(path); - assertEquals((int) (CPU_TIME * 100000) + " 100000", mounts.get(0)); + @Test + void GIVEN_cgroupv1_WHEN_pause_resume_THEN_freeze_state_1_0_V1() throws IOException { + assumeTrue(!ifCgroupV2(), "skip this test case if v2 is enabled."); + LinuxSystemResourceController controller = new LinuxSystemResourceController(platform, true); + doReturn("testComponentName").when(component).getServiceName(); + List processes = new ArrayList<>(); + controller.pauseComponentProcesses(component, processes); + String freezerStateValue = new String(Files.readAllBytes( + CGroupV1.Freezer.getCgroupFreezerStateFilePath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + + assertThat(freezerStateValue, + anyOf(is(LinuxSystemResourceController.CgroupFreezerState.FROZEN.toString()), + is(LinuxSystemResourceController.CgroupFreezerState.FREEZING.toString()))); + + controller.resumeComponentProcesses(component); + freezerStateValue = new String(Files.readAllBytes( + CGroupV1.Freezer.getCgroupFreezerStateFilePath("testComponentName")) + , StandardCharsets.UTF_8).trim(); + assertEquals(freezerStateValue, LinuxSystemResourceController.CgroupFreezerState.THAWED.toString()); + } - Files.deleteIfExists(path); - Files.deleteIfExists(componentNameFolderPath); + private boolean ifCgroupV2() { + return Files.exists(Paths.get("/sys/fs/cgroup/cgroup.controllers")); } } \ No newline at end of file From 3db48d743791da9d29d3a34f3913b9044af7ba03 Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Fri, 11 Nov 2022 17:41:52 +0800 Subject: [PATCH 09/11] feat: linux control group version 2 API support cgroup v2 --- .../unix/linux/LinuxSystemResourceControllerV2Test.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java index 75c61fbfee..13546841ec 100644 --- a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -44,7 +44,6 @@ class LinuxSystemResourceControllerV2Test { @Spy LinuxSystemResourceController linuxSystemResourceControllerV2 = new LinuxSystemResourceController( platform, false); - private static final long MEMORY_IN_KB = 2048000; private static final double CPU_TIME = 0.2; From c1626ceef1bb9c46ae4ead320957adcd13bbcc35 Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Fri, 18 Nov 2022 13:52:30 +0800 Subject: [PATCH 10/11] feat: linux control group version 2 API support cgroup v2 --- .../greengrass/integrationtests/ipc/IPCHibernateTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java index c6ea202df4..059fcd812f 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java @@ -30,6 +30,7 @@ import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -42,6 +43,7 @@ import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; @ExtendWith({GGExtension.class}) class IPCHibernateTest { @@ -92,6 +94,7 @@ void beforeEach(ExtensionContext context) throws Exception { @Test void GIVEN_LifeCycleEventStreamClient_WHEN_pause_resume_component_THEN_target_service_paused_and_resumed() throws Exception { + assumeTrue(!ifCgroupV2(), "skip this test case if v2 is enabled."); GenericExternalService component = (GenericExternalService) kernel.locate(TARGET_COMPONENT_NAME); PauseComponentRequest pauseRequest = new PauseComponentRequest(); @@ -118,5 +121,9 @@ private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(S new String(Files.readAllBytes(CGroupV1.Freezer.getCgroupFreezerStateFilePath(serviceName)), StandardCharsets.UTF_8).trim()); } + + private boolean ifCgroupV2() { + return Files.exists(Paths.get("/sys/fs/cgroup/cgroup.controllers")); + } } From 6004a6442a9ea1599eaa495c8b1148309524345a Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Fri, 18 Nov 2022 15:20:22 +0800 Subject: [PATCH 11/11] feat: linux control group version 2 API support cgroup v2 --- .../aws/greengrass/integrationtests/ipc/IPCHibernateTest.java | 3 ++- .../lifecyclemanager/GenericExternalServiceIntegTest.java | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java index 059fcd812f..c47eaf1292 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.jupiter.MockitoExtension; import software.amazon.awssdk.aws.greengrass.GreengrassCoreIPCClient; import software.amazon.awssdk.aws.greengrass.model.PauseComponentRequest; import software.amazon.awssdk.aws.greengrass.model.ResumeComponentRequest; @@ -45,7 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; -@ExtendWith({GGExtension.class}) +@ExtendWith({GGExtension.class, MockitoExtension.class}) class IPCHibernateTest { private static final String TARGET_COMPONENT_NAME = "HibernateTarget"; private static final String CONTROLLER_COMPONENT_NAME = "HibernateController"; diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index 382b87c057..38064e7df4 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -801,6 +801,7 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers } @Test + @EnabledOnOs({OS.LINUX}) void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service_and_freeze_thaw_cgroup_V2( ExtensionContext context) throws Exception { assumeTrue(ifCgroupV2(), "skip this test case if v1 is enabled."); @@ -869,6 +870,7 @@ private void assertResourceLimitsCgroupV2(long memory, double cpus) throws Excep } @Test + @EnabledOnOs({OS.LINUX}) void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service_and_freeze_thaw_cgroup( ExtensionContext context) throws Exception { assumeTrue(!ifCgroupV2(), "skip this test case if v2 is enabled.");