diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java index 1a38192c9f..597749aabe 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java @@ -10,6 +10,7 @@ import com.aws.greengrass.testcommons.testutilities.GGExtension; import com.aws.greengrass.testcommons.testutilities.TestUtils; import com.aws.greengrass.util.platforms.unix.linux.Cgroup; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -115,7 +116,7 @@ void GIVEN_LifeCycleEventStreamClient_WHEN_pause_resume_component_THEN_target_se private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState.valueOf( - new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName)), + new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)), StandardCharsets.UTF_8).trim()); } } diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index a91155dda5..b785ddbfa3 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -21,6 +21,7 @@ import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler; import com.aws.greengrass.util.Pair; import com.aws.greengrass.util.platforms.unix.linux.Cgroup; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.AfterEach; @@ -751,11 +752,11 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers } private void assertResourceLimits(String componentName, long memory, double cpus) throws Exception { - byte[] buf1 = Files.readAllBytes(Cgroup.Memory.getComponentMemoryLimitPath(componentName)); + byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystem.Memory).getComponentMemoryLimitPath(componentName)); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); - byte[] buf2 = Files.readAllBytes(Cgroup.CPU.getComponentCpuQuotaPath(componentName)); - byte[] buf3 = Files.readAllBytes(Cgroup.CPU.getComponentCpuPeriodPath(componentName)); + byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuQuotaPath(componentName)); + byte[] buf3 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuPeriodPath(componentName)); int quota = Integer.parseInt(new String(buf2, StandardCharsets.UTF_8).trim()); int period = Integer.parseInt(new String(buf3, StandardCharsets.UTF_8).trim()); @@ -799,7 +800,7 @@ void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState - .valueOf(new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName)) + .valueOf(new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)) , StandardCharsets.UTF_8).trim()); } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java index ed4b814ed7..558fce8164 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java @@ -6,18 +6,17 @@ package com.aws.greengrass.util.platforms.unix.linux; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.StringUtils; import java.nio.file.Path; import java.nio.file.Paths; /** - * Represents Linux cgroup v1 subsystems. + * Represents Linux cgroup subsystems. */ @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Cgroup virtual filesystem path " + "cannot be relative") -public enum Cgroup { - Memory("memory"), CPU("cpu,cpuacct"), Freezer("freezer", "freezer"); - +public class Cgroup { private static final String CGROUP_ROOT = "/sys/fs/cgroup"; private static final String GG_NAMESPACE = "greengrass"; private static final String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes"; @@ -25,26 +24,44 @@ public enum Cgroup { private static final String CPU_CFS_QUOTA_US = "cpu.cfs_quota_us"; private static final String CGROUP_PROCS = "cgroup.procs"; private static final String FREEZER_STATE_FILE = "freezer.state"; + private static final String CPU_MAX = "cpu.max"; + private static final String MEMORY_MAX = "memory.max"; + private static final String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control"; + private static final String CGROUP_FREEZE = "cgroup.freeze"; private final String osString; private final String mountSrc; - Cgroup(String str) { - osString = str; - mountSrc = "cgroup"; + /** + * Cgroup constructor. + * + * @param subSystem subController + */ + public Cgroup(CgroupSubSystem subSystem) { + this.osString = subSystem.getOsString(); + this.mountSrc = subSystem.getMountSrc(); } - Cgroup(String str, String mountSrc) { - this.osString = str; - this.mountSrc = mountSrc; + Cgroup(CgroupSubSystemV2 subSystemV2) { + this.osString = subSystemV2.getOsString(); + this.mountSrc = subSystemV2.getMountSrc(); } public static Path getRootPath() { return Paths.get(CGROUP_ROOT); } - public static String rootMountCmd() { - return String.format("mount -t tmpfs cgroup %s", CGROUP_ROOT); + /** + * root mount cmd. + * + * @return mount command string + */ + public String rootMountCmd() { + if (StringUtils.isEmpty(osString)) { + return String.format("mount -t cgroup2 none %s", CGROUP_ROOT); + } else { + return String.format("mount -t tmpfs cgroup %s", CGROUP_ROOT); + } } public String subsystemMountCmd() { @@ -63,8 +80,18 @@ public Path getSubsystemComponentPath(String componentName) { return getSubsystemGGPath().resolve(componentName); } + /** + * get component memory limit path. + * + * @param componentName componentName + * @return memory limit Path + */ public Path getComponentMemoryLimitPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); + if (StringUtils.isEmpty(osString)) { + return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX); + } else { + return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); + } } public Path getComponentCpuPeriodPath(String componentName) { @@ -79,7 +106,34 @@ public Path getCgroupProcsPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); } + /** + * get cgroup freezer path. + * + * @param componentName componentName + * @return cgroup freezer path + */ public Path getCgroupFreezerStateFilePath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); + if (StringUtils.isEmpty(osString)) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); + } else { + return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); + } + } + + public Path getRootSubTreeControlPath() { + return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL); + } + + public Path getGGSubTreeControlPath() { + return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL); } + + public Path getComponentCpuMaxPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CPU_MAX); + } + + public Path getCgroupFreezePath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); + } + } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java new file mode 100644 index 0000000000..c40874030e --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +public enum CgroupSubSystem { + Memory("memory", ""), CPU("cpu,cpuacct", ""), Freezer("freezer", "freezer"); + + private String osString; + private String mountSrc; + + CgroupSubSystem(String osString, String mountSrc) { + this.osString = osString; + this.mountSrc = mountSrc; + } + + /** + * Get the osString associated with this CgroupSubController. + * + * @return the osString associated with this CgroupSubController. + */ + public String getOsString() { + return osString; + } + + /** + * Get the mountSrc associated with this CgroupSubController. + * + * @return the mountSrc associated with this CgroupSubController. + */ + public String getMountSrc() { + return mountSrc; + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java new file mode 100644 index 0000000000..1978d052a6 --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +public enum CgroupSubSystemV2 { + Memory("", ""), CPU("", ""), Freezer("", ""), + Unified("",""); + + private String osString; + private String mountSrc; + + CgroupSubSystemV2(String osString, String mountSrc) { + this.osString = osString; + this.mountSrc = mountSrc; + } + + /** + * Get the osString associated with this CgroupSubController. + * + * @return the osString associated with this CgroupSubController. + */ + public String getOsString() { + return osString; + } + + /** + * Get the mountSrc associated with this CgroupSubController. + * + * @return the mountSrc associated with this CgroupSubController. + */ + public String getMountSrc() { + return mountSrc; + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java deleted file mode 100644 index ff98909911..0000000000 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.aws.greengrass.util.platforms.unix.linux; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * Represents Linux cgroup v2. - */ -@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Cgroup virtual filesystem path " - + "cannot be relative") -public final class CgroupV2 { - - private static final String CGROUP_ROOT = "/sys/fs/cgroup"; - private static final String GG_NAMESPACE = "greengrass"; - private static final String CPU_MAX = "cpu.max"; - private static final String MEMORY_MAX = "memory.max"; - private static final String CGROUP_PROCS = "cgroup.procs"; - private static final String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control"; - private static final String CGROUP_FREEZE = "cgroup.freeze"; - - private CgroupV2() { - } - - public static Path getRootPath() { - return Paths.get(CGROUP_ROOT); - } - - public static String rootMountCmd() { - return String.format("mount -t cgroup2 none %s", CGROUP_ROOT); - } - - public static Path getSubsystemRootPath() { - return Paths.get(CGROUP_ROOT); - } - - public static Path getRootSubTreeControlPath() { - return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL); - } - - public static Path getSubsystemGGPath() { - return getSubsystemRootPath().resolve(GG_NAMESPACE); - } - - public static Path getGGSubTreeControlPath() { - return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL); - } - - public static Path getSubsystemComponentPath(String componentName) { - return getSubsystemGGPath().resolve(componentName); - } - - public static Path getComponentCpuMaxPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CPU_MAX); - } - - public static Path getComponentMemoryMaxPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX); - } - - public static Path getCgroupProcsPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); - } - - public static Path getCgroupFreezePath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); - } -} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java index a2bd992321..ea2d04ed28 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java @@ -34,20 +34,34 @@ public class LinuxSystemResourceController implements SystemResourceController { private static final Logger logger = LogManager.getLogger(LinuxSystemResourceController.class); - private static final String COMPONENT_NAME = "componentName"; - private static final String MEMORY_KEY = "memory"; - private static final String CPUS_KEY = "cpus"; + + protected static final String COMPONENT_NAME = "componentName"; + protected static final String MEMORY_KEY = "memory"; + protected static final String CPUS_KEY = "cpus"; private static final String UNICODE_SPACE = "\\040"; - private static final List RESOURCE_LIMIT_CGROUPS = Arrays.asList(Cgroup.Memory, Cgroup.CPU); + protected Cgroup memoryCgroup; + protected Cgroup cpuCgroup; + protected Cgroup freezerCgroup; + protected Cgroup unifiedCgroup; + protected List resourceLimitCgroups; + protected CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); + + protected LinuxPlatform platform; - private final CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); + public LinuxSystemResourceController() { - protected final LinuxPlatform platform; + } - public LinuxSystemResourceController(LinuxPlatform platform) { + LinuxSystemResourceController(LinuxPlatform platform) { this.platform = platform; + this.memoryCgroup = new Cgroup(CgroupSubSystem.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystem.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystem.Freezer); + resourceLimitCgroups = Arrays.asList( + memoryCgroup, cpuCgroup); } + @Override public void removeResourceController(GreengrassService component) { usedCgroups.forEach(cg -> { @@ -64,36 +78,24 @@ public void removeResourceController(GreengrassService component) { @Override public void updateResourceLimits(GreengrassService component, Map resourceLimit) { try { - if (!Files.exists(Cgroup.Memory.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component, Cgroup.Memory); - } - if (resourceLimit.containsKey(MEMORY_KEY)) { - long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY)); - if (memoryLimitInKB > 0) { - String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); - Files.write(Cgroup.Memory.getComponentMemoryLimitPath(component.getServiceName()), - memoryLimit.getBytes(StandardCharsets.UTF_8)); - } else { - logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) - .log("The provided memory limit is invalid"); - } - } + preUpdateResourceLimits(component, resourceLimit); - if (!Files.exists(Cgroup.CPU.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component, Cgroup.CPU); + if (!Files.exists(cpuCgroup.getSubsystemComponentPath(component.getServiceName()))) { + initializeCgroup(component, cpuCgroup); } if (resourceLimit.containsKey(CPUS_KEY)) { double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY)); if (cpu > 0) { byte[] content = Files.readAllBytes( - Cgroup.CPU.getComponentCpuPeriodPath(component.getServiceName())); + cpuCgroup.getComponentCpuPeriodPath(component.getServiceName())); int cpuPeriodUs = Integer.parseInt(new String(content, StandardCharsets.UTF_8).trim()); int cpuQuotaUs = (int) (cpuPeriodUs * cpu); String cpuQuotaUsStr = Integer.toString(cpuQuotaUs); - Files.write(Cgroup.CPU.getComponentCpuQuotaPath(component.getServiceName()), + Files.write(cpuCgroup.getComponentCpuQuotaPath(component.getServiceName()), cpuQuotaUsStr.getBytes(StandardCharsets.UTF_8)); + } else { logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) .log("The provided cpu limit is invalid"); @@ -105,9 +107,27 @@ public void updateResourceLimits(GreengrassService component, Map resourceLimit) throws IOException { + if (!Files.exists(memoryCgroup.getSubsystemComponentPath(component.getServiceName()))) { + initializeCgroup(component, memoryCgroup); + } + if (resourceLimit.containsKey(MEMORY_KEY)) { + long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY)); + if (memoryLimitInKB > 0) { + String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); + Files.write(memoryCgroup.getComponentMemoryLimitPath(component.getServiceName()), + memoryLimit.getBytes(StandardCharsets.UTF_8)); + } else { + logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) + .log("The provided memory limit is invalid"); + } + } + } + @Override public void resetResourceLimits(GreengrassService component) { - for (Cgroup cg : RESOURCE_LIMIT_CGROUPS) { + for (Cgroup cg : resourceLimitCgroups) { try { if (Files.exists(cg.getSubsystemComponentPath(component.getServiceName()))) { Files.delete(cg.getSubsystemComponentPath(component.getServiceName())); @@ -122,7 +142,7 @@ public void resetResourceLimits(GreengrassService component) { @Override public void addComponentProcess(GreengrassService component, Process process) { - RESOURCE_LIMIT_CGROUPS.forEach(cg -> { + resourceLimitCgroups.forEach(cg -> { try { addComponentProcessToCgroup(component.getServiceName(), process, cg); @@ -139,19 +159,14 @@ public void addComponentProcess(GreengrassService component, Process process) { }, 1, TimeUnit.SECONDS); } catch (IOException e) { - handleErrorAddingPidToCgroup(e, component.getServiceName()); + handleErrorAddingPidToCgroup(e, component.getServiceName()); } }); } @Override public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { - initializeCgroup(component, Cgroup.Freezer); - - for (Process process: processes) { - addComponentProcessToCgroup(component.getServiceName(), process, Cgroup.Freezer); - } - + prePauseComponentProcesses(component, processes); if (CgroupFreezerState.FROZEN.equals(currentFreezerCgroupState(component.getServiceName()))) { return; } @@ -160,6 +175,7 @@ public void pauseComponentProcesses(GreengrassService component, List p StandardOpenOption.TRUNCATE_EXISTING); } + @Override public void resumeComponentProcesses(GreengrassService component) throws IOException { if (CgroupFreezerState.THAWED.equals(currentFreezerCgroupState(component.getServiceName()))) { @@ -170,7 +186,7 @@ public void resumeComponentProcesses(GreengrassService component) throws IOExcep StandardOpenOption.TRUNCATE_EXISTING); } - private void addComponentProcessToCgroup(String component, Process process, Cgroup cg) + protected void addComponentProcessToCgroup(String component, Process process, Cgroup cg) throws IOException { if (!Files.exists(cg.getSubsystemComponentPath(component))) { @@ -219,7 +235,7 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) { } } - private Set getMountedPaths() throws IOException { + protected Set getMountedPaths() throws IOException { Set mountedPaths = new HashSet<>(); Path procMountsPath = Paths.get("/proc/self/mounts"); @@ -244,15 +260,18 @@ private Set getMountedPaths() throws IOException { return mountedPaths; } - private void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { + protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { Set mounts = getMountedPaths(); + if (!mounts.contains(Cgroup.getRootPath().toString())) { - platform.runCmd(Cgroup.rootMountCmd(), o -> {}, "Failed to mount cgroup root"); + platform.runCmd(cgroup.rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); Files.createDirectory(cgroup.getSubsystemRootPath()); } if (!mounts.contains(cgroup.getSubsystemRootPath().toString())) { - platform.runCmd(cgroup.subsystemMountCmd(), o -> {}, "Failed to mount cgroup subsystem"); + platform.runCmd(cgroup.subsystemMountCmd(), o -> { + }, "Failed to mount cgroup subsystem"); } if (!Files.exists(cgroup.getSubsystemGGPath())) { Files.createDirectory(cgroup.getSubsystemGGPath()); @@ -260,6 +279,7 @@ private void initializeCgroup(GreengrassService component, Cgroup cgroup) throws if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); } + usedCgroups.add(cgroup); } @@ -268,8 +288,8 @@ private Set pidsInComponentCgroup(Cgroup cgroup, String component) thro .stream().map(Integer::parseInt).collect(Collectors.toSet()); } - private Path freezerCgroupStateFile(String component) { - return Cgroup.Freezer.getCgroupFreezerStateFilePath(component); + protected Path freezerCgroupStateFile(String component) { + return freezerCgroup.getCgroupFreezerStateFilePath(component); } private CgroupFreezerState currentFreezerCgroupState(String component) throws IOException { @@ -281,6 +301,14 @@ private CgroupFreezerState currentFreezerCgroupState(String component) throws IO return CgroupFreezerState.valueOf(stateFileContent.get(0).trim()); } + protected void prePauseComponentProcesses(GreengrassService component, List processes) throws IOException { + initializeCgroup(component, freezerCgroup); + + for (Process process : processes) { + addComponentProcessToCgroup(component.getServiceName(), process, freezerCgroup); + } + } + public enum CgroupFreezerState { THAWED, FREEZING, FROZEN } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java index 74bdce25fd..a9087ae05a 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java @@ -9,99 +9,49 @@ import com.aws.greengrass.logging.api.Logger; import com.aws.greengrass.logging.impl.LogManager; import com.aws.greengrass.util.Coerce; -import com.aws.greengrass.util.Utils; -import com.aws.greengrass.util.platforms.SystemResourceController; import org.apache.commons.lang3.StringUtils; -import org.zeroturnaround.process.PidUtil; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.HashSet; +import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static org.apache.commons.io.FileUtils.ONE_KB; - -public class LinuxSystemResourceControllerV2 implements SystemResourceController { +/** + * https://www.kernel.org/doc/Documentation/cgroup-v2.txt + */ +public class LinuxSystemResourceControllerV2 extends LinuxSystemResourceController { private static final Logger logger = LogManager.getLogger(LinuxSystemResourceControllerV2.class); - private static final String COMPONENT_NAME = "componentName"; - private static final String MEMORY_KEY = "memory"; - private static final String CPUS_KEY = "cpus"; - - private static final String UNICODE_SPACE = "\\040"; private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids"; - protected final LinuxPlatform platform; - + /** + * Linux system resource controller V2 constrcutors. + * + * @param platform linux platform + */ public LinuxSystemResourceControllerV2(LinuxPlatform platform) { + super(); this.platform = platform; + this.unifiedCgroup = new Cgroup(CgroupSubSystemV2.Unified); + this.memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystemV2.Freezer); + resourceLimitCgroups = Arrays.asList(unifiedCgroup); } - @Override - public void removeResourceController(GreengrassService component) { - try { - // Assumes processes belonging to cgroups would already be terminated/killed. - Files.deleteIfExists(CgroupV2.getSubsystemComponentPath(component.getServiceName())); - } catch (IOException e) { - logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName()) - .log("Failed to remove the resource controller"); - } - } @Override public void updateResourceLimits(GreengrassService component, Map resourceLimit) { try { - if (!Files.exists(CgroupV2.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component); - } - - if (resourceLimit.containsKey(MEMORY_KEY)) { - long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY)); - - if (memoryLimitInKB > 0) { - String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); - Files.write(CgroupV2.getComponentMemoryMaxPath(component.getServiceName()), - memoryLimit.getBytes(StandardCharsets.UTF_8)); - } else { - logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) - .log("The provided memory limit is invalid"); - } - } + super.preUpdateResourceLimits(component, resourceLimit); if (resourceLimit.containsKey(CPUS_KEY)) { double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY)); if (cpu > 0) { - byte[] content = Files.readAllBytes( - CgroupV2.getComponentCpuMaxPath(component.getServiceName())); - String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim(); - String[] cpuMaxContentArr = cpuMaxContent.split(" "); - String cpuMaxStr = "max"; - String cpuPeriodStr = "100000"; - - if (cpuMaxContentArr.length >= 2) { - cpuMaxStr = cpuMaxContentArr[0]; - cpuPeriodStr = cpuMaxContentArr[1]; - - if (!StringUtils.isEmpty(cpuPeriodStr)) { - int period = Integer.parseInt(cpuPeriodStr.trim()); - int max = (int) (period * cpu); - cpuMaxStr = Integer.toString(max); - } - } - - String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr); - Files.write(CgroupV2.getComponentCpuMaxPath(component.getServiceName()), - latestCpuMaxContent.getBytes(StandardCharsets.UTF_8)); + handleCpuLimits(component, cpu); } else { logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) .log("The provided cpu limit is invalid"); @@ -113,48 +63,9 @@ public void updateResourceLimits(GreengrassService component, Map { - try { - addComponentProcessToCgroup(component.getServiceName(), process); - } catch (IOException e) { - handleErrorAddingPidToCgroup(e, component.getServiceName()); - } - }, 1, TimeUnit.SECONDS); - - } catch (IOException e) { - handleErrorAddingPidToCgroup(e, component.getServiceName()); - } - } - @Override public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { - initializeCgroup(component); - - for (Process process : processes) { - addComponentProcessToCgroup(component.getServiceName(), process); - } + prePauseComponentProcesses(component, processes); Files.write(freezerCgroupStateFile(component.getServiceName()), String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8), @@ -168,106 +79,55 @@ public void resumeComponentProcesses(GreengrassService component) throws IOExcep StandardOpenOption.TRUNCATE_EXISTING); } - private void addComponentProcessToCgroup(String component, Process process) - throws IOException { - - if (!Files.exists(CgroupV2.getSubsystemComponentPath(component))) { - logger.atDebug().kv(COMPONENT_NAME, component) - .log("Resource controller is not enabled"); - return; - } - - if (process != null) { - try { - Set childProcesses = platform.getChildPids(process); - childProcesses.add(PidUtil.getPid(process)); - Set pidsInCgroup = pidsInComponentCgroup(component); - if (!Utils.isEmpty(childProcesses) && Objects.nonNull(pidsInCgroup) - && !childProcesses.equals(pidsInCgroup)) { - - // Writing pid to cgroup.procs file should auto add the pid to tasks file - // Once a process is added to a cgroup, its forked child processes inherit its (parent's) settings - for (Integer pid : childProcesses) { - if (pid == null) { - logger.atError().log("The process doesn't exist and is skipped"); - continue; - } - - Files.write(CgroupV2.getCgroupProcsPath(component), - Integer.toString(pid).getBytes(StandardCharsets.UTF_8)); - } - } - } catch (InterruptedException e) { - logger.atWarn().setCause(e) - .log("Interrupted while getting processes to add to system limit controller"); - Thread.currentThread().interrupt(); - } - } - } + @Override + protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { + Set mounts = getMountedPaths(); - private void handleErrorAddingPidToCgroup(IOException e, String component) { - // The process might have exited (if it's a short running process). - // Check the exception message here to avoid the exception stacktrace failing the tests. - if (e.getMessage() != null && e.getMessage().contains("No such process")) { - logger.atWarn().kv(COMPONENT_NAME, component) - .log("Failed to add pid to the cgroupv2 because the process doesn't exist anymore"); - } else { - logger.atError().setCause(e).kv(COMPONENT_NAME, component) - .log("Failed to add pid to the cgroupv2"); + if (!mounts.contains(Cgroup.getRootPath().toString())) { + platform.runCmd(cgroup.rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); + Files.createDirectory(cgroup.getSubsystemRootPath()); } - } - - private Set getMountedPaths() throws IOException { - Set mountedPaths = new HashSet<>(); - Path procMountsPath = Paths.get("/proc/self/mounts"); - List mounts = Files.readAllLines(procMountsPath); - for (String mount : mounts) { - String[] split = mount.split(" "); - // As reported in fstab(5) manpage, struct is: - // 1st field is volume name - // 2nd field is path with spaces escaped as \040 - // 3rd field is fs type - // 4th field is mount options - // 5th field is used by dump(8) (ignored) - // 6th field is fsck order (ignored) - if (split.length < 6) { - continue; - } - - // We only need the path of the mounts to verify whether cgroup is mounted - String path = split[1].replace(UNICODE_SPACE, " "); - mountedPaths.add(path); + if (!Files.exists(cgroup.getSubsystemGGPath())) { + Files.createDirectory(cgroup.getSubsystemGGPath()); } - return mountedPaths; - } - - private void initializeCgroup(GreengrassService component) throws IOException { - Set mounts = getMountedPaths(); - - if (!mounts.contains(CgroupV2.getRootPath().toString())) { - platform.runCmd(CgroupV2.rootMountCmd(), o -> { - }, "Failed to mount cgroup2 root"); - Utils.createPaths(CgroupV2.getSubsystemRootPath()); + if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { + Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); } //Enable controllers for root group - Files.write(CgroupV2.getRootSubTreeControlPath(), + Files.write(cgroup.getRootSubTreeControlPath(), CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); - - Utils.createPaths(CgroupV2.getSubsystemGGPath()); //Enable controllers for gg group - Files.write(CgroupV2.getGGSubTreeControlPath(), + Files.write(cgroup.getGGSubTreeControlPath(), CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); - Utils.createPaths(CgroupV2.getSubsystemComponentPath(component.getServiceName())); - } - private Set pidsInComponentCgroup(String component) throws IOException { - return Files.readAllLines(CgroupV2.getCgroupProcsPath(component)) - .stream().map(Integer::parseInt).collect(Collectors.toSet()); + usedCgroups.add(cgroup); } - private Path freezerCgroupStateFile(String component) { - return CgroupV2.getCgroupFreezePath(component); + private void handleCpuLimits(GreengrassService component, double cpu) throws IOException { + byte[] content = Files.readAllBytes( + cpuCgroup.getComponentCpuMaxPath(component.getServiceName())); + String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim(); + String[] cpuMaxContentArr = cpuMaxContent.split(" "); + String cpuMaxStr = "max"; + String cpuPeriodStr = "100000"; + + if (cpuMaxContentArr.length >= 2) { + cpuMaxStr = cpuMaxContentArr[0]; + cpuPeriodStr = cpuMaxContentArr[1]; + + if (!StringUtils.isEmpty(cpuPeriodStr)) { + int period = Integer.parseInt(cpuPeriodStr.trim()); + int max = (int) (period * cpu); + cpuMaxStr = Integer.toString(max); + } + } + + String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr); + Files.write(cpuCgroup.getComponentCpuMaxPath(component.getServiceName()), + latestCpuMaxContent.getBytes(StandardCharsets.UTF_8)); } + } diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java index 920616c1b2..2a8511284c 100644 --- a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -14,7 +14,7 @@ import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import java.io.File; @@ -29,7 +29,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; @ExtendWith({MockitoExtension.class, GGExtension.class}) @DisabledOnOs(OS.WINDOWS) @@ -60,13 +61,10 @@ void GIVEN_cgroupv2_WHEN_memory_limit_updated_THEN_memory_limit_file_updated() t file.createNewFile(); } - try (MockedStatic utilities = mockStatic(CgroupV2.class)) { - utilities.when(() -> CgroupV2.getComponentMemoryMaxPath(COMPONENT_NAME)) - .thenReturn(path); - utilities.when(() -> CgroupV2.getSubsystemComponentPath(COMPONENT_NAME)) - .thenReturn(componentNameFolderPath); - systemResourceController.updateResourceLimits(component, resourceLimit); - } + Cgroup memoryCgroup = Mockito.mock(Cgroup.class, withSettings().useConstructor(CgroupSubSystemV2.Memory)); + when(memoryCgroup.getComponentMemoryLimitPath(COMPONENT_NAME)).thenReturn(path); + when(memoryCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); + systemResourceController.updateResourceLimits(component, resourceLimit); List mounts = Files.readAllLines(path); assertEquals(String.valueOf(MEMORY_IN_KB * 1024), mounts.get(0)); @@ -92,13 +90,10 @@ void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws Files.write(path, "max 100000".getBytes(StandardCharsets.UTF_8)); - try (MockedStatic utilities = mockStatic(CgroupV2.class)) { - utilities.when(() -> CgroupV2.getComponentCpuMaxPath(COMPONENT_NAME)) - .thenReturn(path); - utilities.when(() -> CgroupV2.getSubsystemComponentPath(COMPONENT_NAME)) - .thenReturn(componentNameFolderPath); - systemResourceController.updateResourceLimits(component, resourceLimit); - } + Cgroup cpuCgroup = Mockito.mock(Cgroup.class, withSettings().useConstructor(CgroupSubSystemV2.Memory)); + when(cpuCgroup.getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); + when(cpuCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); + systemResourceController.updateResourceLimits(component, resourceLimit); List mounts = Files.readAllLines(path); assertEquals((int) (CPU_TIME * 100000) + " 100000", mounts.get(0));