From ea1823aeb6fa581743457fd44a1606e293d26793 Mon Sep 17 00:00:00 2001 From: "changxin.dong" Date: Mon, 10 Oct 2022 17:31:10 +0800 Subject: [PATCH] feat: linux control group version 2 API support cgroup v2 --- .../ipc/IPCHibernateTest.java | 3 +- .../GenericExternalServiceIntegTest.java | 9 +- .../util/platforms/unix/linux/Cgroup.java | 82 +++++++++-- .../platforms/unix/linux/CgroupSubSystem.java | 36 +++++ .../unix/linux/CgroupSubSystemV2.java | 37 +++++ .../unix/linux/CgroupV2FreezerState.java | 26 ++++ .../platforms/unix/linux/LinuxPlatform.java | 23 ++- .../linux/LinuxSystemResourceController.java | 110 +++++++++------ .../LinuxSystemResourceControllerV2.java | 133 ++++++++++++++++++ .../LinuxSystemResourceControllerV2Test.java | 104 ++++++++++++++ 10 files changed, 502 insertions(+), 61 deletions(-) create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java create mode 100644 src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java create mode 100644 src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java index 1a38192c9f..597749aabe 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java @@ -10,6 +10,7 @@ import com.aws.greengrass.testcommons.testutilities.GGExtension; import com.aws.greengrass.testcommons.testutilities.TestUtils; import com.aws.greengrass.util.platforms.unix.linux.Cgroup; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -115,7 +116,7 @@ void GIVEN_LifeCycleEventStreamClient_WHEN_pause_resume_component_THEN_target_se private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState.valueOf( - new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName)), + new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)), StandardCharsets.UTF_8).trim()); } } diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java index df454156c0..607eda9609 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java @@ -21,6 +21,7 @@ import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler; import com.aws.greengrass.util.Pair; import com.aws.greengrass.util.platforms.unix.linux.Cgroup; +import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem; import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.AfterEach; @@ -713,11 +714,11 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers } private void assertResourceLimits(String componentName, long memory, double cpus) throws Exception { - byte[] buf1 = Files.readAllBytes(Cgroup.Memory.getComponentMemoryLimitPath(componentName)); + byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystem.Memory).getComponentMemoryLimitPath(componentName)); assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim()))); - byte[] buf2 = Files.readAllBytes(Cgroup.CPU.getComponentCpuQuotaPath(componentName)); - byte[] buf3 = Files.readAllBytes(Cgroup.CPU.getComponentCpuPeriodPath(componentName)); + byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuQuotaPath(componentName)); + byte[] buf3 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuPeriodPath(componentName)); int quota = Integer.parseInt(new String(buf2, StandardCharsets.UTF_8).trim()); int period = Integer.parseInt(new String(buf3, StandardCharsets.UTF_8).trim()); @@ -761,7 +762,7 @@ void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName) throws IOException { return LinuxSystemResourceController.CgroupFreezerState - .valueOf(new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName)) + .valueOf(new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)) , StandardCharsets.UTF_8).trim()); } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java index ed4b814ed7..558fce8164 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java @@ -6,18 +6,17 @@ package com.aws.greengrass.util.platforms.unix.linux; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.StringUtils; import java.nio.file.Path; import java.nio.file.Paths; /** - * Represents Linux cgroup v1 subsystems. + * Represents Linux cgroup subsystems. */ @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Cgroup virtual filesystem path " + "cannot be relative") -public enum Cgroup { - Memory("memory"), CPU("cpu,cpuacct"), Freezer("freezer", "freezer"); - +public class Cgroup { private static final String CGROUP_ROOT = "/sys/fs/cgroup"; private static final String GG_NAMESPACE = "greengrass"; private static final String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes"; @@ -25,26 +24,44 @@ public enum Cgroup { private static final String CPU_CFS_QUOTA_US = "cpu.cfs_quota_us"; private static final String CGROUP_PROCS = "cgroup.procs"; private static final String FREEZER_STATE_FILE = "freezer.state"; + private static final String CPU_MAX = "cpu.max"; + private static final String MEMORY_MAX = "memory.max"; + private static final String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control"; + private static final String CGROUP_FREEZE = "cgroup.freeze"; private final String osString; private final String mountSrc; - Cgroup(String str) { - osString = str; - mountSrc = "cgroup"; + /** + * Cgroup constructor. + * + * @param subSystem subController + */ + public Cgroup(CgroupSubSystem subSystem) { + this.osString = subSystem.getOsString(); + this.mountSrc = subSystem.getMountSrc(); } - Cgroup(String str, String mountSrc) { - this.osString = str; - this.mountSrc = mountSrc; + Cgroup(CgroupSubSystemV2 subSystemV2) { + this.osString = subSystemV2.getOsString(); + this.mountSrc = subSystemV2.getMountSrc(); } public static Path getRootPath() { return Paths.get(CGROUP_ROOT); } - public static String rootMountCmd() { - return String.format("mount -t tmpfs cgroup %s", CGROUP_ROOT); + /** + * root mount cmd. + * + * @return mount command string + */ + public String rootMountCmd() { + if (StringUtils.isEmpty(osString)) { + return String.format("mount -t cgroup2 none %s", CGROUP_ROOT); + } else { + return String.format("mount -t tmpfs cgroup %s", CGROUP_ROOT); + } } public String subsystemMountCmd() { @@ -63,8 +80,18 @@ public Path getSubsystemComponentPath(String componentName) { return getSubsystemGGPath().resolve(componentName); } + /** + * get component memory limit path. + * + * @param componentName componentName + * @return memory limit Path + */ public Path getComponentMemoryLimitPath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); + if (StringUtils.isEmpty(osString)) { + return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX); + } else { + return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS); + } } public Path getComponentCpuPeriodPath(String componentName) { @@ -79,7 +106,34 @@ public Path getCgroupProcsPath(String componentName) { return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS); } + /** + * get cgroup freezer path. + * + * @param componentName componentName + * @return cgroup freezer path + */ public Path getCgroupFreezerStateFilePath(String componentName) { - return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); + if (StringUtils.isEmpty(osString)) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); + } else { + return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE); + } + } + + public Path getRootSubTreeControlPath() { + return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL); + } + + public Path getGGSubTreeControlPath() { + return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL); } + + public Path getComponentCpuMaxPath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CPU_MAX); + } + + public Path getCgroupFreezePath(String componentName) { + return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE); + } + } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java new file mode 100644 index 0000000000..c40874030e --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +public enum CgroupSubSystem { + Memory("memory", ""), CPU("cpu,cpuacct", ""), Freezer("freezer", "freezer"); + + private String osString; + private String mountSrc; + + CgroupSubSystem(String osString, String mountSrc) { + this.osString = osString; + this.mountSrc = mountSrc; + } + + /** + * Get the osString associated with this CgroupSubController. + * + * @return the osString associated with this CgroupSubController. + */ + public String getOsString() { + return osString; + } + + /** + * Get the mountSrc associated with this CgroupSubController. + * + * @return the mountSrc associated with this CgroupSubController. + */ + public String getMountSrc() { + return mountSrc; + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java new file mode 100644 index 0000000000..1978d052a6 --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +public enum CgroupSubSystemV2 { + Memory("", ""), CPU("", ""), Freezer("", ""), + Unified("",""); + + private String osString; + private String mountSrc; + + CgroupSubSystemV2(String osString, String mountSrc) { + this.osString = osString; + this.mountSrc = mountSrc; + } + + /** + * Get the osString associated with this CgroupSubController. + * + * @return the osString associated with this CgroupSubController. + */ + public String getOsString() { + return osString; + } + + /** + * Get the mountSrc associated with this CgroupSubController. + * + * @return the mountSrc associated with this CgroupSubController. + */ + public String getMountSrc() { + return mountSrc; + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java new file mode 100644 index 0000000000..09ebdb79b1 --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +public enum CgroupV2FreezerState { + THAWED(0), + FROZEN(1); + + private int index; + + CgroupV2FreezerState(int index) { + this.index = index; + } + + /** + * Get the index value associated with this CgroupV2FreezerState. + * + * @return the integer index value associated with this CgroupV2FreezerState. + */ + public int getIndex() { + return index; + } +} diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java index 83dcd8c9af..6bb3944688 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java @@ -7,12 +7,33 @@ import com.aws.greengrass.util.platforms.SystemResourceController; import com.aws.greengrass.util.platforms.unix.UnixPlatform; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Cgroup Controller virtual filesystem path cannot be relative") public class LinuxPlatform extends UnixPlatform { - SystemResourceController systemResourceController = new LinuxSystemResourceController(this); + private static final String CGROUP_ROOT = "/sys/fs/cgroup"; + private static final String CGROUP_CONTROLLERS = "cgroup.controllers"; + + SystemResourceController systemResourceController; @Override public SystemResourceController getSystemResourceController() { + //if the path exists, identify it as cgroupv1, otherwise identify it as cgroupv2 + if (Files.exists(getControllersRootPath())) { + systemResourceController = new LinuxSystemResourceControllerV2(this); + } else { + systemResourceController = new LinuxSystemResourceController(this); + } + return systemResourceController; } + + private Path getControllersRootPath() { + return Paths.get(CGROUP_ROOT).resolve(CGROUP_CONTROLLERS); + } } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java index a2bd992321..ea2d04ed28 100644 --- a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java @@ -34,20 +34,34 @@ public class LinuxSystemResourceController implements SystemResourceController { private static final Logger logger = LogManager.getLogger(LinuxSystemResourceController.class); - private static final String COMPONENT_NAME = "componentName"; - private static final String MEMORY_KEY = "memory"; - private static final String CPUS_KEY = "cpus"; + + protected static final String COMPONENT_NAME = "componentName"; + protected static final String MEMORY_KEY = "memory"; + protected static final String CPUS_KEY = "cpus"; private static final String UNICODE_SPACE = "\\040"; - private static final List RESOURCE_LIMIT_CGROUPS = Arrays.asList(Cgroup.Memory, Cgroup.CPU); + protected Cgroup memoryCgroup; + protected Cgroup cpuCgroup; + protected Cgroup freezerCgroup; + protected Cgroup unifiedCgroup; + protected List resourceLimitCgroups; + protected CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); + + protected LinuxPlatform platform; - private final CopyOnWriteArrayList usedCgroups = new CopyOnWriteArrayList<>(); + public LinuxSystemResourceController() { - protected final LinuxPlatform platform; + } - public LinuxSystemResourceController(LinuxPlatform platform) { + LinuxSystemResourceController(LinuxPlatform platform) { this.platform = platform; + this.memoryCgroup = new Cgroup(CgroupSubSystem.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystem.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystem.Freezer); + resourceLimitCgroups = Arrays.asList( + memoryCgroup, cpuCgroup); } + @Override public void removeResourceController(GreengrassService component) { usedCgroups.forEach(cg -> { @@ -64,36 +78,24 @@ public void removeResourceController(GreengrassService component) { @Override public void updateResourceLimits(GreengrassService component, Map resourceLimit) { try { - if (!Files.exists(Cgroup.Memory.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component, Cgroup.Memory); - } - if (resourceLimit.containsKey(MEMORY_KEY)) { - long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY)); - if (memoryLimitInKB > 0) { - String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); - Files.write(Cgroup.Memory.getComponentMemoryLimitPath(component.getServiceName()), - memoryLimit.getBytes(StandardCharsets.UTF_8)); - } else { - logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) - .log("The provided memory limit is invalid"); - } - } + preUpdateResourceLimits(component, resourceLimit); - if (!Files.exists(Cgroup.CPU.getSubsystemComponentPath(component.getServiceName()))) { - initializeCgroup(component, Cgroup.CPU); + if (!Files.exists(cpuCgroup.getSubsystemComponentPath(component.getServiceName()))) { + initializeCgroup(component, cpuCgroup); } if (resourceLimit.containsKey(CPUS_KEY)) { double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY)); if (cpu > 0) { byte[] content = Files.readAllBytes( - Cgroup.CPU.getComponentCpuPeriodPath(component.getServiceName())); + cpuCgroup.getComponentCpuPeriodPath(component.getServiceName())); int cpuPeriodUs = Integer.parseInt(new String(content, StandardCharsets.UTF_8).trim()); int cpuQuotaUs = (int) (cpuPeriodUs * cpu); String cpuQuotaUsStr = Integer.toString(cpuQuotaUs); - Files.write(Cgroup.CPU.getComponentCpuQuotaPath(component.getServiceName()), + Files.write(cpuCgroup.getComponentCpuQuotaPath(component.getServiceName()), cpuQuotaUsStr.getBytes(StandardCharsets.UTF_8)); + } else { logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) .log("The provided cpu limit is invalid"); @@ -105,9 +107,27 @@ public void updateResourceLimits(GreengrassService component, Map resourceLimit) throws IOException { + if (!Files.exists(memoryCgroup.getSubsystemComponentPath(component.getServiceName()))) { + initializeCgroup(component, memoryCgroup); + } + if (resourceLimit.containsKey(MEMORY_KEY)) { + long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY)); + if (memoryLimitInKB > 0) { + String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB); + Files.write(memoryCgroup.getComponentMemoryLimitPath(component.getServiceName()), + memoryLimit.getBytes(StandardCharsets.UTF_8)); + } else { + logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB) + .log("The provided memory limit is invalid"); + } + } + } + @Override public void resetResourceLimits(GreengrassService component) { - for (Cgroup cg : RESOURCE_LIMIT_CGROUPS) { + for (Cgroup cg : resourceLimitCgroups) { try { if (Files.exists(cg.getSubsystemComponentPath(component.getServiceName()))) { Files.delete(cg.getSubsystemComponentPath(component.getServiceName())); @@ -122,7 +142,7 @@ public void resetResourceLimits(GreengrassService component) { @Override public void addComponentProcess(GreengrassService component, Process process) { - RESOURCE_LIMIT_CGROUPS.forEach(cg -> { + resourceLimitCgroups.forEach(cg -> { try { addComponentProcessToCgroup(component.getServiceName(), process, cg); @@ -139,19 +159,14 @@ public void addComponentProcess(GreengrassService component, Process process) { }, 1, TimeUnit.SECONDS); } catch (IOException e) { - handleErrorAddingPidToCgroup(e, component.getServiceName()); + handleErrorAddingPidToCgroup(e, component.getServiceName()); } }); } @Override public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { - initializeCgroup(component, Cgroup.Freezer); - - for (Process process: processes) { - addComponentProcessToCgroup(component.getServiceName(), process, Cgroup.Freezer); - } - + prePauseComponentProcesses(component, processes); if (CgroupFreezerState.FROZEN.equals(currentFreezerCgroupState(component.getServiceName()))) { return; } @@ -160,6 +175,7 @@ public void pauseComponentProcesses(GreengrassService component, List p StandardOpenOption.TRUNCATE_EXISTING); } + @Override public void resumeComponentProcesses(GreengrassService component) throws IOException { if (CgroupFreezerState.THAWED.equals(currentFreezerCgroupState(component.getServiceName()))) { @@ -170,7 +186,7 @@ public void resumeComponentProcesses(GreengrassService component) throws IOExcep StandardOpenOption.TRUNCATE_EXISTING); } - private void addComponentProcessToCgroup(String component, Process process, Cgroup cg) + protected void addComponentProcessToCgroup(String component, Process process, Cgroup cg) throws IOException { if (!Files.exists(cg.getSubsystemComponentPath(component))) { @@ -219,7 +235,7 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) { } } - private Set getMountedPaths() throws IOException { + protected Set getMountedPaths() throws IOException { Set mountedPaths = new HashSet<>(); Path procMountsPath = Paths.get("/proc/self/mounts"); @@ -244,15 +260,18 @@ private Set getMountedPaths() throws IOException { return mountedPaths; } - private void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { + protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { Set mounts = getMountedPaths(); + if (!mounts.contains(Cgroup.getRootPath().toString())) { - platform.runCmd(Cgroup.rootMountCmd(), o -> {}, "Failed to mount cgroup root"); + platform.runCmd(cgroup.rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); Files.createDirectory(cgroup.getSubsystemRootPath()); } if (!mounts.contains(cgroup.getSubsystemRootPath().toString())) { - platform.runCmd(cgroup.subsystemMountCmd(), o -> {}, "Failed to mount cgroup subsystem"); + platform.runCmd(cgroup.subsystemMountCmd(), o -> { + }, "Failed to mount cgroup subsystem"); } if (!Files.exists(cgroup.getSubsystemGGPath())) { Files.createDirectory(cgroup.getSubsystemGGPath()); @@ -260,6 +279,7 @@ private void initializeCgroup(GreengrassService component, Cgroup cgroup) throws if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); } + usedCgroups.add(cgroup); } @@ -268,8 +288,8 @@ private Set pidsInComponentCgroup(Cgroup cgroup, String component) thro .stream().map(Integer::parseInt).collect(Collectors.toSet()); } - private Path freezerCgroupStateFile(String component) { - return Cgroup.Freezer.getCgroupFreezerStateFilePath(component); + protected Path freezerCgroupStateFile(String component) { + return freezerCgroup.getCgroupFreezerStateFilePath(component); } private CgroupFreezerState currentFreezerCgroupState(String component) throws IOException { @@ -281,6 +301,14 @@ private CgroupFreezerState currentFreezerCgroupState(String component) throws IO return CgroupFreezerState.valueOf(stateFileContent.get(0).trim()); } + protected void prePauseComponentProcesses(GreengrassService component, List processes) throws IOException { + initializeCgroup(component, freezerCgroup); + + for (Process process : processes) { + addComponentProcessToCgroup(component.getServiceName(), process, freezerCgroup); + } + } + public enum CgroupFreezerState { THAWED, FREEZING, FROZEN } diff --git a/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java new file mode 100644 index 0000000000..a9087ae05a --- /dev/null +++ b/src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java @@ -0,0 +1,133 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import com.aws.greengrass.lifecyclemanager.GreengrassService; +import com.aws.greengrass.logging.api.Logger; +import com.aws.greengrass.logging.impl.LogManager; +import com.aws.greengrass.util.Coerce; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * https://www.kernel.org/doc/Documentation/cgroup-v2.txt + */ +public class LinuxSystemResourceControllerV2 extends LinuxSystemResourceController { + private static final Logger logger = LogManager.getLogger(LinuxSystemResourceControllerV2.class); + private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids"; + + /** + * Linux system resource controller V2 constrcutors. + * + * @param platform linux platform + */ + public LinuxSystemResourceControllerV2(LinuxPlatform platform) { + super(); + this.platform = platform; + this.unifiedCgroup = new Cgroup(CgroupSubSystemV2.Unified); + this.memoryCgroup = new Cgroup(CgroupSubSystemV2.Memory); + this.cpuCgroup = new Cgroup(CgroupSubSystemV2.CPU); + this.freezerCgroup = new Cgroup(CgroupSubSystemV2.Freezer); + resourceLimitCgroups = Arrays.asList(unifiedCgroup); + } + + + @Override + public void updateResourceLimits(GreengrassService component, Map resourceLimit) { + try { + super.preUpdateResourceLimits(component, resourceLimit); + + if (resourceLimit.containsKey(CPUS_KEY)) { + double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY)); + if (cpu > 0) { + handleCpuLimits(component, cpu); + } else { + logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu) + .log("The provided cpu limit is invalid"); + } + } + } catch (IOException e) { + logger.atError().setCause(e).kv(COMPONENT_NAME, component.getServiceName()) + .log("Failed to apply resource limits"); + } + } + + @Override + public void pauseComponentProcesses(GreengrassService component, List processes) throws IOException { + prePauseComponentProcesses(component, processes); + + Files.write(freezerCgroupStateFile(component.getServiceName()), + String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } + + @Override + public void resumeComponentProcesses(GreengrassService component) throws IOException { + Files.write(freezerCgroupStateFile(component.getServiceName()), + String.valueOf(CgroupV2FreezerState.THAWED.getIndex()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.TRUNCATE_EXISTING); + } + + @Override + protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException { + Set mounts = getMountedPaths(); + + if (!mounts.contains(Cgroup.getRootPath().toString())) { + platform.runCmd(cgroup.rootMountCmd(), o -> { + }, "Failed to mount cgroup root"); + Files.createDirectory(cgroup.getSubsystemRootPath()); + } + + if (!Files.exists(cgroup.getSubsystemGGPath())) { + Files.createDirectory(cgroup.getSubsystemGGPath()); + } + if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) { + Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName())); + } + + //Enable controllers for root group + Files.write(cgroup.getRootSubTreeControlPath(), + CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); + //Enable controllers for gg group + Files.write(cgroup.getGGSubTreeControlPath(), + CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8)); + + usedCgroups.add(cgroup); + } + + private void handleCpuLimits(GreengrassService component, double cpu) throws IOException { + byte[] content = Files.readAllBytes( + cpuCgroup.getComponentCpuMaxPath(component.getServiceName())); + String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim(); + String[] cpuMaxContentArr = cpuMaxContent.split(" "); + String cpuMaxStr = "max"; + String cpuPeriodStr = "100000"; + + if (cpuMaxContentArr.length >= 2) { + cpuMaxStr = cpuMaxContentArr[0]; + cpuPeriodStr = cpuMaxContentArr[1]; + + if (!StringUtils.isEmpty(cpuPeriodStr)) { + int period = Integer.parseInt(cpuPeriodStr.trim()); + int max = (int) (period * cpu); + cpuMaxStr = Integer.toString(max); + } + } + + String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr); + Files.write(cpuCgroup.getComponentCpuMaxPath(component.getServiceName()), + latestCpuMaxContent.getBytes(StandardCharsets.UTF_8)); + } + +} diff --git a/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java new file mode 100644 index 0000000000..2a8511284c --- /dev/null +++ b/src/test/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2Test.java @@ -0,0 +1,104 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.util.platforms.unix.linux; + +import com.aws.greengrass.lifecyclemanager.GreengrassService; +import com.aws.greengrass.testcommons.testutilities.GGExtension; +import com.aws.greengrass.util.Utils; +import com.aws.greengrass.util.platforms.SystemResourceController; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +@ExtendWith({MockitoExtension.class, GGExtension.class}) +@DisabledOnOs(OS.WINDOWS) +class LinuxSystemResourceControllerV2Test { + private final SystemResourceController systemResourceController = new LinuxSystemResourceControllerV2(new LinuxPlatform()); + @Mock + GreengrassService component; + + private static final String FILE_PATH = "/cgroupv2test"; + private static final String CGROUP_MEMORY_LIMIT_FILE_NAME = "memory.txt"; + private static final String CGROUP_CPU_LIMIT_FILE_NAME = "cpu.txt"; + private static final long MEMORY_IN_KB = 2048000; + private static final double CPU_TIME = 0.5; + private static final String COMPONENT_NAME = "testComponentName"; + + @Test + void GIVEN_cgroupv2_WHEN_memory_limit_updated_THEN_memory_limit_file_updated() throws IOException { + Map resourceLimit = new HashMap<>(); + resourceLimit.put("memory", String.valueOf(MEMORY_IN_KB)); + doReturn("testComponentName").when(component).getServiceName(); + + Path path = Paths.get(FILE_PATH + "/" + CGROUP_MEMORY_LIMIT_FILE_NAME); + + Path componentNameFolderPath = Paths.get(FILE_PATH); + Utils.createPaths(componentNameFolderPath); + File file = new File(FILE_PATH + "/" + CGROUP_MEMORY_LIMIT_FILE_NAME); + if (!Files.exists(path)) { + file.createNewFile(); + } + + Cgroup memoryCgroup = Mockito.mock(Cgroup.class, withSettings().useConstructor(CgroupSubSystemV2.Memory)); + when(memoryCgroup.getComponentMemoryLimitPath(COMPONENT_NAME)).thenReturn(path); + when(memoryCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); + systemResourceController.updateResourceLimits(component, resourceLimit); + + List mounts = Files.readAllLines(path); + assertEquals(String.valueOf(MEMORY_IN_KB * 1024), mounts.get(0)); + + Files.deleteIfExists(path); + Files.deleteIfExists(componentNameFolderPath); + } + + @Test + void GIVEN_cgroupv2_WHEN_cpu_limit_updated_THEN_cpu_limit_file_updated() throws IOException { + Map resourceLimit = new HashMap<>(); + resourceLimit.put("cpus", String.valueOf(CPU_TIME)); + doReturn("testComponentName").when(component).getServiceName(); + + Path path = Paths.get(FILE_PATH + "/" + CGROUP_CPU_LIMIT_FILE_NAME); + + Path componentNameFolderPath = Paths.get(FILE_PATH); + Utils.createPaths(componentNameFolderPath); + File file = new File(FILE_PATH + "/" + CGROUP_CPU_LIMIT_FILE_NAME); + if (!Files.exists(path)) { + file.createNewFile(); + } + + Files.write(path, "max 100000".getBytes(StandardCharsets.UTF_8)); + + Cgroup cpuCgroup = Mockito.mock(Cgroup.class, withSettings().useConstructor(CgroupSubSystemV2.Memory)); + when(cpuCgroup.getComponentCpuMaxPath(COMPONENT_NAME)).thenReturn(path); + when(cpuCgroup.getSubsystemComponentPath(COMPONENT_NAME)).thenReturn(componentNameFolderPath); + systemResourceController.updateResourceLimits(component, resourceLimit); + + List mounts = Files.readAllLines(path); + assertEquals((int) (CPU_TIME * 100000) + " 100000", mounts.get(0)); + + Files.deleteIfExists(path); + Files.deleteIfExists(componentNameFolderPath); + } +} \ No newline at end of file