Skip to content

Commit

Permalink
feat: Linux Control Group version 2 API support (cgroup v2) (aws-gree…
Browse files Browse the repository at this point in the history 
…ngrass#1262)
  • Loading branch information
@ChangxinDong
ChangxinDong committed Oct 8, 2022
1 parent f3f4935 commit 5013bf9
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 40 deletions.
37 changes: 23 additions & 14 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,12 @@

package com.aws.greengrass.util.platforms.unix.linux;

import com.aws.greengrass.util.Utils;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

Expand All @@ -20,11 +24,12 @@ public enum CgroupV2 {

private static final String CGROUP_ROOT = "/sys/fs/cgroup";
private static final String GG_NAMESPACE = "greengrass";
private static final String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes";
private static final String CPU_CFS_PERIOD_US = "cpu.cfs_period_us";
private static final String CPU_CFS_QUOTA_US = "cpu.cfs_quota_us";
private static final String CPU_MAX = "cpu.max";
private static final String MEMORY_MAX = "memory.max";
private static final String CGROUP_PROCS = "cgroup.procs";
private static final String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control";
private static final String FREEZER_STATE_FILE = "freezer.state";
private static final String CGROUP_FREEZE = "cgroup.freeze";
private final String osString;

CgroupV2(String str) {
Expand All @@ -43,31 +48,35 @@ public Path getSubsystemRootPath() {
return Paths.get(CGROUP_ROOT);
}

public Path getRootSubTreeControlPath() {
return getSubsystemRootPath().resolve(CGROUP_SUBTREE_CONTROL);
}

public Path getSubsystemGGPath() {
return getSubsystemRootPath().resolve(GG_NAMESPACE);
}

public Path getSubsystemComponentPath(String componentName) {
return getSubsystemGGPath().resolve(componentName);
public Path getGGSubTreeControlPath() {
return getSubsystemGGPath().resolve(CGROUP_SUBTREE_CONTROL);
}

public Path getComponentMemoryLimitPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_MEMORY_LIMITS);
public Path getSubsystemComponentPath(String componentName) {
return getSubsystemGGPath().resolve(componentName);
}

public Path getComponentCpuPeriodPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CPU_CFS_PERIOD_US);
public Path getComponentCpuMaxPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CPU_MAX);
}

public Path getComponentCpuQuotaPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CPU_CFS_QUOTA_US);
public Path getComponentMemoryMaxPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(MEMORY_MAX);
}

public Path getCgroupProcsPath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_PROCS);
}

public Path getCgroupFreezerStateFilePath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE);
public Path getCgroupFreezePath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE);
}
}
26 changes: 26 additions & 0 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupV2FreezerState.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/

package com.aws.greengrass.util.platforms.unix.linux;

public enum CgroupV2FreezerState {
THAWED(0),
FROZEN(1);

private int index;

CgroupV2FreezerState(int index) {
this.index = index;
}

/**
* Get the index value associated with this CgroupV2FreezerState.
*
* @return the integer index value associated with this CgroupV2FreezerState.
*/
public int getIndex() {
return index;
}
}
62 changes: 36 additions & 26 deletions ...in/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.aws.greengrass.util.Coerce;
import com.aws.greengrass.util.Utils;
import com.aws.greengrass.util.platforms.SystemResourceController;
import org.apache.commons.lang3.StringUtils;
import org.zeroturnaround.process.PidUtil;

import java.io.IOException;
Expand Down Expand Up @@ -38,7 +39,9 @@ public class LinuxSystemResourceControllerV2 implements SystemResourceController
private static final String COMPONENT_NAME = "componentName";
private static final String MEMORY_KEY = "memory";
private static final String CPUS_KEY = "cpus";

private static final String UNICODE_SPACE = "\\040";
private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids";
private static final List<CgroupV2> RESOURCE_LIMIT_CGROUPS = Arrays.asList(CgroupV2.Memory, CgroupV2.CPU);

private final CopyOnWriteArrayList<CgroupV2> usedCgroups = new CopyOnWriteArrayList<>();
Expand Down Expand Up @@ -71,9 +74,10 @@ public void updateResourceLimits(GreengrassService component, Map<String, Object

if (resourceLimit.containsKey(MEMORY_KEY)) {
long memoryLimitInKB = Coerce.toLong(resourceLimit.get(MEMORY_KEY));

if (memoryLimitInKB > 0) {
String memoryLimit = Long.toString(memoryLimitInKB * ONE_KB);
Files.write(CgroupV2.Memory.getComponentMemoryLimitPath(component.getServiceName()),
Files.write(CgroupV2.Memory.getComponentMemoryMaxPath(component.getServiceName()),
memoryLimit.getBytes(StandardCharsets.UTF_8));
} else {
logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(MEMORY_KEY, memoryLimitInKB)
Expand All @@ -85,14 +89,26 @@ public void updateResourceLimits(GreengrassService component, Map<String, Object
double cpu = Coerce.toDouble(resourceLimit.get(CPUS_KEY));
if (cpu > 0) {
byte[] content = Files.readAllBytes(
CgroupV2.CPU.getComponentCpuPeriodPath(component.getServiceName()));
int cpuPeriodUs = Integer.parseInt(new String(content, StandardCharsets.UTF_8).trim());

int cpuQuotaUs = (int) (cpuPeriodUs * cpu);
String cpuQuotaUsStr = Integer.toString(cpuQuotaUs);
CgroupV2.CPU.getComponentCpuMaxPath(component.getServiceName()));
String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim();
String[] cpuMaxContentArr = cpuMaxContent.split(" ");
String cpuMaxStr = "max";
String cpuPeriodStr = "100000";

if (cpuMaxContentArr.length >= 2) {
cpuMaxStr = cpuMaxContentArr[0];
cpuPeriodStr = cpuMaxContentArr[1];

if (!StringUtils.isEmpty(cpuPeriodStr)) {
int period = Integer.parseInt(cpuPeriodStr.trim());
int max = (int) (period * cpu);
cpuMaxStr = Integer.toString(max);
}
}

Files.write(CgroupV2.CPU.getComponentCpuQuotaPath(component.getServiceName()),
cpuQuotaUsStr.getBytes(StandardCharsets.UTF_8));
String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr);
Files.write(CgroupV2.CPU.getComponentCpuMaxPath(component.getServiceName()),
latestCpuMaxContent.getBytes(StandardCharsets.UTF_8));
} else {
logger.atWarn().kv(COMPONENT_NAME, component.getServiceName()).kv(CPUS_KEY, cpu)
.log("The provided cpu limit is invalid");
Expand Down Expand Up @@ -151,21 +167,13 @@ public void pauseComponentProcesses(GreengrassService component, List<Process> p
addComponentProcessToCgroup(component.getServiceName(), process, CgroupV2.Freezer);
}

if (LinuxSystemResourceControllerV2.CgroupFreezerState.FROZEN.equals(
currentFreezerCgroupState(component.getServiceName()))) {
return;
}
Files.write(freezerCgroupStateFile(component.getServiceName()),
LinuxSystemResourceControllerV2.CgroupFreezerState.FROZEN.toString().getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
}

@Override
public void resumeComponentProcesses(GreengrassService component) throws IOException {
if (LinuxSystemResourceControllerV2.CgroupFreezerState.THAWED.equals(
currentFreezerCgroupState(component.getServiceName()))) {
return;
}
Files.write(freezerCgroupStateFile(component.getServiceName()),
LinuxSystemResourceControllerV2.CgroupFreezerState.THAWED.toString().getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
Expand Down Expand Up @@ -213,10 +221,10 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) {
// Check the exception message here to avoid the exception stacktrace failing the tests.
if (e.getMessage() != null && e.getMessage().contains("No such process")) {
logger.atWarn().kv(COMPONENT_NAME, component)
.log("Failed to add pid to the cgroup because the process doesn't exist anymore");
.log("Failed to add pid to the cgroupv2 because the process doesn't exist anymore");
} else {
logger.atError().setCause(e).kv(COMPONENT_NAME, component)
.log("Failed to add pid to the cgroup");
.log("Failed to add pid to the cgroupv2");
}
}

Expand Down Expand Up @@ -251,16 +259,18 @@ private void initializeCgroup(GreengrassService component, CgroupV2 cgroup) thro
if (!mounts.contains(CgroupV2.getRootPath().toString())) {
platform.runCmd(CgroupV2.rootMountCmd(), o -> {
}, "Failed to mount cgroup2 root");
Files.createDirectory(cgroup.getSubsystemRootPath());
Utils.createPaths(cgroup.getSubsystemRootPath());
}

if (!Files.exists(cgroup.getSubsystemGGPath())) {
Files.createDirectory(cgroup.getSubsystemGGPath());
}
//Enable controllers for root group
Files.write(cgroup.getRootSubTreeControlPath(),
CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8));

if (!Files.exists(cgroup.getSubsystemComponentPath(component.getServiceName()))) {
Files.createDirectory(cgroup.getSubsystemComponentPath(component.getServiceName()));
}
Utils.createPaths(cgroup.getSubsystemGGPath());
//Enable controllers for gg group
Files.write(cgroup.getGGSubTreeControlPath(),
CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8));
Utils.createPaths(cgroup.getSubsystemComponentPath(component.getServiceName()));

usedCgroups.add(cgroup);
}
Expand All @@ -271,7 +281,7 @@ private Set<Integer> pidsInComponentCgroup(CgroupV2 cgroup, String component) th
}

private Path freezerCgroupStateFile(String component) {
return Cgroup.Freezer.getCgroupFreezerStateFilePath(component);
return CgroupV2.Freezer.getCgroupFreezePath(component);
}

private LinuxSystemResourceControllerV2.CgroupFreezerState currentFreezerCgroupState(String component)
Expand Down

0 comments on commit 5013bf9

Please sign in to comment.