Skip to content

Commit

Permalink
feat: linux control group version 2 API support cgroup v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChangxinDong
ChangxinDong committed Nov 3, 2022
1 parent 8132b86 commit 1b95b7d
Show file tree
Hide file tree
Showing 9 changed files with 281 additions and 262 deletions.
7 changes: 3 additions & 4 deletions ...com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2;
import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceControllerV2;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
Expand Down Expand Up @@ -667,8 +666,8 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re
Field controllerField = LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS");
setFinalStatic(controllerField, Paths.get(componentPathString + "/memory.max"));
systemResourceController = linuxPlatform.getSystemResourceController();
LinuxSystemResourceControllerV2 controllerV2 = (LinuxSystemResourceControllerV2) systemResourceController;
Field memoryCgroupField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("memoryCgroup");
LinuxSystemResourceController controllerV2 = (LinuxSystemResourceController) systemResourceController;
Field memoryCgroupField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("memoryCgroup");
memoryCgroupField.setAccessible(true);
Cgroup memoryCgroup = (Cgroup) memoryCgroupField.get(controllerV2);
Field subsystem = memoryCgroup.getClass().getDeclaredField("subSystem");
Expand All @@ -677,7 +676,7 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re
Field f = cg.getClass().getInterfaces()[0].getDeclaredField("CGROUP_ROOT");
setFinalStatic(f, Paths.get(ROOT_PATH_STRING));

Field mountsField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("MOUNT_PATH");
Field mountsField = LinuxSystemResourceController.class.getSuperclass().getDeclaredField("MOUNT_PATH");
mountsField.setAccessible(true);
String mountPathFile = rootGGPathString + "/mountPath.txt";
final Path mountPathFilePath = Paths.get(mountPathFile);
Expand Down
47 changes: 47 additions & 0 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,16 @@

package com.aws.greengrass.util.platforms.unix.linux;

import com.aws.greengrass.lifecyclemanager.GreengrassService;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CGroupSubSystemPath virtual filesystem path cannot be relative")
Expand All @@ -24,6 +30,8 @@ public interface CGroupSubSystemPath {
String MEMORY_MAX = "memory.max";
String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control";
String CGROUP_FREEZE = "cgroup.freeze";
String MOUNT_PATH = "/proc/self/mounts";
String UNICODE_SPACE = "\\040";

default Path getRootPath() {
return CGROUP_ROOT;
Expand Down Expand Up @@ -70,4 +78,43 @@ default Path getComponentCpuMaxPath(String componentName) {
default Path getCgroupFreezePath(String componentName) {
return null;
}

void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException;

void handleCpuLimits(GreengrassService component, double cpu) throws IOException;

void pauseComponentProcessesCore(GreengrassService component, List<Process> processes) throws IOException;

void resumeComponentProcesses(GreengrassService component) throws IOException;

/**
* Get mounted paths.
*
* @return A set of String
* @throws IOException IOException
*/
default Set<String> getMountedPaths() throws IOException {
Set<String> mountedPaths = new HashSet<>();

Path procMountsPath = Paths.get(MOUNT_PATH);
List<String> mounts = Files.readAllLines(procMountsPath);
for (String mount : mounts) {
String[] split = mount.split(" ");
// As reported in fstab(5) manpage, struct is:
// 1st field is volume name
// 2nd field is path with spaces escaped as \040
// 3rd field is fs type
// 4th field is mount options
// 5th field is used by dump(8) (ignored)
// 6th field is fsck order (ignored)
if (split.length < 6) {
continue;
}

// We only need the path of the mounts to verify whether cgroup is mounted
String path = split[1].replace(UNICODE_SPACE, " ");
mountedPaths.add(path);
}
return mountedPaths;
}
}
24 changes: 24 additions & 0 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/Cgroup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@

package com.aws.greengrass.util.platforms.unix.linux;

import com.aws.greengrass.lifecyclemanager.GreengrassService;

import java.io.IOException;
import java.nio.file.Path;
import java.util.List;

/**
* Represents Linux cgroup subsystems.
Expand Down Expand Up @@ -94,4 +98,24 @@ public Path getCgroupFreezePath(String componentName) {
return subSystem.getCgroupFreezePath(componentName);
}

public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException {
subSystem.initializeCgroup(component, platform);
}

public void handleCpuLimits(GreengrassService component, double cpu) throws IOException {
subSystem.handleCpuLimits(component, cpu);
}

public void pauseComponentProcessesCore(GreengrassService component, List<Process> processes)
throws IOException {
subSystem.pauseComponentProcessesCore(component, processes);
}

public void resumeComponentProcesses(GreengrassService component) throws IOException {
subSystem.resumeComponentProcesses(component);
}

protected Path freezerCgroupStateFile(String component) {
return getCgroupFreezerStateFilePath(component);
}
}
77 changes: 77 additions & 0 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,17 @@

package com.aws.greengrass.util.platforms.unix.linux;

import com.aws.greengrass.lifecyclemanager.GreengrassService;
import com.aws.greengrass.util.Utils;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.Set;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CgroupSubSystem virtual filesystem path cannot be relative")
Expand Down Expand Up @@ -89,4 +97,73 @@ public Path getCgroupProcsPath(String componentName) {
public Path getCgroupFreezerStateFilePath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(FREEZER_STATE_FILE);
}

@Override
public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException {
Set<String> mounts = getMountedPaths();

if (!mounts.contains(getRootPath().toString())) {
platform.runCmd(rootMountCmd(), o -> {
}, "Failed to mount cgroup root");
Files.createDirectory(getSubsystemRootPath());
}

if (!mounts.contains(getSubsystemRootPath().toString())) {
platform.runCmd(subsystemMountCmd(), o -> {
}, "Failed to mount cgroup subsystem");
}
if (!Files.exists(getSubsystemGGPath())) {
Files.createDirectory(getSubsystemGGPath());
}
if (!Files.exists(getSubsystemComponentPath(component.getServiceName()))) {
Files.createDirectory(getSubsystemComponentPath(component.getServiceName()));
}
}

@Override
public void handleCpuLimits(GreengrassService component, double cpu) throws IOException {
byte[] content = Files.readAllBytes(
getComponentCpuPeriodPath(component.getServiceName()));
int cpuPeriodUs = Integer.parseInt(new String(content, StandardCharsets.UTF_8).trim());

int cpuQuotaUs = (int) (cpuPeriodUs * cpu);
String cpuQuotaUsStr = Integer.toString(cpuQuotaUs);

Files.write(getComponentCpuQuotaPath(component.getServiceName()),
cpuQuotaUsStr.getBytes(StandardCharsets.UTF_8));
}

@Override
public void pauseComponentProcessesCore(GreengrassService component, List<Process> processes)
throws IOException {
if (LinuxSystemResourceController.CgroupFreezerState.FROZEN.equals(
currentFreezerCgroupState(component.getServiceName()))) {
return;
}
Files.write(getCgroupFreezerStateFilePath(component.getServiceName()),
LinuxSystemResourceController.CgroupFreezerState.FROZEN.toString().getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
}

@Override
public void resumeComponentProcesses(GreengrassService component) throws IOException {
if (LinuxSystemResourceController.CgroupFreezerState.THAWED.equals(
currentFreezerCgroupState(component.getServiceName()))) {
return;
}

Files.write(getCgroupFreezerStateFilePath(component.getServiceName()),
LinuxSystemResourceController.CgroupFreezerState.THAWED.toString().getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
}

private LinuxSystemResourceController.CgroupFreezerState currentFreezerCgroupState(String component)
throws IOException {
List<String> stateFileContent =
Files.readAllLines(getCgroupFreezerStateFilePath(component));
if (Utils.isEmpty(stateFileContent) || stateFileContent.size() != 1) {
throw new IOException("Unexpected error reading freezer cgroup state");
}
return LinuxSystemResourceController.CgroupFreezerState.valueOf(stateFileContent.get(0).trim());
}
}
74 changes: 74 additions & 0 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,23 @@

package com.aws.greengrass.util.platforms.unix.linux;

import com.aws.greengrass.lifecyclemanager.GreengrassService;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.StringUtils;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.Set;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CgroupSubSystemV2 virtual filesystem path cannot be relative")
public enum CgroupSubSystemV2 implements CGroupSubSystemPath {
Memory, CPU, Freezer, Unified;
private static final String CGROUP_SUBTREE_CONTROL_CONTENT = "+cpuset +cpu +io +memory +pids";

@Override
public String rootMountCmd() {
Expand Down Expand Up @@ -68,4 +77,69 @@ public Path getComponentCpuMaxPath(String componentName) {
public Path getCgroupFreezePath(String componentName) {
return getSubsystemComponentPath(componentName).resolve(CGROUP_FREEZE);
}

@Override
public void initializeCgroup(GreengrassService component, LinuxPlatform platform) throws IOException {
Set<String> mounts = getMountedPaths();

if (!mounts.contains(getRootPath().toString())) {
platform.runCmd(rootMountCmd(), o -> {
}, "Failed to mount cgroup root");
Files.createDirectory(getSubsystemRootPath());
}

if (!Files.exists(getSubsystemGGPath())) {
Files.createDirectory(getSubsystemGGPath());
}
if (!Files.exists(getSubsystemComponentPath(component.getServiceName()))) {
Files.createDirectory(getSubsystemComponentPath(component.getServiceName()));
}

//Enable controllers for root group
Files.write(getRootSubTreeControlPath(),
CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8));
//Enable controllers for gg group
Files.write(getGGSubTreeControlPath(),
CGROUP_SUBTREE_CONTROL_CONTENT.getBytes(StandardCharsets.UTF_8));
}

@Override
public void handleCpuLimits(GreengrassService component, double cpu) throws IOException {
byte[] content = Files.readAllBytes(
getComponentCpuMaxPath(component.getServiceName()));
String cpuMaxContent = new String(content, StandardCharsets.UTF_8).trim();
String[] cpuMaxContentArr = cpuMaxContent.split(" ");
String cpuMaxStr = "max";
String cpuPeriodStr = "100000";

if (cpuMaxContentArr.length >= 2) {
cpuMaxStr = cpuMaxContentArr[0];
cpuPeriodStr = cpuMaxContentArr[1];

if (!StringUtils.isEmpty(cpuPeriodStr)) {
int period = Integer.parseInt(cpuPeriodStr.trim());
int max = (int) (period * cpu);
cpuMaxStr = Integer.toString(max);
}
}

String latestCpuMaxContent = String.format("%s %s", cpuMaxStr, cpuPeriodStr);
Files.write(getComponentCpuMaxPath(component.getServiceName()),
latestCpuMaxContent.getBytes(StandardCharsets.UTF_8));
}

@Override
public void pauseComponentProcessesCore(GreengrassService component, List<Process> processes)
throws IOException {
Files.write(getCgroupFreezerStateFilePath(component.getServiceName()),
String.valueOf(CgroupV2FreezerState.FROZEN.getIndex()).getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
}

@Override
public void resumeComponentProcesses(GreengrassService component) throws IOException {
Files.write(getCgroupFreezerStateFilePath(component.getServiceName()),
String.valueOf(CgroupV2FreezerState.THAWED.getIndex()).getBytes(StandardCharsets.UTF_8),
StandardOpenOption.TRUNCATE_EXISTING);
}
}
7 changes: 2 additions & 5 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,15 @@
justification = "Cgroup Controller virtual filesystem path cannot be relative")
public class LinuxPlatform extends UnixPlatform {
private static final Path CGROUP_CONTROLLERS = Paths.get("/sys/fs/cgroup/cgroup.controllers");
SystemResourceController systemResourceController;

@Override
public SystemResourceController getSystemResourceController() {
//if the path exists, identify it as cgroupv2, otherwise identify it as cgroupv1
if (Files.exists(CGROUP_CONTROLLERS)) {
systemResourceController = new LinuxSystemResourceControllerV2(this);
return new LinuxSystemResourceController(this, false);
} else {
systemResourceController = new LinuxSystemResourceController(this);
return new LinuxSystemResourceController(this, true);
}

return systemResourceController;
}

}
Loading

0 comments on commit 1b95b7d

Please sign in to comment.