feat: pipe system logs to nucleus log path

aws-greengrass · Sep 24, 2024 · da6fefc · da6fefc
1 parent 991fa35
commit da6fefc
Show file tree

Hide file tree

Showing 16 changed files with 125 additions and 15 deletions.
diff --git a/scripts/greengrass.service.template b/scripts/greengrass.service.template
@@ -8,7 +8,7 @@ PIDFile=REPLACE_WITH_GG_LOADER_PID_FILE
 RemainAfterExit=no
 Restart=on-failure
 RestartSec=10
-ExecStart=/bin/sh REPLACE_WITH_GG_LOADER_FILE
+ExecStart=/bin/sh -c "REPLACE_WITH_GG_LOADER_FILE >> REPLACE_WITH_NUCLEUS_LOG_FILE 2>&1"
 KillMode=mixed
 
 [Install]

diff --git a/src/main/java/com/aws/greengrass/deployment/DeviceConfiguration.java b/src/main/java/com/aws/greengrass/deployment/DeviceConfiguration.java
@@ -129,7 +129,6 @@ public class DeviceConfiguration {
     public static final String FALLBACK_VERSION = "0.0.0";
     private final Configuration config;
     private final KernelCommandLine kernelCommandLine;
-
     private final Validator deTildeValidator;
     private final Validator regionValidator;
     private final AtomicBoolean rootCA3Downloaded = new AtomicBoolean(false);

diff --git a/src/main/java/com/aws/greengrass/deployment/KernelUpdateDeploymentTask.java b/src/main/java/com/aws/greengrass/deployment/KernelUpdateDeploymentTask.java
@@ -19,10 +19,14 @@
 import com.aws.greengrass.lifecyclemanager.Kernel;
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
 import com.aws.greengrass.logging.api.Logger;
+import com.aws.greengrass.util.NucleusLogsSummarizer;
 import com.aws.greengrass.util.Pair;
 import com.aws.greengrass.util.Utils;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CancellationException;
@@ -43,6 +47,7 @@ public class KernelUpdateDeploymentTask implements DeploymentTask {
     private final Deployment deployment;
     private final ComponentManager componentManager;
     private final CompletableFuture<DeploymentResult> deploymentResultCompletableFuture;
+    private final Path nucleusLogsPath;
 
     /**
      * Constructor for DefaultDeploymentTask.
@@ -59,6 +64,7 @@ public KernelUpdateDeploymentTask(Kernel kernel, Logger logger, Deployment deplo
         this.logger = logger.dfltKv(DEPLOYMENT_ID_LOG_KEY, deployment.getGreengrassDeploymentId());
         this.componentManager = componentManager;
         this.deploymentResultCompletableFuture = new CompletableFuture<>();
+        this.nucleusLogsPath = kernel.getNucleusPaths().nucleusLogsPath();
     }
 
     @SuppressWarnings({"PMD.AvoidDuplicateLiterals"})
@@ -135,6 +141,7 @@ private void waitForServicesToStart() {
                         getDeploymentStatusDetails());
             }
         }
+
         deploymentResultCompletableFuture.complete(result);
     }
 
@@ -149,10 +156,20 @@ private void saveDeploymentStatusDetails(Throwable failureCause) throws IOExcept
 
     private DeploymentException getDeploymentStatusDetails() {
         if (Utils.isEmpty(deployment.getStageDetails())) {
+            String nucleusLogs;
+            try {
+                nucleusLogs = new String(Files.readAllBytes(this.nucleusLogsPath), StandardCharsets.UTF_8);
+            } catch (IOException e) {
+                logger.atWarn().log("Unable to read Nucleus logs for restart failure", e);
+                nucleusLogs = "Unable to read Nucleus logs for restart failure, "
+                        + "please look at the device logs for more info.";
+            }
             return new DeploymentException(
-                    "Nucleus update workflow failed to restart Nucleus. See loader logs for more details",
+                    String.format("Nucleus update workflow failed to restart Nucleus.%n%s",
+                            NucleusLogsSummarizer.summarizeLogs(nucleusLogs)),
                     DeploymentErrorCode.NUCLEUS_RESTART_FAILURE);
         }
+
         List<DeploymentErrorCode> errorStack = deployment.getErrorStack() == null ? Collections.emptyList()
                 : deployment.getErrorStack().stream().map(DeploymentErrorCode::valueOf).collect(Collectors.toList());
 

diff --git a/src/main/java/com/aws/greengrass/easysetup/DeviceProvisioningHelper.java b/src/main/java/com/aws/greengrass/easysetup/DeviceProvisioningHelper.java
@@ -301,9 +301,9 @@ public void updateKernelConfigWithIotConfiguration(Kernel kernel, ThingInfo thin
         Path certFilePath = certPath.resolve("thingCert.crt");
         Files.write(certFilePath, thing.certificatePem.getBytes(StandardCharsets.UTF_8));
 
-        new DeviceConfiguration(kernel.getConfig(), kernel.getKernelCommandLine(), thing.thingName, thing.dataEndpoint,
-                thing.credEndpoint, privKeyFilePath.toString(), certFilePath.toString(), caFilePath.toString(),
-                awsRegion, roleAliasName);
+        new DeviceConfiguration(kernel.getConfig(), kernel.getKernelCommandLine(),
+                thing.thingName, thing.dataEndpoint, thing.credEndpoint, privKeyFilePath.toString(),
+                certFilePath.toString(), caFilePath.toString(), awsRegion, roleAliasName);
         // Make sure tlog persists the device configuration
         kernel.getContext().waitForPublishQueueToClear();
         outStream.println("Created device configuration");

diff --git a/src/main/java/com/aws/greengrass/easysetup/GreengrassSetup.java b/src/main/java/com/aws/greengrass/easysetup/GreengrassSetup.java
@@ -334,7 +334,8 @@ void performSetup() throws IOException, DeviceConfigurationException, URISyntaxE
         if (setupSystemService) {
             kernel.getContext().get(KernelLifecycle.class).softShutdown(30);
             boolean ok = kernel.getContext().get(SystemServiceUtilsFactory.class).getInstance()
-                    .setupSystemService(kernel.getContext().get(KernelAlternatives.class), kernelStart);
+                    .setupSystemService(kernel.getContext().get(KernelAlternatives.class), kernel.getNucleusPaths(),
+                            kernelStart);
             if (ok) {
                 outStream.println("Successfully set up Nucleus as a system service");
                 // Nucleus will be launched by OS as a service

diff --git a/src/main/java/com/aws/greengrass/lifecyclemanager/KernelAlternatives.java b/src/main/java/com/aws/greengrass/lifecyclemanager/KernelAlternatives.java
@@ -30,6 +30,7 @@
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
@@ -394,6 +395,14 @@ public void prepareBootstrap(String deploymentId) throws IOException {
         setupLinkToDirectory(getCurrentDir(), newLaunchDir);
         Files.delete(getNewDir());
         logger.atInfo().log("Finished setup of launch directory for new Nucleus");
+
+        cleanupNucleusLogs();
+    }
+
+    protected void cleanupNucleusLogs() throws IOException {
+        logger.atDebug().kv("logs-path", getNucleusLogPath().toAbsolutePath()).log("Cleaning up nucleus logs");
+        Files.newBufferedWriter(getNucleusLogPath().toAbsolutePath(), StandardOpenOption.TRUNCATE_EXISTING).close();
+        logger.atDebug().log("Finished cleaning up nucleus logs");
     }
 
     /**
@@ -529,4 +538,8 @@ private void cleanupLaunchDirectorySingleLevel(File filePath) throws IOException
         }
         Files.deleteIfExists(filePath.toPath());
     }
+
+    public Path getNucleusLogPath() {
+        return nucleusPaths.nucleusLogsPath().toAbsolutePath();
+    }
 }
diff --git a/src/main/java/com/aws/greengrass/util/NucleusLogsSummarizer.java b/src/main/java/com/aws/greengrass/util/NucleusLogsSummarizer.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.aws.greengrass.util;
+
+import java.util.Scanner;
+
+public final class NucleusLogsSummarizer {
+    public static final String STARTING_SUBSEQUENCE = "+ j=3";
+    public static final String ENDING_SUBSEQUENCE_REGEX = "Nucleus exited ([0-9])*. Retrying 3 times";
+
+    private NucleusLogsSummarizer() {
+    }
+
+    /**
+     * Summarizes loader logs that can be published as part of the deployment status FSS message when deployment fails
+     * with NRF.
+     *
+     * @param blob  string blob containing loader logs
+     * @return      string containing summarized logs
+     */
+    public static String summarizeLogs(String blob) {
+        try (Scanner scanner = new Scanner(blob)) {
+            StringBuilder parsedLogsStringBuilder = new StringBuilder();
+
+            // Skip until the last restart failure
+            while (scanner.hasNextLine()) {
+                String line = scanner.nextLine();
+                // process the line
+                if (line.equals(STARTING_SUBSEQUENCE)) {
+                    break;
+                }
+            }
+
+            while (scanner.hasNextLine()) {
+                String line = scanner.nextLine();
+
+                if (line.matches(ENDING_SUBSEQUENCE_REGEX)) {
+                    parsedLogsStringBuilder.append(line);
+                    break;
+                }
+
+                if (line.startsWith("+")) {
+                    continue;
+                }
+
+                parsedLogsStringBuilder.append(line).append(System.lineSeparator());
+            }
+
+            scanner.close();
+            return parsedLogsStringBuilder.toString();
+        }
+    }
+}
diff --git a/src/main/java/com/aws/greengrass/util/NucleusPaths.java b/src/main/java/com/aws/greengrass/util/NucleusPaths.java
@@ -6,16 +6,19 @@
 package com.aws.greengrass.util;
 
 import com.aws.greengrass.componentmanager.models.ComponentIdentifier;
+import com.aws.greengrass.logging.impl.LogManager;
 
 import java.io.IOException;
 import java.nio.file.Path;
 
 import static com.aws.greengrass.componentmanager.ComponentStore.ARTIFACTS_DECOMPRESSED_DIRECTORY;
 import static com.aws.greengrass.componentmanager.ComponentStore.ARTIFACT_DIRECTORY;
 import static com.aws.greengrass.componentmanager.ComponentStore.RECIPE_DIRECTORY;
+import static com.aws.greengrass.deployment.DeviceConfiguration.DEFAULT_NUCLEUS_COMPONENT_NAME;
 
 @SuppressWarnings("checkstyle:MissingJavadocMethod")
 public class NucleusPaths {
+    public static final String NUCLEUS_LOG_FILE_NAME = DEFAULT_NUCLEUS_COMPONENT_NAME + ".log";
     private Path rootPath;
     private Path workPath;
     private Path componentStorePath;
@@ -191,4 +194,8 @@ public static void setLoggerPath(Path p) throws IOException {
         Utils.createPaths(p);
         Permissions.setLoggerPermission(p);
     }
+
+    public Path nucleusLogsPath() {
+        return LogManager.getRootLogConfiguration().getStoreDirectory().resolve(NUCLEUS_LOG_FILE_NAME).toAbsolutePath();
+    }
 }
diff --git a/src/main/java/com/aws/greengrass/util/orchestration/InitUtils.java b/src/main/java/com/aws/greengrass/util/orchestration/InitUtils.java
@@ -8,12 +8,13 @@
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
 import com.aws.greengrass.logging.api.Logger;
 import com.aws.greengrass.logging.impl.LogManager;
+import com.aws.greengrass.util.NucleusPaths;
 
 public class InitUtils implements SystemServiceUtils {
     protected static final Logger logger = LogManager.getLogger(InitUtils.class);
 
     @Override
-    public boolean setupSystemService(KernelAlternatives kernelAlternatives, boolean start) {
+    public boolean setupSystemService(KernelAlternatives kernelAlternatives, NucleusPaths nucleusPaths, boolean start) {
         logger.atError().log("System service registration is not implemented for this device");
         return false;
     }

diff --git a/src/main/java/com/aws/greengrass/util/orchestration/ProcdUtils.java b/src/main/java/com/aws/greengrass/util/orchestration/ProcdUtils.java
@@ -8,6 +8,7 @@
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
 import com.aws.greengrass.logging.api.Logger;
 import com.aws.greengrass.logging.impl.LogManager;
+import com.aws.greengrass.util.NucleusPaths;
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
@@ -30,7 +31,7 @@ public class ProcdUtils implements SystemServiceUtils {
     private static final String PROCD_SERVICE_TEMPLATE = "greengrass.service.procd.template";
 
     @Override
-    public boolean setupSystemService(KernelAlternatives kernelAlternatives, boolean start) {
+    public boolean setupSystemService(KernelAlternatives kernelAlternatives, NucleusPaths nucleusPaths, boolean start) {
         logger.atInfo(LOG_EVENT_NAME).log("Start procd setup");
         try {
             kernelAlternatives.setupInitLaunchDirIfAbsent();

diff --git a/src/main/java/com/aws/greengrass/util/orchestration/SystemServiceUtils.java b/src/main/java/com/aws/greengrass/util/orchestration/SystemServiceUtils.java
@@ -8,6 +8,7 @@
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
 import com.aws.greengrass.logging.api.Logger;
 import com.aws.greengrass.util.Exec;
+import com.aws.greengrass.util.NucleusPaths;
 import com.aws.greengrass.util.platforms.Platform;
 
 import java.io.IOException;
@@ -17,10 +18,11 @@ public interface SystemServiceUtils {
      * Setup Greengrass as a system service.
      *
      * @param kernelAlternatives KernelAlternatives instance which manages launch directory
+     * @param nucleusPaths NucleusPaths instance which manages Nucleus root paths
      * @param start Whether or not to start the service right away
      * @return true if setup is successful, false otherwise
      */
-    boolean setupSystemService(KernelAlternatives kernelAlternatives, boolean start);
+    boolean setupSystemService(KernelAlternatives kernelAlternatives, NucleusPaths nucleusPaths, boolean start);
 
     /**
      * Simply run a command with privileges.

diff --git a/src/main/java/com/aws/greengrass/util/orchestration/SystemdUtils.java b/src/main/java/com/aws/greengrass/util/orchestration/SystemdUtils.java
@@ -8,6 +8,7 @@
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
 import com.aws.greengrass.logging.api.Logger;
 import com.aws.greengrass.logging.impl.LogManager;
+import com.aws.greengrass.util.NucleusPaths;
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
@@ -22,13 +23,14 @@ public class SystemdUtils implements SystemServiceUtils {
     protected static final Logger logger = LogManager.getLogger(SystemdUtils.class);
     private static final String PID_FILE_PARAM = "REPLACE_WITH_GG_LOADER_PID_FILE";
     private static final String LOADER_FILE_PARAM = "REPLACE_WITH_GG_LOADER_FILE";
+    private static final String NUCLEUS_LOG_FILE_PARAM = "REPLACE_WITH_NUCLEUS_LOG_FILE";
     private static final String SERVICE_CONFIG_FILE_PATH = "/etc/systemd/system/greengrass.service";
     private static final String LOG_EVENT_NAME = "systemd-setup";
     private static final String SYSTEMD_SERVICE_FILE = "greengrass.service";
     private static final String SYSTEMD_SERVICE_TEMPLATE = "greengrass.service.template";
 
     @Override
-    public boolean setupSystemService(KernelAlternatives kernelAlternatives, boolean start) {
+    public boolean setupSystemService(KernelAlternatives kernelAlternatives, NucleusPaths nucleusPaths, boolean start) {
         logger.atDebug(LOG_EVENT_NAME).log("Start systemd setup");
         try {
             kernelAlternatives.setupInitLaunchDirIfAbsent();
@@ -72,7 +74,8 @@ private void interpolateServiceTemplate(Path src, Path dst, KernelAlternatives k
             String line = r.readLine();
             while (line != null) {
                 w.write(line.replace(PID_FILE_PARAM, kernelAlternatives.getLoaderPidPath().toString())
-                        .replace(LOADER_FILE_PARAM, kernelAlternatives.getLoaderPath().toString()));
+                        .replace(LOADER_FILE_PARAM, kernelAlternatives.getLoaderPath().toString())
+                        .replace(NUCLEUS_LOG_FILE_PARAM, kernelAlternatives.getNucleusLogPath().toString()));
                 w.newLine();
                 line = r.readLine();
             }

diff --git a/src/main/java/com/aws/greengrass/util/orchestration/WinswUtils.java b/src/main/java/com/aws/greengrass/util/orchestration/WinswUtils.java
@@ -35,7 +35,7 @@ public WinswUtils(NucleusPaths nucleusPaths) {
     }
 
     @Override
-    public boolean setupSystemService(KernelAlternatives kernelAlternatives, boolean start) {
+    public boolean setupSystemService(KernelAlternatives kernelAlternatives, NucleusPaths nucleusPaths, boolean start) {
         logger.atDebug(LOG_EVENT_NAME).log("Start Windows service setup");
         try {
             kernelAlternatives.setupInitLaunchDirIfAbsent();

diff --git a/src/test/java/com/aws/greengrass/deployment/KernelUpdateDeploymentTaskTest.java b/src/test/java/com/aws/greengrass/deployment/KernelUpdateDeploymentTaskTest.java
@@ -19,6 +19,7 @@
 import com.aws.greengrass.logging.api.Logger;
 import com.aws.greengrass.logging.impl.LogManager;
 import com.aws.greengrass.testcommons.testutilities.GGExtension;
+import com.aws.greengrass.util.NucleusPaths;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -74,6 +75,8 @@ class KernelUpdateDeploymentTaskTest {
     GreengrassService mainService;
     @Mock
     ComponentManager componentManager;
+    @Mock
+    NucleusPaths nucleusPaths;
     ExecutorService executorService;
 
     KernelUpdateDeploymentTask task;
@@ -83,6 +86,7 @@ void beforeEach() throws Exception {
         lenient().doReturn(kernelAlternatives).when(context).get(KernelAlternatives.class);
         lenient().doReturn(deploymentDirectoryManager).when(context).get(DeploymentDirectoryManager.class);
         lenient().doReturn(context).when(kernel).getContext();
+        lenient().doReturn(nucleusPaths).when(kernel).getNucleusPaths();
         lenient().doReturn("A").when(greengrassService).getName();
         lenient().doReturn(mainService).when(kernel).getMain();
         lenient().doReturn(true).when(greengrassService).shouldAutoStart();

diff --git a/src/test/java/com/aws/greengrass/lifecyclemanager/KernelAlternativesTest.java b/src/test/java/com/aws/greengrass/lifecyclemanager/KernelAlternativesTest.java
@@ -35,14 +35,18 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.internal.verification.VerificationModeFactory.times;
 
 @ExtendWith({GGExtension.class, MockitoExtension.class})
 class KernelAlternativesTest {
     @TempDir
     Path altsDir;
+    @Mock
+    NucleusPaths nucleusPaths;
 
     private KernelAlternatives kernelAlternatives;
     @Mock
@@ -54,7 +58,7 @@ class KernelAlternativesTest {
     void beforeEach() throws IOException {
         NucleusPaths paths = new NucleusPaths();
         paths.setKernelAltsPath(altsDir);
-        kernelAlternatives = new KernelAlternatives(paths);
+        kernelAlternatives = spy(new KernelAlternatives(paths));
     }
 
     @Test
@@ -106,6 +110,7 @@ void GIVEN_broken_dir_with_pending_rollback_bootstrap_WHEN_determine_deployment_
     void GIVEN_kernel_update_WHEN_success_THEN_launch_dir_update_correctly() throws Exception {
         Path initPath = createRandomDirectory();
         kernelAlternatives.setupLinkToDirectory(kernelAlternatives.getCurrentDir(), initPath);
+        doNothing().when(kernelAlternatives).cleanupNucleusLogs();
 
         String mockDeploymentId = "mockDeployment";
         kernelAlternatives.prepareBootstrap(mockDeploymentId);
@@ -126,6 +131,7 @@ void GIVEN_kernel_update_with_same_deployment_id_WHEN_success_THEN_launch_dir_up
         Path launchPath = altsDir.resolve(mockDeploymentId);
         Files.createDirectories(launchPath);
         kernelAlternatives.setupLinkToDirectory(kernelAlternatives.getCurrentDir(), launchPath);
+        doNothing().when(kernelAlternatives).cleanupNucleusLogs();
 
         kernelAlternatives.prepareBootstrap(mockDeploymentId);
         assertEquals(launchPath, Files.readSymbolicLink(kernelAlternatives.getCurrentDir()));
@@ -180,6 +186,7 @@ void GIVEN_initDirPointingWrongLocation_WHEN_redirectInitDir_THEN_dirIsRedirecte
     void GIVEN_kernel_update_WHEN_failure_THEN_launch_dir_rollback_correctly() throws Exception {
         Path initPath = createRandomDirectory();
         kernelAlternatives.setupLinkToDirectory(kernelAlternatives.getCurrentDir(), initPath);
+        doNothing().when(kernelAlternatives).cleanupNucleusLogs();
 
         String mockDeploymentId = "mockDeployment";
         Path expectedNewLaunchPath = altsDir.resolve(mockDeploymentId);

diff --git a/src/test/java/com/aws/greengrass/lifecyclemanager/LogManagerHelperTest.java b/src/test/java/com/aws/greengrass/lifecyclemanager/LogManagerHelperTest.java
@@ -87,7 +87,6 @@ class LogManagerHelperTest {
     private Configuration configuration;
     @Mock
     private KernelCommandLine kernelCommandLine;
-
     @Captor
     ArgumentCaptor<ChildChanged> childChangedArgumentCaptor;