diff --git a/src/integrationtests/java/com/aws/greengrass/integrationtests/metrics/MetricsEmitterTest.java b/src/integrationtests/java/com/aws/greengrass/integrationtests/metrics/MetricsEmitterTest.java index 89cde5227..50be25570 100644 --- a/src/integrationtests/java/com/aws/greengrass/integrationtests/metrics/MetricsEmitterTest.java +++ b/src/integrationtests/java/com/aws/greengrass/integrationtests/metrics/MetricsEmitterTest.java @@ -40,9 +40,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.exceptions.verification.TooManyActualInvocations; -import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.stubbing.Answer; import software.amazon.awssdk.aws.greengrass.AuthorizeClientDeviceActionResponseHandler; import software.amazon.awssdk.aws.greengrass.GetClientDeviceAuthTokenResponseHandler; import software.amazon.awssdk.aws.greengrass.GreengrassCoreIPCClient; @@ -76,6 +74,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import static com.aws.greengrass.clientdevices.auth.helpers.TestHelpers.eventuallyTrue; import static com.aws.greengrass.clientdevices.auth.metrics.ClientDeviceAuthMetrics .METRIC_GET_CLIENT_DEVICE_AUTH_TOKEN_SUCCESS; import static com.aws.greengrass.clientdevices.auth.metrics.ClientDeviceAuthMetrics @@ -90,47 +89,37 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith({MockitoExtension.class, GGExtension.class}) public class MetricsEmitterTest { private static final long TEST_TIME_OUT_SEC = 10L; - private Clock clock; - private Kernel kernel; - private DomainEvents domainEvents; - private ClientDeviceAuthMetrics metricSpy; - private SessionManager sessionManager; + Clock clock; + Kernel kernel; + DomainEvents domainEvents; + CollectOnceClientDeviceAuthMetrics clientDeviceAuthMetrics; + SessionManager sessionManager; @Mock - private GreengrassServiceClientFactory clientFactory; + GreengrassServiceClientFactory clientFactory; @Mock - private GreengrassV2DataClient client; + GreengrassV2DataClient client; @Mock - private DeviceAuthClient deviceAuthClient; + DeviceAuthClient deviceAuthClient; @Mock - private SessionConfig sessionConfig; + SessionConfig sessionConfig; @Mock - private CreateIoTThingSession createIoTThingSession; + CreateIoTThingSession createIoTThingSession; @TempDir Path rootDir; - //Result captor class created to save and verify the results of scheduled emitting calls - private static class ResultCaptor implements Answer { - @Getter - private T result = null; - - @Override - public T answer(InvocationOnMock invocationOnMock) throws Throwable { - result = (T) invocationOnMock.callRealMethod(); - return result; - } - } - @BeforeEach void beforeEach(ExtensionContext context) throws DeviceConfigurationException { ignoreExceptionOfType(context, TooManyActualInvocations.class); @@ -140,13 +129,13 @@ void beforeEach(ExtensionContext context) throws DeviceConfigurationException { //Set this property for kernel to scan its own classpath to find plugins System.setProperty("aws.greengrass.scanSelfClasspath", "true"); clock = Clock.systemUTC(); - metricSpy = spy(new ClientDeviceAuthMetrics(clock)); + clientDeviceAuthMetrics = spy(new CollectOnceClientDeviceAuthMetrics(clock)); domainEvents = new DomainEvents(); sessionManager = new SessionManager(domainEvents); kernel = new Kernel(); kernel.getContext().put(DomainEvents.class, domainEvents); kernel.getContext().put(Clock.class, clock); - kernel.getContext().put(ClientDeviceAuthMetrics.class, metricSpy); + kernel.getContext().put(ClientDeviceAuthMetrics.class, clientDeviceAuthMetrics); kernel.getContext().put(CreateIoTThingSession.class, createIoTThingSession); kernel.getContext().put(GreengrassServiceClientFactory.class, clientFactory); kernel.getContext().put(SessionManager.class, sessionManager); @@ -239,7 +228,7 @@ void GIVEN_kernelRunningWithMetricsConfig_WHEN_subscribeToCertificateUpdate_THEN Metric expectedMetric = buildMetric(METRIC_SUBSCRIBE_TO_CERTIFICATE_UPDATES_SUCCESS); - //Subscribe to certificate updates + // subscribe to certificate updates try (EventStreamRPCConnection connection = IPCTestUtils.getEventStreamRpcConnection(kernel, "BrokerSubscribingToCertUpdates")) { GreengrassCoreIPCClient ipcClient = new GreengrassCoreIPCClient(connection); @@ -248,21 +237,7 @@ void GIVEN_kernelRunningWithMetricsConfig_WHEN_subscribeToCertificateUpdate_THEN subscribeToCertUpdates(ipcClient, request, cb.getRight()); } - //Reset the metric spy and capture the emitted metrics when the emitter runs - Mockito.reset(metricSpy); - ResultCaptor> resultMetrics = new ResultCaptor<>(); - doAnswer(resultMetrics).when(metricSpy).collectMetrics(); - - //Wait for the emitter to run - Mockito.verify(metricSpy, Mockito.timeout(2000L).atLeastOnce()).emitMetrics(); - - List collectedMetrics = resultMetrics.getResult(); - assertEquals(1, collectedMetrics.size()); - assertEquals(expectedMetric.getName(), collectedMetrics.get(0).getName()); - assertEquals(expectedMetric.getUnit(), collectedMetrics.get(0).getUnit()); - assertEquals(expectedMetric.getValue(), collectedMetrics.get(0).getValue()); - assertEquals(expectedMetric.getNamespace(), collectedMetrics.get(0).getNamespace()); - assertEquals(expectedMetric.getAggregation(), collectedMetrics.get(0).getAggregation()); + assertMetricEmitted(expectedMetric); } @Test @@ -284,11 +259,6 @@ void GIVEN_kernelRunningWithMetricsConfig_WHEN_verifyClientDeviceIdentity_THEN_s Metric expectedMetric = buildMetric(METRIC_VERIFY_CLIENT_DEVICE_IDENTITY_SUCCESS); startNucleusWithConfig("metricsConfig.yaml"); - //Reset the metric spy and capture the emitted metrics when the emitter runs - Mockito.reset(metricSpy); - ResultCaptor> resultMetrics = new ResultCaptor<>(); - doAnswer(resultMetrics).when(metricSpy).collectMetrics(); - //Verify client device identity try (EventStreamRPCConnection connection = IPCTestUtils.getEventStreamRpcConnection(kernel, "BrokerSubscribingToCertUpdates")) { @@ -300,16 +270,7 @@ void GIVEN_kernelRunningWithMetricsConfig_WHEN_verifyClientDeviceIdentity_THEN_s ipcClient.verifyClientDeviceIdentity(request, Optional.empty()).getResponse(); } - //Wait for the emitter to run - Mockito.verify(metricSpy, Mockito.timeout(2000L).atLeastOnce()).emitMetrics(); - - List collectedMetrics = resultMetrics.getResult(); - assertEquals(1, collectedMetrics.size()); - assertEquals(expectedMetric.getName(), collectedMetrics.get(0).getName()); - assertEquals(expectedMetric.getUnit(), collectedMetrics.get(0).getUnit()); - assertEquals(expectedMetric.getValue(), collectedMetrics.get(0).getValue()); - assertEquals(expectedMetric.getNamespace(), collectedMetrics.get(0).getNamespace()); - assertEquals(expectedMetric.getAggregation(), collectedMetrics.get(0).getAggregation()); + assertMetricEmitted(expectedMetric); } @Test @@ -337,21 +298,7 @@ void GIVEN_kernelRunningWithMetricsConfig_WHEN_authorizeClientDeviceAction_THEN_ authzClientDeviceAction(ipcClient, request, cb.getRight()); } - //Reset the metric spy and capture the emitted metrics when the emitter runs - Mockito.reset(metricSpy); - ResultCaptor> resultMetrics = new ResultCaptor<>(); - doAnswer(resultMetrics).when(metricSpy).collectMetrics(); - - //Wait for the emitter to run - Mockito.verify(metricSpy, Mockito.timeout(2000L).atLeastOnce()).emitMetrics(); - - List collectedMetrics = resultMetrics.getResult(); - assertEquals(1, collectedMetrics.size()); - assertEquals(expectedMetric.getName(), collectedMetrics.get(0).getName()); - assertEquals(expectedMetric.getUnit(), collectedMetrics.get(0).getUnit()); - assertEquals(expectedMetric.getValue(), collectedMetrics.get(0).getValue()); - assertEquals(expectedMetric.getNamespace(), collectedMetrics.get(0).getNamespace()); - assertEquals(expectedMetric.getAggregation(), collectedMetrics.get(0).getAggregation()); + assertMetricEmitted(expectedMetric); } @Test @@ -377,20 +324,54 @@ void GIVEN_kernelRunningWithMetricsConfig_WHEN_getClientDeviceAuthToken_THEN_suc clientDeviceAuthToken(ipcClient, request, cb.getRight()); } - //Reset the metric spy and capture the emitted metrics when the emitter runs - Mockito.reset(metricSpy); - ResultCaptor> resultMetrics = new ResultCaptor<>(); - doAnswer(resultMetrics).when(metricSpy).collectMetrics(); - - //Wait for the emitter to run - Mockito.verify(metricSpy, Mockito.timeout(2000L).atLeastOnce()).emitMetrics(); + assertMetricEmitted(expectedMetric); + } - List collectedMetrics = resultMetrics.getResult(); + private void assertMetricEmitted(Metric metric) throws InterruptedException { + // let the emitter run multiple times, + // allows us to verify later that extra metrics weren't emitted + verify(clientDeviceAuthMetrics, Mockito.timeout(3000L).atLeast(2)).emitMetrics(); + // metrics are emitted periodically from when CDA started. + // to avoid a race with the verification above, we simply wait for the first non-empty metrics to be emitted. + assertTrue(eventuallyTrue(() -> clientDeviceAuthMetrics.getCollectedMetrics() != null)); + // ensure that for a single expected metric, we only collected once + assertFalse(clientDeviceAuthMetrics.isExtraMetricsCollected()); + + List collectedMetrics = clientDeviceAuthMetrics.getCollectedMetrics(); assertEquals(1, collectedMetrics.size()); - assertEquals(expectedMetric.getName(), collectedMetrics.get(0).getName()); - assertEquals(expectedMetric.getUnit(), collectedMetrics.get(0).getUnit()); - assertEquals(expectedMetric.getValue(), collectedMetrics.get(0).getValue()); - assertEquals(expectedMetric.getNamespace(), collectedMetrics.get(0).getNamespace()); - assertEquals(expectedMetric.getAggregation(), collectedMetrics.get(0).getAggregation()); + assertEquals(metric.getName(), collectedMetrics.get(0).getName()); + assertEquals(metric.getUnit(), collectedMetrics.get(0).getUnit()); + assertEquals(metric.getValue(), collectedMetrics.get(0).getValue()); + assertEquals(metric.getNamespace(), collectedMetrics.get(0).getNamespace()); + assertEquals(metric.getAggregation(), collectedMetrics.get(0).getAggregation()); + } + + /** + * {@link ClientDeviceAuthMetrics} where only the first (non-empty) metrics are collected. + */ + static class CollectOnceClientDeviceAuthMetrics extends ClientDeviceAuthMetrics { + + @Getter + List collectedMetrics; + + @Getter + boolean extraMetricsCollected; + + public CollectOnceClientDeviceAuthMetrics(Clock clock) { + super(clock); + } + + @Override + public List collectMetrics() { + List collectedMetrics = super.collectMetrics(); + if (!collectedMetrics.isEmpty()) { + if (this.collectedMetrics == null) { + this.collectedMetrics = collectedMetrics; + } else { + this.extraMetricsCollected = true; + } + } + return collectedMetrics; + } } }