From 277ceeec167421c7da7a7e7c29eab95035b71a03 Mon Sep 17 00:00:00 2001 From: Joseph Cosentino Date: Thu, 11 Jan 2024 13:30:22 -0800 Subject: [PATCH] feat: basic authn benchmark (#416) --- .github/scripts/benchmark-comment.js | 66 +++++++++ .github/workflows/benchmark.yml | 36 +++++ benchmark/.gitignore | 1 + benchmark/pom.xml | 11 ++ benchmark/run-benchmarks.sh | 6 +- .../benchmark/AuthorizationBenchmarks.java | 136 ++++++++++++++++++ .../RuleExpressionEvaluationBenchmark.java | 74 ---------- 7 files changed, 253 insertions(+), 77 deletions(-) create mode 100644 .github/scripts/benchmark-comment.js create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/.gitignore create mode 100644 benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/AuthorizationBenchmarks.java delete mode 100644 benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/RuleExpressionEvaluationBenchmark.java diff --git a/.github/scripts/benchmark-comment.js b/.github/scripts/benchmark-comment.js new file mode 100644 index 000000000..5e3419f84 --- /dev/null +++ b/.github/scripts/benchmark-comment.js @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +readJson = (file) => { + const fs = require('fs'); + return JSON.parse(fs.readFileSync(file, 'utf8')); +}; + +jmhResultsAsMarkdownTable = (results) => { + const header = '| Benchmark | Score |\n| --- | --- |'; + const rows = results.map(benchmarkInfo => { + const benchmarkName = benchmarkInfo.benchmark; + const score = benchmarkInfo.primaryMetric.score.toFixed(2); + const scoreUnit = benchmarkInfo.primaryMetric.scoreUnit; + return `| ${benchmarkName} | ${score} ${scoreUnit} |`; + }); + return `${header}\n${rows.join('\n')}`; +}; + +replaceComment = async (github, context, prefix, body) => { + console.log(`calling listComments(${context.issue.number}, ${context.repo.owner}, ${context.repo.repo})`) + let listResp = await github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + }); + console.log(`listComments resp: ${JSON.stringify(listResp)}`); + + const comment = listResp.data.find(comment => comment.body?.startsWith(prefix)); + if (comment) { + console.log(`calling updateComment(${context.repo.owner}, ${context.repo.repo}, ${comment.id}, ${body})`); + let updateResp = await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: comment.id, + body: body, + }); + console.log(`updateComment resp: ${JSON.stringify(updateResp)}`); + return; + } else { + console.log(`no existing comment found with prefix: ${prefix}, creating new one...`) + } + + console.log(`calling createComment(${context.issue.number}, ${context.repo.owner}, ${context.repo.repo}, ${body})}`); + let createResp = await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: body, + }); + console.log(`createComment resp: ${JSON.stringify(createResp)}`); +}; + +module.exports = async ({github, context, core}) => { + const commentTitle = 'Benchmark Results'; + const commentHeader = `${commentTitle}\n---\n\n`; + + const {RESULTS_FILE} = process.env; + const results = readJson(RESULTS_FILE); + + const resultsTable = jmhResultsAsMarkdownTable(results); + const body = `${commentHeader}${resultsTable}`; + await replaceComment(github, context, commentTitle, body); +}; diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..6a0f94563 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,36 @@ +name: Benchmarks + +on: + push: + branches: + - main + pull_request: + branches: '*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Run Benchmarks + run: ./run-benchmarks.sh + working-directory: ./benchmark + - name: Upload Results + uses: actions/upload-artifact@v1.0.0 + with: + name: Benchmark Results + path: ./benchmark/jmh-result.json + - name: Comment With Results + uses: actions/github-script@v7 + env: + RESULTS_FILE: ./benchmark/jmh-result.json + with: + script: | + const postBenchmarkComment = require('./.github/scripts/benchmark-comment.js'); + await postBenchmarkComment({github, context, core}); diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 000000000..48d0a7996 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1 @@ +jmh-result.json diff --git a/benchmark/pom.xml b/benchmark/pom.xml index cad1a4658..cbf37dc94 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -28,6 +28,17 @@ ${jmh.version} provided + + org.projectlombok + lombok + 1.18.22 + provided + + + com.aws.greengrass + nucleus + 2.6.0-SNAPSHOT + com.aws.greengrass client-devices-auth diff --git a/benchmark/run-benchmarks.sh b/benchmark/run-benchmarks.sh index f0dba3686..9b352365f 100755 --- a/benchmark/run-benchmarks.sh +++ b/benchmark/run-benchmarks.sh @@ -8,11 +8,11 @@ set -e # install CDA locally cd ./.. -mvn clean install -DskipTests +mvn -B -ntp clean install -DskipTests cd - # build benchmark project -mvn clean package +mvn -B -ntp clean package # run all benchmarks -java -jar target/benchmarks.jar +java -jar target/benchmarks.jar -rf json diff --git a/benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/AuthorizationBenchmarks.java b/benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/AuthorizationBenchmarks.java new file mode 100644 index 000000000..3da8fc0aa --- /dev/null +++ b/benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/AuthorizationBenchmarks.java @@ -0,0 +1,136 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.aws.greengrass.clientdevices.auth.benchmark; + +import com.aws.greengrass.clientdevices.auth.AuthorizationRequest; +import com.aws.greengrass.clientdevices.auth.DeviceAuthClient; +import com.aws.greengrass.clientdevices.auth.configuration.AuthorizationPolicyStatement; +import com.aws.greengrass.clientdevices.auth.configuration.GroupConfiguration; +import com.aws.greengrass.clientdevices.auth.configuration.GroupDefinition; +import com.aws.greengrass.clientdevices.auth.configuration.GroupManager; +import com.aws.greengrass.clientdevices.auth.configuration.parser.ParseException; +import com.aws.greengrass.clientdevices.auth.exception.AuthorizationException; +import com.aws.greengrass.clientdevices.auth.session.Session; +import com.aws.greengrass.clientdevices.auth.session.SessionManager; +import com.aws.greengrass.clientdevices.auth.session.attribute.AttributeProvider; +import com.aws.greengrass.clientdevices.auth.session.attribute.DeviceAttribute; +import com.aws.greengrass.clientdevices.auth.session.attribute.StringLiteralAttribute; +import com.aws.greengrass.clientdevices.auth.session.attribute.WildcardSuffixAttribute; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.TimeUnit; + + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 3) +@Measurement(iterations = 5) +public class AuthorizationBenchmarks { + + @State(Scope.Thread) + public static class SimpleAuthRequest extends PolicyTestState { + + final AuthorizationRequest basicRequest = AuthorizationRequest.builder() + .operation("mqtt:publish") + .resource("mqtt:topic:humidity") + .sessionId("sessionId") + .build(); + + @Setup + public void doSetup() throws ParseException, AuthorizationException { + sessionManager.registerSession("sessionId", FakeSession.forDevice("MyThingName")); + groupManager.setGroupConfiguration(GroupConfiguration.builder() + .definitions(Collections.singletonMap( + "group1", GroupDefinition.builder() + .selectionRule("thingName: " + "MyThingName") + .policyName("policy1") + .build())) + .policies(Collections.singletonMap( + "policy1", Collections.singletonMap( + "Statement1", AuthorizationPolicyStatement.builder() + .statementDescription("Policy description") + .effect(AuthorizationPolicyStatement.Effect.ALLOW) + .resources(new HashSet<>(Collections.singleton("mqtt:topic:humidity"))) + .operations(new HashSet<>(Collections.singleton("mqtt:publish"))) + .build()))) + .build()); + } + } + + @Benchmark + public boolean GIVEN_single_group_permission_WHEN_simple_auth_request_THEN_successful_auth(SimpleAuthRequest state) throws Exception { + return state.deviceAuthClient.canDevicePerform(state.basicRequest); + } + + static abstract class PolicyTestState { + final FakeSessionManager sessionManager = new FakeSessionManager(); + final GroupManager groupManager = new GroupManager(); + final DeviceAuthClient deviceAuthClient = new DeviceAuthClient(sessionManager, groupManager, null); + } + + static class FakeSession implements Session { + private final String thingName; + private final boolean isComponent; + + static FakeSession forComponent() { + return new FakeSession(null, true); + } + + static FakeSession forDevice(String thingName) { + return new FakeSession(thingName, false); + } + + private FakeSession(String thingName, boolean isComponent) { + this.thingName = thingName; + this.isComponent = isComponent; + } + + @Override + public AttributeProvider getAttributeProvider(String attributeProviderNameSpace) { + throw new UnsupportedOperationException(); + } + + @Override + public DeviceAttribute getSessionAttribute(String ns, String name) { + if ("Component".equalsIgnoreCase(ns) && name.equalsIgnoreCase("component")) { + return isComponent ? new StringLiteralAttribute("component") : null; + } + if ("Thing".equalsIgnoreCase(ns) && name.equalsIgnoreCase("thingName")) { + return new WildcardSuffixAttribute(thingName); + } + throw new UnsupportedOperationException(String.format("Attribute %s.%s not supported", ns, name)); + } + } + + private static class FakeSessionManager extends SessionManager { + private final Map sessions = new HashMap<>(); + + public FakeSessionManager() { + super(null); + } + + void registerSession(String id, Session session) { + sessions.put(id, session); + } + + @Override + public Session findSession(String id) { + return sessions.get(id); + } + } +} diff --git a/benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/RuleExpressionEvaluationBenchmark.java b/benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/RuleExpressionEvaluationBenchmark.java deleted file mode 100644 index 932acd427..000000000 --- a/benchmark/src/main/java/com/aws/greengrass/clientdevices/auth/benchmark/RuleExpressionEvaluationBenchmark.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.aws.greengrass.clientdevices.auth.benchmark; - -import com.aws.greengrass.clientdevices.auth.configuration.ExpressionVisitor; -import com.aws.greengrass.clientdevices.auth.configuration.parser.ASTStart; -import com.aws.greengrass.clientdevices.auth.configuration.parser.RuleExpression; -import com.aws.greengrass.clientdevices.auth.session.Session; -import com.aws.greengrass.clientdevices.auth.session.attribute.AttributeProvider; -import com.aws.greengrass.clientdevices.auth.session.attribute.DeviceAttribute; -import com.aws.greengrass.clientdevices.auth.session.attribute.WildcardSuffixAttribute; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Mode; - -import java.io.StringReader; - - -public class RuleExpressionEvaluationBenchmark { - - @Benchmark - @BenchmarkMode(Mode.Throughput) - public void evaluateThingExpressionWithThingName() throws Exception { - ASTStart tree = new RuleExpression(new StringReader("thingName: MyThingName")).Start(); - Session session = new FakeSession("MyThingName"); - new ExpressionVisitor().visit(tree, session); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - public void evaluateWildcardPrefixThingExpressionWithThingName() throws Exception { - ASTStart tree = new RuleExpression(new StringReader("thingName: *ThingName")).Start(); - Session session = new FakeSession("MyThingName"); - new ExpressionVisitor().visit(tree, session); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - public void evaluateWildcardSuffixThingExpressionWithThingName() throws Exception { - ASTStart tree = new RuleExpression(new StringReader("thingName: MyThing*")).Start(); - Session session = new FakeSession("MyThingName"); - new ExpressionVisitor().visit(tree, session); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - public void evaluateWildcardPrefixAndSuffixThingExpressionWithThingName() throws Exception { - ASTStart tree = new RuleExpression(new StringReader("thingName: *ThingName*")).Start(); - Session session = new FakeSession("MyThingName"); - new ExpressionVisitor().visit(tree, session); - } - - static class FakeSession implements Session { - - private final DeviceAttribute attribute; - - FakeSession(String thingName) { - this.attribute = new WildcardSuffixAttribute(thingName); - } - - @Override - public AttributeProvider getAttributeProvider(String attributeProviderNameSpace) { - throw new UnsupportedOperationException(); - } - - @Override - public DeviceAttribute getSessionAttribute(String attributeNamespace, String attributeName) { - return attribute; - } - } -}