From 8867987f37372e36dea962719b1e6b51156c7d17 Mon Sep 17 00:00:00 2001 From: Alberto Mercurio Date: Sat, 4 May 2024 16:58:39 +0200 Subject: [PATCH 1/5] Add benchmark tracking --- .github/workflows/Benchmarks.yml | 60 ++++++++++++++++++++++++++++++++ benchmarks/.gitignore | 1 + benchmarks/Project.toml | 2 ++ benchmarks/runbenchmarks.jl | 29 +++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 .github/workflows/Benchmarks.yml create mode 100644 benchmarks/.gitignore create mode 100644 benchmarks/Project.toml create mode 100644 benchmarks/runbenchmarks.jl diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml new file mode 100644 index 00000000..53a6c990 --- /dev/null +++ b/.github/workflows/Benchmarks.yml @@ -0,0 +1,60 @@ +name: Benchmark Tracking +on: + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: write + deployments: write + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: '1' + arch: x64 + - uses: actions/cache@v4 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - name: Run benchmark + run: | + cd benchmarks + julia --project --threads=2 --color=yes -e ' + using Pkg; + Pkg.develop(PackageSpec(path=joinpath(pwd(), ".."))); + Pkg.instantiate(); + include("runbenchmarks.jl")' + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Julia benchmark result + tool: "julia" + output-file-path: benchmarks/benchmarks_output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: ${{ github.event_name != 'pull_request' }} + # Show alert with commit comment on detecting possible performance regression + alert-threshold: "200%" + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: "@ktrz,@findmyway" diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 00000000..866cecaf --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1 @@ +/benchmarks_output.json \ No newline at end of file diff --git a/benchmarks/Project.toml b/benchmarks/Project.toml new file mode 100644 index 00000000..05a4894b --- /dev/null +++ b/benchmarks/Project.toml @@ -0,0 +1,2 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" diff --git a/benchmarks/runbenchmarks.jl b/benchmarks/runbenchmarks.jl new file mode 100644 index 00000000..1b461ca6 --- /dev/null +++ b/benchmarks/runbenchmarks.jl @@ -0,0 +1,29 @@ +using BenchmarkTools +using QuantumToolbox + +fib(n) = n <= 1 ? 1 : fib(n - 2) + fib(n - 1) + +suite = BenchmarkGroup() + +suite["steadystate"] = BenchmarkGroup(["steadystate"]) + + +## steadystate ## + +N = 50 +Δ = 0.1 +F = 2 +γ = 1 +a = destroy(N) +H = Δ * a' * a + F * (a + a') +c_ops = [sqrt(γ) * a] + +suite["steadystate"]["driven-dissipative harmonic oscillator"] = @benchmarkable steadystate($H, $c_ops) + +## end ## + + +tune!(suite) +results = run(suite, verbose = true) + +BenchmarkTools.save("benchmarks_output.json", mean(results)) \ No newline at end of file From 612a43591a3e67e13299c81fa6488d0731b6ad87 Mon Sep 17 00:00:00 2001 From: Alberto Mercurio Date: Sat, 4 May 2024 17:29:44 +0200 Subject: [PATCH 2/5] Fix error and minor changes --- .github/workflows/Benchmarks.yml | 12 ++++++------ benchmarks/Project.toml | 1 + benchmarks/runbenchmarks.jl | 5 +++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index 53a6c990..54041006 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -45,16 +45,16 @@ jobs: Pkg.instantiate(); include("runbenchmarks.jl")' - - name: Store benchmark result + - name: Parse & Upload Benchmark Results uses: benchmark-action/github-action-benchmark@v1 with: - name: Julia benchmark result + name: Benchmark results tool: "julia" output-file-path: benchmarks/benchmarks_output.json + summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: ${{ github.event_name != 'pull_request' }} - # Show alert with commit comment on detecting possible performance regression + comment-always: true alert-threshold: "200%" - comment-on-alert: true fail-on-alert: true - alert-comment-cc-users: "@ktrz,@findmyway" + benchmark-data-dir-path: benchmarks + auto-push: ${{ github.event_name != 'pull_request' }} diff --git a/benchmarks/Project.toml b/benchmarks/Project.toml index 05a4894b..32e8b0b0 100644 --- a/benchmarks/Project.toml +++ b/benchmarks/Project.toml @@ -1,2 +1,3 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +QuantumToolbox = "6c2fb7c5-b903-41d2-bc5e-5a7c320b9fab" diff --git a/benchmarks/runbenchmarks.jl b/benchmarks/runbenchmarks.jl index 1b461ca6..4a568dac 100644 --- a/benchmarks/runbenchmarks.jl +++ b/benchmarks/runbenchmarks.jl @@ -23,7 +23,8 @@ suite["steadystate"]["driven-dissipative harmonic oscillator"] = @benchmarkable ## end ## -tune!(suite) +BenchmarkTools.tune!(suite) results = run(suite, verbose = true) +display(median(results)) -BenchmarkTools.save("benchmarks_output.json", mean(results)) \ No newline at end of file +BenchmarkTools.save(joinpath(@__DIR__, "benchmarks_output.json"), median(results)) \ No newline at end of file From 67a333fc7a7ce87f3bb9d9a09e0d022c1ae40e57 Mon Sep 17 00:00:00 2001 From: Alberto Mercurio Date: Sat, 4 May 2024 19:29:07 +0200 Subject: [PATCH 3/5] Remove Fibonacci function --- benchmarks/runbenchmarks.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/runbenchmarks.jl b/benchmarks/runbenchmarks.jl index 4a568dac..dea58b58 100644 --- a/benchmarks/runbenchmarks.jl +++ b/benchmarks/runbenchmarks.jl @@ -1,8 +1,6 @@ using BenchmarkTools using QuantumToolbox -fib(n) = n <= 1 ? 1 : fib(n - 2) + fib(n - 1) - suite = BenchmarkGroup() suite["steadystate"] = BenchmarkGroup(["steadystate"]) From 12786face0cee88bf4a6c6214462f5d1ddd022ba Mon Sep 17 00:00:00 2001 From: Yi-Te Huang <44385685+ytdHuang@users.noreply.github.com> Date: Sun, 5 May 2024 15:39:24 +0800 Subject: [PATCH 4/5] Modify Benchmarks.yml --- .github/workflows/Benchmarks.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index 54041006..43d3f565 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -2,12 +2,22 @@ name: Benchmark Tracking on: push: branches: - - main + - 'main' + paths-ignore: + - 'docs/**' pull_request: branches: - - main + - 'main' + paths-ignore: + - 'docs/**' + types: + - opened + - reopened + - synchronize + - ready_for_review permissions: + actions: write contents: write deployments: write @@ -20,22 +30,14 @@ concurrency: jobs: benchmark: runs-on: ubuntu-latest + if: ${{ !github.event.pull_request.draft }} steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 with: version: '1' arch: x64 - - uses: actions/cache@v4 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- + - uses: julia-actions/cache@v1 - name: Run benchmark run: | cd benchmarks From d61667db1e643eef221db9a91dcbcaafdca8c9f1 Mon Sep 17 00:00:00 2001 From: Yi-Te Huang Date: Sun, 5 May 2024 17:49:05 +0800 Subject: [PATCH 5/5] modify `.gitignore` --- .gitignore | 5 ++++- benchmarks/.gitignore | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 benchmarks/.gitignore diff --git a/.gitignore b/.gitignore index 31d66670..483c0b92 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ .DS_Store + *.jl.*.cov *.jl.cov *.jl.mem Manifest.toml docs/build/ -.vscode \ No newline at end of file + +.vscode +*.json \ No newline at end of file diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore deleted file mode 100644 index 866cecaf..00000000 --- a/benchmarks/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/benchmarks_output.json \ No newline at end of file