From b2465e7c3023c5b2a7656ba75c4234a62d13f72b Mon Sep 17 00:00:00 2001 From: jason yang Date: Thu, 5 Oct 2023 08:44:01 +0000 Subject: [PATCH] init push Signed-off-by: jason yang --- .dockerignore | 8 + .github/ISSUE_TEMPLATE.md | 30 + .github/dependabot.yml | 6 + .github/workflows/golang-build-test.yml | 23 + .github/workflows/golangci-lint.yml | 27 + .github/workflows/gotest-coverage.yml | 37 + .github/workflows/license-check.yml | 27 + .gitignore | 10 + .golangci.yml | 62 + .testcoverage.yml | 28 + CHANGELOG.md | 9 + CODE_OF_CONDUCT.md | 1 + CONTRIBUTING.md | 1 + Dockerfile | 16 + LICENSE | 201 +++ MAINTAINERS.md | 1 + NOTICE | 3 + README.md | 30 + SECURITY.md | 1 + doc/apptainer.png | Bin 0 -> 57745 bytes doc/workflow.plantuml | 45 + go.mod | 46 + go.sum | 124 ++ internal/cgroup/cgroup.go | 60 + internal/cgroup/parser/parser.go | 75 ++ internal/monitor/instance.go | 94 ++ internal/network/wrapper.go | 135 ++ internal/push/push.go | 38 + internal/util/util.go | 16 + main.go | 335 +++++ storage/diskmetricstore.go | 609 +++++++++ storage/diskmetricstore_test.go | 1571 +++++++++++++++++++++++ storage/interface.go | 177 +++ testutil/main/main.go | 29 + testutil/metric_families.go | 41 + 35 files changed, 3916 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/golang-build-test.yml create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 .github/workflows/gotest-coverage.yml create mode 100644 .github/workflows/license-check.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 .testcoverage.yml create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 MAINTAINERS.md create mode 100644 NOTICE create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 doc/apptainer.png create mode 100644 doc/workflow.plantuml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/cgroup/cgroup.go create mode 100644 internal/cgroup/parser/parser.go create mode 100644 internal/monitor/instance.go create mode 100644 internal/network/wrapper.go create mode 100644 internal/push/push.go create mode 100644 internal/util/util.go create mode 100644 main.go create mode 100644 storage/diskmetricstore.go create mode 100644 storage/diskmetricstore_test.go create mode 100644 storage/interface.go create mode 100644 testutil/main/main.go create mode 100644 testutil/metric_families.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0b35429 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.build/ +.tarballs/ + +!.build/linux-amd64/ +!.build/linux-armv7 +!.build/linux-arm64 +!.build/linux-ppc64le +!.build/linux-s390x diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..464fa00 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,30 @@ +## Feature request +**Use case. Why is this important?** + +*“Nice to have” is not a good use case. :)* + +## Bug Report +**What did you do?** + +**What did you expect to see?** + +**What did you see instead? Under which circumstances?** + +**Environment** + +* System information: + + Insert output of `uname -srm` here. + +* Apptheus version: + + Insert output of `apptheus--version` here. + +* Apptheus command line: + + Insert full command line. + +* Logs: +``` +Insert Apptheus logs relevant to the issue here. +``` diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..202ae23 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/golang-build-test.yml b/.github/workflows/golang-build-test.yml new file mode 100644 index 0000000..1c4540b --- /dev/null +++ b/.github/workflows/golang-build-test.yml @@ -0,0 +1,23 @@ +name: golang-build-test +on: + push: + branches: + - master + - main + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21.x' + - name: Install dependencies + run: go get . + - name: Build + run: go build -v ./... + - name: Test with the Go CLI + run: go test ./... \ No newline at end of file diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..7c983b4 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,27 @@ +name: golangci-lint +on: + push: + branches: + - master + - main + pull_request: + +permissions: + contents: read + pull-requests: read + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: "latest" + args: --timeout=30m \ No newline at end of file diff --git a/.github/workflows/gotest-coverage.yml b/.github/workflows/gotest-coverage.yml new file mode 100644 index 0000000..ac1c21b --- /dev/null +++ b/.github/workflows/gotest-coverage.yml @@ -0,0 +1,37 @@ +name: gotest-coverage +on: + push: + branches: + - master + - main + pull_request: + +permissions: + contents: read + pull-requests: read + +jobs: + go-test-coverage: + name: Go test coverage check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21.x' + + - name: Install dependencies + run: go get . + + - name: Build + run: go build -v ./... + + - name: generate test coverage + run: go test ./... -coverprofile=./cover.out + + - name: check test coverage + uses: vladopajic/go-test-coverage@v2 + with: + config: ./.testcoverage.yml \ No newline at end of file diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml new file mode 100644 index 0000000..a61b966 --- /dev/null +++ b/.github/workflows/license-check.yml @@ -0,0 +1,27 @@ +name: license-check +on: + push: + branches: + - master + - main + pull_request: + +permissions: + contents: read + pull-requests: read + +jobs: + license-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21.x' + - name: Install dependencies + run: go get . + - name: Install go-license + run: go install github.com/google/go-licenses@latest + - name: Check license + run: go-licenses check --include_tests github.com/jasonyangshadow/apptheus... \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..57ee0ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +/apptheus +/.build +/.release +/.tarballs +*.test +*~ +*.exe +*.tar.gz +/vendor +/testutil/main/main diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..a457343 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,62 @@ +linters: + enable-all: true + disable: + - maligned + - interfacer + - scopelint + - golint + - dupl + - funlen + - gomnd + - lll + - gochecknoglobals + - varnamelen + - ireturn + - gomoddirectives + - godox + - gocyclo + - exhaustivestruct + - exhaustruct + - tagliatelle + - wsl + - forbidigo + - makezero + - depguard + - wrapcheck + - gocritic + - gci + - godot + - cyclop + - gocognit + - maintidx + - goerr113 + - errname + - nilnil + - prealloc + - ifshort + - nlreturn + - exhaustive + - nestif + - forcetypeassert + - containedctx + - contextcheck + - wastedassign + - promlinter + - nonamedreturns + - nosnakecase + - gochecknoinits + - goconst + - goheader + - paralleltest + - errcheck + - thelper + - usestdlibvars + +linters-settings: + goheader: + values: + const: + COMPANY: CIQ, Inc + template: |- + SPDX-FileCopyrightText: Copyright (c) {{ YEAR-RANGE }}, {{ COMPANY }}. All rights reserved + SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/.testcoverage.yml b/.testcoverage.yml new file mode 100644 index 0000000..186cc6b --- /dev/null +++ b/.testcoverage.yml @@ -0,0 +1,28 @@ +# (mandatory) +# Path to coverprofile file (output of `go test -coverprofile` command) +profile: cover.out + +# Holds coverage thresholds percentages, values should be in range [0-100] +threshold: + # (optional; default 0) + # The minimum coverage that each file should have + file: 80 + + # (optional; default 0) + # The minimum coverage that each package should have + package: 80 + + # (optional; default 0) + # The minimum total coverage project should have + total: 80 + +#override: + # Increase coverage threshold to 100% for `foo` package (default is 80, as configured above) + #- threshold: 100 + # path: ^pkg/lib/foo$ + +# Holds regexp rules which will exclude matched files or packages from coverage statistics +exclude: + # Exclude files or packages matching their paths + paths: + - ^storage/interface \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..008eb93 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +## 0.1.0 / 2023-10-01 + +### Feature +1. Add SO_PASSCRED verification + +### Changes +1. Removed most unused code +2. Add new ci configuration for github actions +3. Refactor existing code base \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..8d65dea --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1 @@ +# Apptheus Community Code of Conduct diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4d218d9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +# Contributing \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4e561a8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +ARG ARCH="amd64" +ARG OS="linux" +FROM quay.io/prometheus/busybox-${OS}-${ARCH}:latest +LABEL maintainer="The Apptheus Contributors" + +ARG ARCH="amd64" +ARG OS="linux" +COPY --chown=nobody:nobody apptheus /bin/apptheus + +EXPOSE 9091 +RUN mkdir -p /apptheus && chown nobody:nobody /apptheus +WORKDIR /apptheus + +USER 65534 + +ENTRYPOINT [ "/bin/apptheus" ] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..7af87e3 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1 @@ +* Jason Yang @JasonYangShadow diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..c5d4190 --- /dev/null +++ b/NOTICE @@ -0,0 +1,3 @@ +Apptheus for ephemeral and batch jobs. + +Copyright 2023-2024 The Apptheus Contributors \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..96990c8 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# Apptheus + +A redesigned Prometheus Pushgateway for ephemeral and batch jobs. + +## Background +To provide a unified way of collecting the Apptainer stats data. We plan to employ the cgroup feature, which requires putting starter (starter-suid) program under a created +sub cgroup so that container stats can be collected and visualized. + +To collect the cgroup stats, we are planning to deeply custormize the [Pushgateway](https://github.com/jasonyangshadow/apptheus) tool, tailing features and adding additional security policy. We call this tool `Apptheus`, meaning Apptainer links to Prometheus. + +> Note that this tool can be used for monitoring any programs, this tool comes from the development of one Apptainer RFE. + +## Features +1. Disabled the default Pushgateway's push endpoints for security purpose, so users can not directly push the data to Apptheus. Push can only be called via internal function calls. +2. Added a customized verification step, any incoming request through a unix socket will be verified (Check whether the process is trusted one). +3. Apptheus can manipulate the cgroup and put the process into a newly created cgroup sub group and collect cgroup stat. +4. The only available endpoint is: +``` +GET /metrics +``` + +## Workflow + +> Note that Apptheus should be started with privileges, which means the unix socket created by Apptheus is also privileged, so during the implementation, the permission of this newly created unix socket is changed to `0o777`, that is also the reason why we need to do additional security check, i.e., checking whether the program is trusted. + +### Apptainer uses Apptheus +![workflow](doc/apptainer.png) + +https://github.com/JasonYangShadow/apptheus/assets/2051711/b33c5f20-a030-4b91-a6a7-bc62fe1fc6b8 + diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..553d692 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1 @@ +# Reporting a security issue \ No newline at end of file diff --git a/doc/apptainer.png b/doc/apptainer.png new file mode 100644 index 0000000000000000000000000000000000000000..9903e6ba8d8086d805991771a9d25480e403f421 GIT binary patch literal 57745 zcmdSBbySpZ*9JODw@3-n-D7}AixR>}H{yT-lF|)Q5>f(^A|Wx--AIWb-6h?vbmzGT z)c5zE@BLz}^Ve}LVb(J<_jAYI*S_|(_k+KJoD?251vUf%!IOS0t^|Rgkc0n&m{-9S z6w&$?@Qck>Lc>T!&l%!#Ku8s+yXP-XcCVZ+0)gc7wc~se_v8CHb%3;$oQ5<3Tre z+<_V^agvt4Z^nD84ySU3jH|flhcW^JixVlUHhLdQ&DlJd{0+2|&7H7(9Gi)lxwI+q zq1_*TEzu6PGI8b2?;h`7FAp2DWLmW`0RyubA&oe2x)7gltaWn;M|Mc@X0aJ1ypP{iXG8tgHG$^%z@ z#Eb|ApVYVe&`}hTtwM;Iyo9mLNlC^gf})OJX1Bj#<9If!Qdzhta94AgSL03#5sO9v z=Dh@2^TwG^tRi2(u+;uo(--HYp1;HQo~G|-Ku*%mX6@8x14ND&LS}V~Nz1^PUZXIz z6-`>(-)>qvD0aInBs8ELD{fS`4|mS9TYw_Z(=g^R7~z;kdGLL6>DTkZ-~q|@7RxI^ z&ApK-6fYzgIp^J6$`shI6sE$zHJLjYe|qE-v;D)EooS8I?$!VfZsPNj2eCaLsDtYY z)pN5@!%|AZBT^4A=g4Y(Q%#@KzKEhS?8dD5pe)BH*VaiU)aYP`E^`_&NJ8dSiBXoS zrhqfN{!Kj~4{A5dlNrUE*T>%$br#+2e-}q1s^b=4Je?iq1RG~b3$udz+#5YAW!mu6 zPsTBRYwMq+DaYrGCV4wcr4AgLay~gM1h}{JE2g@uv~4qco!#A^RU9Oe9n_JNcOHkY z&t>2J9D+mkC=VmeP;8&aank3Q2BrD)q;vdg%iE>f{ z>doox&G_CsN0Gg|`#Pyu|3eLx2Z39p#KbryCxHe;-Vm(^m{&bC5t@_ zUJ+sn`d^8qEj+yXoN)WRBW`-mdaXMAL(7u|?#Zt0*N&Zs@w*crT)LtK4VjhDfG0vA z8^qGM7cU^up3z8Ghip(fvRz#1`g0-9#3=)Zj9V`8Y=k3kPA-;bydhS|=_rD&LdSIz z;dlbM5hpKl=-QiV+`3hi1hC(h4K*Vox6S6-5gXTR)9;7CA&WC!%DLnY(I+NEC!Z+?r4wAg9_r8%pMwaO@Z?qA}_5NVl~r$N?46*p(d{iEpq9I!88L<|mdc zcIrnJ+gGO?o>*dZcah*h=Y6LT9v^OfjqiW9yx583F?std)dNv=o~nqOUeQ-y*8ksc z^05^r{XP}fS#d+D?#aQX+{!9U6?S>E6Gx1VC@_!XZkTL>P>T6@9h-91hDn9vqFq63 z6Y@UEt%4cr+|PA%GPQcU;>jSjvnTkLKQ)He-S|X!dAV-gVr9>i9?BJ-cYltTK7mW)ZD!{eA@srKsE9+3~38 z*+F>X>%BZ8mYUO{TKuzlXco2OrKm6eXpsxwYPl~mf{Q5e@#d(jiib;;oRhmURw?R2M{A9i8E!&%Obh6eU0&mpRB z-@g4hwC?}R6Nm#56<<#$^B)On{eM|V{rdnQg6d4iG(=0VExF$F9VB*H4VXm%N zumpHg)w^qlTW!_CE+W?nd&=R}&iy%qI^Vl5H)Rh0mid@|<);X%T+O0|#wM0A8~OOu z)HyVqv-Ui3>4KonxI@XJm(nWXXYstIBI{pU<~fbWq$7H}NPclZd2N?41!^N=1a?bw z&Yc-U`uGC^O^+sAZsJo3Y_EM;ot>Fc&u(@=Jf7f>pBcy!6>u*8Ira6uVSFi2OhojJj6ML#d>tcNimY|l>{p_ON6`{9mRiKjmobza*& z5Q)-8n#28y@Th}>>w5>-kq-1B#U=yxy{TiT$ALIF@_CxuA)WDGuy{Z0{5h>ughBSR zWB)sy;^*pg42`+AN71>?hcg^HlF^)+Vl@G|ww1S*b!7V7lO1J~o;tB;Ib0Vzsz`o4 zgOE!Qa#*i6W|4D+8CQO3{Qh0hn?!o=JY*{t7M=UEo31UAo|;TW!!ZC(okYQOwiyC- zig!Oh*3i(n4|7G$-Hj{eNPKTjvo1REi9=`L2* z7?`-Y$v8UXhR#!-@7#}{&PU3rSG#UTLR0o9Cll6CgkXaK!oObfI%&TJ+klBY3L>b) zqG^Bn-mACm}H z?iyN)`oQ{c3?`#kn3(CRSyA0_T+~_@e%cb>KNOwcq@G^oFSK>CU5oIQ7vbLt=9?I) zKF(JNhKV}BAt&#`1wCw_1NqMxre&g`HCqM|CauRwJQ9ww)MWH>Ym8n7CB01nrZbE_TNI$t9eu{V2Et&wOIJiCHaCY+ll;=7q!$OKX;1Br|w&+67XzP}Ln8x_U(?^VDAV2~=4|6t7`z zwGFq#i-)@98m+rzfa7p9GI~>A-zTI54vOB)uRPV!eyHZOOJhX3M|gisS;aPi%*WfC zQoz<1G3oA}vZZQkon}6i`z?sv#z!gVaz@>i;e&Nd*;Rwu6%W7F*PoxR>j{0ank8#> zdpuhDD`l8{mdUvi353OKkkrX@xA)=Ii}i-C5rbN;>$SDQ?y1jS)Us@J9mHvO{@xgu zNm$Q0=JBq}L(GLaq3>=8-%i5GqVIbVTsk$r16E2Ap`izRqvvg+!~l@+g01HsFZ<#> zRZ`Mo)sdK|ijq>x(UQVX62B~bMr25owHmkHh zzTfAG;=Vg=ngssY4UY4!t^KYQpu)5++D+mtdw@rM5|@PMoLVJ zY)-+Y;Y`(00lSr6wgiD)-ubMhIXT&6^|x!CbaZrhH}4%Cc74f)pC%Q)%S4*TKM$kj z+3qUeE&60DBO{c`o;HW7UT;CW)?6J?9wDJ*^*kN((Ng(;EX`wisi$!AY;S&U&SiBF zFDNL8)GfeN8y_*{xD8n@VWD~*5eIWh ze#X}8e_^U_ECcVuSNo;Iz&Zr`J>0WG<};YYhUXamh%|I{3&U{S0M(jXK!2yl1*(GKUK@<6lBp zFNw@0R^K?q|CGg1sRdst>MA_}G>0vu+jXR9ajD!2I$3V@NC*vbgVb5{e6Q@htD8@` ztk`-!*8OZdaYhqTm-+Y00UAXwJF+$NKEf@kcLzH@@-@D}sGI8k;@p$m*vbb;*DI~H zFY7fN)S{5V@8u4V>SsLzyGm})M@wJ5z=Y^tvpl`3i^ge(xY-MvKDLG2Ak(geXk39k zTi#HDcu=4XK_O~&O6met#E?&tUw3glNb>t5@u6stbj5#sQ`1E1CWIN~_lGO#s)><2 z(L>JTzax&6kDS?Hzl>1oP(?_qg|aaMy_EQPgAquLJ3!tE0B|5y6OmoeespPM>wgd9;Qz|+(BOd1iKtubLC@H z-OerOK!$Rf6~rh!G7=4nhU@W@bsdH`~kn(D~-9 ziXRPjRTD`Nu%b#dXIhtWKEIk%J&o{RjelbpR#`B zV3F39%_Grj4bhn>n)v>`(x{w?3A;ff1$C14L1b9$*Ed$eM)e!zN!OwF^Ymbn zOQc}iPl9Zqk%@`&h}z;u{(!`6(U-NkhJe}CP8*FXzDP^WuIrle5lblhRG zdT+#Zf5HVyg%)4%;jW>QI9zu3Mm*SN5A$gU6o`jt&z4fP^KNo)h{(~>Hv-Yr2JoXj zP~#i5uro0!DYb}e%=P!L!|bag92yx=Fz`9q*|iGwu@EiT`N_jYhRS>XED5aK+|+{h zoCpxd+nx?&s=}7M5ky2pY}GFsu}vgM^KBM8t70jgh@#Zu-xN~7vLu)E-F64m5eNhh zDd(qy{O8{y1|s6G-MnW;J4t@yMl(aMEh4Br=JuOj2FajLC{BCBhHSF-G8Z%dhBVUk zw{ihh5suYYAtt1yl9|pKuaEZza-Rt@FuXBf6C2K}J&QFMR;{*Kl#4>38uaz`J%bt; zJIzgfeG?(#vX`yze&bt!Q9;_zwec@(viKgoTw29bIYu%LN99?glED-MIZyYCxN!}$_^j?=81P072Hfh57nIsOIB<-zo4_I;3=IthK`I+wtwn8ke_4iHT2>aZR}B zH^^xq0%8@C?~jvIwX<2IiS_aQ4dO+etl_P_nS@TInnf>rX%3;v7Bg%hRdVYkEh;Z>`@L zKmurtzQoVj+S;?-JP8SjHy8v_Wh;X@qrP}Io4w2?sy=w>Lp8J?Oo%7gE6urX6%-Ui zbar(?xw&7uM5+jer48$)j*V$Y&EH>cJ)~roi(?($hi&@W;ztfdXtL<8^WP$k(#pVF zMsV($X4dT(Ic2oBKM4TInR^sQH1{(ZG4@I@Z{{{&|5?B`A|i-un5tTpDm!@1$24j} z+xl8LgJExceSLwOi=1qH(|@mV0y071)Vxk%#l<|Of*;F{rrr=#+AjA3k3h$xumzd> zr%xXH%uS8_&3&CLwQi@*0&@T}zBe@$J=@M4h%n|?wV!vzpL9F^<&8zuEYkZS>8?Dt z>;W|mjH3t{U%uBXag-nf&dY~wFjfC;gC9<8wB^m%PTO8n3Zs>6A@~eLl zbG9J*YSdLE9KLuAHOQcPRks5$SN&guMj&d8v7QiI4x!5lK>r);%9LSDbZcuX1%}7H z9efao(7^u|Y179ql)=f8csr`sB9Y?&?tDLy-rMN|mN4cFYMUA3&C_gHQPG zC`FC*pnG5$^`=VuFdDb0VMA__T^;}g@*O!Z-4{~`2t)+qe|U)tPOBsZ4?+Y1?l+L% zFY^ky6br4Zx>vz>^C`f1(|^Xu`W=F%Y;5Ndj4h%2!c(^$8&ADJyn0mgjdK_K3i3-r z-ySI2y>lmMnC%fZ$7JU$Rl2zq`~vI_BnFb_m;sCCu0Gjn%gD(1`t|GP<|Ya{?vF~l zER7blF)iBhOzqdMJYxyME`^1KD+5_Wz*6=n-E}yl#E6=kn-4adXq>!zmzS4S<+59` z-5MNhwx4w|*MSY3z<@v!XGx#~K%kla=0_v!ymPd>3XGgXE!$|arut(1^RbKZDapHQ zPv)WX^Yc3Hr@JX3BJ8AXpRkHeht>eF&xL`|q?Fp~ud4Yqla+R+;1qzTs}x&KKO~)7odm$Cr>D2r@lk4d zw9G>O<2|n1w{LTDnk{uFspo1x=jG4#r_AXDZb3mqqm!kUgB@({&mi?~XMIvsNT}-h zH(vnQ!VYVcrX09UL|Pb+Nur=S31MM`*RS^iuRlHBFDifVDt^KqacI8E34q$N17Afn;#)zu`j zwnbha#qrzt8kAJN>iaA!CU%7iv*Md#iX=#Mg7^%v=(6lQROnb(($dr44mJZK)hZks z2SgKJvyLGhcP>)aPx!{Wva+%UG~KRehl>%PF%$g<3E2#}MG2}@c2mO!o)*27b#gic zn+gD>!OJ=hH`DnbuWAcBKDRGTzNoKJDh)M|C?e*L0d}Auf3;AOK&WMZ0#!4h%I=$i zI!LiG;wVU|GU%GBkWm?>j$;4(pRJCXlOm|g8+%!Xp|NA zK$ztOLGWyDPCrUcT3S&gB#CLIgKsh@IC#9mhK1lW0zn061){CBHI~=(JIA)bRjqh3 zGP3P#wKD6a^5&)d;@+@)@4+idXm^-6UjzCyHKiLB6&@We~1 zmeta;qFrP4G;8$P7kj&3BfOlPL5-$Exg8(3OSiHIub2!(KgzY*UmGXcqgpCns{uSM z0$y%O7ylsXWn(;bu1>Ao_tkiZi6r-7smE1@ISfjj;%?NA87tB${vc@;7Z(p*94)s> zQP7(1{KQEY$4#B)y<(ibwPS+E*dQ!&?Zwty3vrnHe#iK5<~vZ?(9z3?)S9pF6OB6h z%miVs`vMQJ+MH?Z?pBBzJW3ScVsA~OEl!RTuw!Rpn%^DLS^P@MrA16aqN-MjGP;%$ zptiULj)>=*L0eS3sQZ~8&|oa5uUatSHqB~bu;Q}llTS~ByfaRBxwp6X9hD&9&$YnO zPpo_>C%(0RxVv7lm;k5+mQ@ZzZ)`=Vs<1Kc7JPLmPyO{>BfLyr?D)j``iD`>qaCvm zQ&s|rsv^r_eEPr#j`lXD9)R5LyrHRSad9zxQ`r9J4V+5Br@pr;2jB8~0Emvnkw4d1 zA-P(I3hc6H>wIt2BAlz-YW90eI7_0-S_Ke03zD&jm^Y{x!CA!Lx1tYRTv&K%XsApy z@0J1N@$Nf!^}geh#_?HD?>>MH4P>bUJ*7otnjSdup)>PTs?9sKGTUZwdO*n&;9Z}r z75%;N^pfvwNko8tesMot)d9F$oGlkGz)MT}38}C~YN)38VQXryzAb*qt*sUR+F#YNU`hQN_SCMhM%b+m zf}|lM-(nWztH(dyJ#e`6N8UGJ^}B77i+@)SGLBy$l!OEZ;$UOT$jCTP`;dW?=n618 zb=+(uOZlbX<$l!~;30skzVicPW6dhZ+c;Z3p;6mAJ3Hz+uS@FBDxSmPa4jt@kdN?( zfLj*`0A0kmKRqn-o8<)mqpVSYPod#ifbuO$9vlG;VPtL(2>9RR-g69*jSmx z;LkGn3menlK;o?im`g(@g|JHVjFwa?uMJ(6bMMynHb}B>Qc}i!_@JqvpzwPm9x~fC zyd?x?1OO;aajr-UxI?v5`rA&Fb9C*2zViP`XFH`Lf1A6*h}NUoNgc1@@gQ z@S;N(*!O8+4}X-i59HPr_@)v>g@`LHT|S8SDHD=A94cMr;t)9Y(iOc5BE1eZf3SUe zwJH$EBft@MJPOAA3tpO9Z7BUW`7p#)E)p)I{Tu*>o4UMdii%B&$8(9a&J zhhM2H|C6m=YKuHh22u)6&u`CTpH?ZeP!pM~6Ue zfSW-Qm1UuJz~>LQ7ePc4U}k2%eY?j&G80LtrbPa-u8kkCUq1@;rs_Na$OZ=o%Suac zE)%i&hJ$jpcGCr?yr6#`Se8SuO$iw0&%Xl z4`b(yh>B)sWxbZS5i21=4)HLogDvA%evwvY_ULF79?$; zW)M?baej7u_s$)YsS1$yfvkTSA8Y__Fo3QiV3#^)ThZ-`C1%6d?Xe*#5GX#jC{?0>As<~F zv`2LBKR#F$v4-58Zsdn~^Y^v?ehb13xHmXEB%%G^UglH$aLK(~=7sf~Fnj%D)G355 z2B6!%$nmnSTr@C0$^Ci zQnWp*SQZreB5NU8B5iU-igtw2BfEJsy6uoEku{9Mso5L8Gg6}?X-5+dX~2cm$gmR6|eIW zh#`8CY@A591OIa8H=h-^fo20 zLcQ-n&oxUAHcLMI)pE!a-HwcbCI<2jBQyg<=4Wvu)-H5(^g>R*u{VXW?460jsAy=4 zY+M_c8V!MXf+~LI;>E5+1!5!zIodO4iLc_}Lej7INKfAtfn>hPCk{JKYE;T*{5Q=e z$lDt=AX}>>fOG_e5F>%R9U2tW$yr+$&g~du=dpFM zV+?HHX8$Ni-qr}5L`X1P)vN0OvT>tF`uRFPSYYM8E67%e+dDg>Fa#aeIENh~bjpiH zc9(i)yFordK(jVn#9~0ksZnPl05#x#wLVepAW(Dd=1o}G$;rvg%!|@+Nu{R=^Tu>- zdow$$L+ijyq~1|#7QB!EnL^}%3L87S$#9{8uCA`RIS?%x{E1cvb6l2^{4|!qYo=pv z8WBEl`#p*DG#xGi@k6ut;}C3NqMd z|1QXsF#JQ1Ih@d6M1U09e60L-);D^sCPC*>y+pY9@W8_Q$*z3HR6z>Lc;9`>ORy8C1P_(*0p#m}7^mp8-2H1$iR0tdyz~pt|>>M1X0H?htDv= z{zsuf8}ZZ~ptz`Omq!*xUUh$9p|=^22eG69pZ4ei5%_&7lXg8MhT#8K0dA*#@7JiZ zvNBW4jdiBFjFxEJe+55VOdWqgyOoH*h zB!1Q=CMK4amaeWR9#>F+nA!^4U!R=JQqMcGXWazJP+M#!?Kqu)fOB&w-Q?^2_-l*h zH$vXNb({;60-|o~Oat%NT&Vv-~TDNCGl_-f7E!~G!6}-Y_>>S9>p_mwxaT-(Ht}2zT1YzK1a-DC~j*^p; zNjTJsEhZC?GAHMqJJvuLM%XWO#=|tRC&s@%+bEgaS?&uF-unU2&!_xBq?NT^IJ4}^ zj{=6aUZEC5M}qM2M~j*Y*TXF`9aph!hl$iyx?JswLJ;33PZs!q8fuHamN_R}{CQen zU|lS6>7;xtHsNU`(taUNo|LSG@cJ75OCstF2dzu39C^(Q;VklQD;a5XgjWu=l+@Hn z2?>*e+N_w)PWLDK7&T+thrzs@!YFf?b8)}YVlY#V1L}E`&m3S2niC^CdjNeISk=8( zS}$I_(9rk+kfkNG0K5MeIF#AhS<`{c#=@UE_ypGhTTK=tx_*6WKog_{IMn>ssX$@? zi2`uapQB~#AOb`PHw)4c2|4c=t$1oPGad9nGxNcEL0uQk$J%X!e@_M z`8EH6pk!jq zukw9fg*=VK7O)orp$$k|Z-iXjeML=qU$t;{zbRyWs5Ry0?H3K%nnjNG3+*33Y)NVW zG7*q|?YHKnwZ>aKt9N=~*;nmG)8rEY4cN#I4NYqy1L@s0UgP+qNGF z3UN<4$3;Cl>$w@s{lq_IK%CleuyVnn!}f7Wf3;yl zFL@2BdWYubIlSMUPMT=kh3tREoZPfj6^T~5XC>kt2S)XM5_2s(U$pXRwKFdhsH8MH zc!cpzW57^RQSrW zfb}`%MC9N#t<#?%T!Vk~^z^*EcC+7uA965d8YT=42`E~tNc9WoWkNA_pG>S0xF5U` zO1JZHHBUft{ee0Q`lWX!&TQWs1IZpedZg*x{Yk9#RB0|0czjMts({sA&nQyGem)81 z1%}${e5W|J{0^!fP>gGj2J*aV3qVInX?Qg#=D6&y86YL4baNH5%6UzvxHMx=brDQy zpwfbT1mJ@E!8FF<9&IVY|J$sVyk_ys8OC%wg?_{5&*@;{(+fGxuG-bkkq98NI%#Lo zpjVWEShmfVYN>6l(j`PAu`13+D0#UOI~QQm5fKqUC5^`@!S8MecU;XW zWB)lhn}U+`p?k~zaBCh!j;$m=YHK8(Hsf^*0PV(zWpdUO5E~$&0Bz4nA4rxFOnzb= zKu{X~;$T5a<~xgAyN!)aQBl$ITpQm~_Z6TX?Sbe)jMSC#D9OoN@ML|m!;ym3xag%| zm8>Afn?=ztz-s*gFl(V#e(W1+z-Iy9tG1rUuC__P6JojxG+i2j0iZ$6nH?}Zm@Y?+ zFiZ$Zwo9H%f0u6eT}20f@E0foH;Y=)y7B)l-q3`0M)zVnks(ZbK{>A8j^dmzSawtnARBmzkZ4XjUV0N^;l86RI? z`;~s_)fD;(amuN+B!J+mAnyc@54!hiMM3qo3WFb%N$@Z+KZ!WafyiC_Jq>{XO;?~q z0g7REoSnyHg({RvVZxn21hnmM#cyG) zI+zLUv1k1%is+5?5C1eM7N$df`-Qlpqa(m`bekI=8sk zmjSZI&h~arR#siH%bgnP|A#NEN8yrEEo9}@XeGV&$ZP_kt@K4>Kp+{k^-U&-iMqTX z)duVV8{(j7$5n>~${;K>lIM!U1$urO%t!?w%d>hBDFyW7ZYWXb1GVPh;Gn#`T<2tl z3>L+y`1%Q`2}5q({;t7q(hHL|c^l~={nhjrx%fzw^6BL; zDB^&Wpgb(u7NogbTU#62*bq~%%PB&HzXhvkYiolN1jvNmo&v#&@^@zbpe!_u9tTJ% z4DQ#MF3M=XHI1ck0B~uksjM{OSbz2dDLn|F*1F(Bx~~F9LtefVMgA#S{6k6kUtenA z{a=`<9vhTNn%7om9%bOi+X4OKy95UA)e2&AfexJ(6l~(DB^1!V`mT@~w?p)aID2M3^devFp{$vinW7>gPNjt10+vU@?eBs&flWOE z@|}e1`RP6=FLDCE`~`wM6Em}~j}IP|K*XS<+J|o+q0cHoDqI8N24~j53D7!%Z}Qau zoj&V^!kS6c-nN^CKv-1t<(W z%kg#8)G)|;ouZgjKGF!-f~M|iP%k9e^Lqp;OW8Rl23O|G=e#wJK;`$uDwg9JA~khm zQ6p2k#+4r^7l_01PJ&?7&_uAUT>Lqn_TJ@|whcf9U+vFeK;>97znBi9pg!mqeeA z2U;AJYTJNO75w@|C1CsGivtwLL^Vpe!)hLgaO{aNCKeXx)6$>7T19~_<%2^OF$e;x zH(N^&yOMq92Oi_zJhxpAI*p`pao%@}N}n-m{Xbobpac6#4IpEia9-t-Y5&NV2S|pG zj}Iu=f?A^?&?Gc{nwuYEZyh0LV3Q0=SD-INrIkUB-6m6%n}v^$Pf+j}$PHVkO15rb z*)FF?b^##s(=nFKqFW^r;Rb2fdT%V(S{2af0%~2&fgb#S zRbJ2m0M-b)u3nSuS!9LHO2c3{SXjmY%K>%Q@Zq;v09N?eZa&JeB$nuR5cG6xkT#Ip zZJHnZ8}NX6fjIN>D@bjMje3$nH3`t&{x{a2*!`}3W|Yc$JW-*4{yGEjg|b(pd;7KL zry#9Z&s}}MPs6eapdDm*w@2`qA`;E6rd%ovKobY}2Jx`+{5L`R&zT{0g#X$E_)e!P zg7I+WtP}uEf4|D)?mDVSy1dl>?3Js!;z3p!7x3Objd1kaqQT!iV(r`vh|Vm`If0nI zXJk?4>lpfb!ll=9dZBMKe(kTV%<6See9-^ z$gi>08>Xd$SNXwSY8?pd8$o8Xf3v#{(MoH;Y8MmT6IKAt1-*BVHC}Wjih^d* zS@0B2GXt<;;tH^oV&lH+L`0x*+ZUh8dA^lI&C^+<($?73Rp^-GcJ?D_bON;>?kvhB z;lTL}U%o`gzyO65WIBj84;uD+>hk7$lx4&isa(xh!v9Ym%2ih<9 zB7WEZ8;u9rFA_BR$y-2$&(7u~b)Sz^uXI>e$uUDdKigezN}kKEFL(5^wG;C;lxtm9EqiNXz1;=-xv^J5W)sdYi&K3b1=`WneWZ z9<&K=n3t@m^WPo_&u~rhr2faMBI4Sjp-rG!i6+)~opnimPnze|2oDsRmzwHr5DA2I*7Jk{ z%IP0I-h<`!bMwuDio)SrD7O0{*V+tWua@JAc~mkgTN_mZvravGxZ}aMPg!NCta0_5AFPa{R=BIL9$px%QawzDf!?5BvAPXVC>v0 zC%>%(C#|e-K*i@SD)9iufTj?}H`kojk+Jn7i&DMPWy?Au{=PqsVPqk4&z@$sixNfD z(rfLive0&ACOk>H(~b3)CIBl31N~cN{b3{y*8r50Q-2xsU8ZvvAj=a#&tVdYQLx9XL!_go4h(U}*s*xev+m5Xb5=ERNQeVVhV zq_rD~HIQw2C-P=JB>g!L^d7Rr<&m`64pVdm+4xYNj;Pg)IN$?W+D0t0vHc)I_SplI z<5rn1HXFVJg@&+iG*>xov%*Avf@lJS;h*DQ_ySGw7(=#;)mUC1?a=k9!Qpak?oXcJ zS~RzGbaZrfe%6av|2?184UT>RtLurko?o4W4uE3-mb69x4)_VzBvb;57h*b~&g`AN z(gGZNj+z2e`^l+b85>C&f(_v&-+b{vuV(sX-MJ!>olNVn>^mtG&Z#oX2|_$Atu6@p zLe`*7V?`7QA&iWvz@vbwtY?+uIcJqHxWXtZS_|?VV1_{GK|@2UqFV$_L_ppJ|5$7u z^u7Gb9#Y+ERf1ppFo%f6>JB%w=HNv!l?$9 z8!=@6;JIygfVKpZh4l1vplIk+JF9Y{FQEO!!wZq&QYZFE@{G<@_?d=YH}61A4iM7iupZ{&9@&QU-{sHKkI@({Sp{AYy zjf*v}j}rtP-l>vSCO2%&w}IO2!-o%n$$mQdavK79lWbkvP51kj8Q8pK{g(=sQDmX` zf03C~ut@CIC*se8W7(8H=R5uXqJRBT7<;-Va?&Wn|&(*(ji~O(81M1TG{5bi(mn>c1H+QQq#l>XXeUv-5K}{8uz60XEIGglonv z)Y|_V0jwR!BFR8pt*Y791i7WnUkXvPr}VoGWNfGTLLLX2H_%prd4OzUyBCywkOChD z1_+zriCg4d8pq)#po#<-Fpz3N7-!sWm;rs>fYnG`X7YfpIKEyix?Tv0!Tn>@n4|KAGenLA2n5^>xRNI43U-7Nm# zR`T(A@4{9fhBq_WbLAGt-($FNsQKhqt>#6G8tl5m6st z4U^;bkpm5KX>##eB_{LA45~;Bx2XHuym zcsDM-?~3xbP2tbZd>R3u@ebt3kQ?Nq5}El5))%cIOt7dEv=L||O@I`+9vuTULqQhV^&rxu2W{L~UQa-QJm9@pX&lfTA&~C7V0sr_Ge9c& z^Ad=Xf0lY-7*l+N|E25UKOWb=K=QKuL*;t9I|N;N13KWA>7-#Q?Ds-%19awtK-`#w zgidUvd#^_ZfCiSjJRl{$&3$~+5FQZp78UvUj_&P+u0&`h{j1D9CFG+MxO#V74=iKP z)7|9T-8UVsp&lv%i+>8*EmShiu8LioBbz@F8a;*wJZVAG+X7zPdm@^+uhtm#4Ea&j z^=Ks?Pg=~az=<2X@7<~+^reE1u7lp{kBpBrwGeZ7!fWCR^WIfGH&G8IHk7XE^S*kz zqi)cB?M_hi`?S}7O>mj!R>gZgCTlXUhvvs8Xw!Xxx+JF7_frwd_Qv;GRsS=`HW*u0 zMe<6qYU`J3F$Fx(&Eb{J?8t@zmps1uIY1rg^7g@phD;C?m|J|dwE{X zQD$+m?N=|fe%BxQ&pA8CJnsAwOrSav$ng=! z_)c`D)r*!NSfXw<;f1wssRRb1vHT80iBZbQ$EO4vo^v_0&6Zx#UbITK{IQ>x~nPqFs&06i}6BP?yHeZRX^fk)}X>kw+(++&Vks=4Y{UId^ibX zVq{bZ2bxhlpp5+)%I#P3>2r#QHh(-;W#tSEoe#jqNjs<|J%2SFW3?rqwah$C7%@>a z_lrr7#b|&I&z;Awu-0XU?JP9_mX0O%{dyHKs+eThd9{j}87FVc#d8cuZt?*|S7OUn z+`Ho`!2M{x9nyi@N+j5A^!gPOi2@wV$+tNohk9i4;_h>enAhHK6-YflZX!#;z%`>F z=}27Jq^HCA$JqTKn|WGdq6#W4fR2D1^DahRz??C=$=Hu+3HF=nA5ADS0tBOck~nt* zZSON-TB`%oL?xFv+gEZ+J(9%sqNVw}p}MqCGzK&=XHqb6P1yrkR1C>YB~Y)t5k!o- zta)M5Ut#>ZaFc_LX_}RoIF?tFuiu1u(F*^y({|W$B9Q3Wy3kO?jtDP?*-*kq_LqzY zuk1H0Zcl9G)qnejFl4klB`|Nemg;a}jNa9Y_{axto;pC!4Z_stz9i74fWHIaMGJnm3S~;4_hjtKZ>60o zlJ7^EY-!NG#{OeJy3SBLqr>A)t9rlBfnX~w7H@voLYcrbDZMDc2-%_p56W z7t^Afmj9kxk3|#K>xZPERR)|kIHvYri00Th$3+x_IC04|;Kgyy?kE|smtR`s3G}Q4 zZt}+$2CBI~{``EUd$CZf#Ev>~`y|GE|8!?A(~eB!=w}9oOZCn{IHw#n_c6;pjl1D1 zyIYQuM?~c_Br|G z3EhobaCPpGyl0ps-|Y1lK`N^Ig-cAs?<#zk?$DdXKRXqUnm8{TXerp_lr?-aMgY?} zJxKiO>Df7JjOFh)3IEn>V9LiE`0n{~MEU2|rpH>$-^W`pAFlhq(q7bd156c z7EfQAwYaR#BebY`8Ru=FY7&y!$_%FU9)PC?=Y&K)dGg-fRc zf1$(3a^LSIuDcj{p5+niV|cJ($qZNQJr1pBhZ=7Kf4<$SARg^)XcV+^xFhGKwH>C; zaMN~o*1gacY~@3Ox%R;>=M-s2)HV~l$KjmX3o+ZCwY6CSe9cP8jcR7~ine|vr9KzV z5_4Q{AzxiPz2(cLv%uxFbfszkfca#_{eTpayMJmsljqhkqqQf&A!5l_5Rq9DlN6FQe{68hxu8ke6E%h ziK8M2EiTcytmpbSoibwHe(w~F}m!#rMxl zjJ~L7C8l9D+z$AGlVc|X2Wh_Fr6j}HG*5clIPWhW-sT}PNL-WqF0EiHvSk9?sPd7d zX=dmIr}5!IMis032+b&S8af90%g?2(j(P>#_<8P?uf*5Srtv6vLV{CJRN`;yvjH>$ zkpO_4xSGFL{HXA!VrBhQRyjo1o}JJ&u)#x@_UO_@bz=w?=JUc=I$wG0vB7z9b?-~S z9~dstY*O4(R~xFD^fFC1m9|;uuMH0dgw4}7zARo- z(fhZBKBv!0-8j7ts~5ye0aEPc7Aj_{*{B=aA|62sUVjNRZi(+hTvkIK2V%t z>Cz7wIV0&+VeglsFis6l$Q@1bGZ}*{8zzK-@0Xet;7zHk9%{=Bqk7SvrnfFCwK9XC zre#<5N*O&ksY`UfI{JM|oTtT{Klsh*Lz|9%5jvi$*uVn}_kV`D9VxTjQozQG94RGX znPYOALBx0ugVhmS7{HCguP}p`BZY$YkJIF&ANd6kZM{bxRyT4}ns05riN?k%FxdY^ zlt!P&EPpWmJic<8{PhLduuy{`}{skV2#8F`~pN8Z7y;OwG0 zCOr=`m754v;?YsX+%71BZx{`-k*M^i@cv`b^NO7j)80sq$bqX@Rbp{z%wgx#1iibf zk@4y1?x&_tH&Q}?USj6I;83L^{3YyYV7v>FoNOk!5-!jFk;BiIwpsq~(Us33xOJkK zpM>`IQCStLRcZYiVV9+6c(oizw!Wgw`=h)dg>E)Ma3RA}c=YjmH|7FA)zX_*+HqVl zF|5H9q72{RP8Q44> z*XMy{7VB^OI>LSwNof_%Nw5t&uSJ6m&5ayonl4`_k2`b=ypWw*SeD%CYJ zGho35WU!g$9|LBpLq``)Jw27a`QFn})mVC)r_ClICCJ|PG5^&Hr|fBic(vob`>|OoD2k67R=H7shfCsHYGoY~!)$IsTjFYG~>*St|ebZu>d{pX&Y5 zZg~ASM7IJP2W%u?EL$Ob>kG`%>Z;a2@&9W)u$H`4b-?|fTnZO2$q znnoJ)yhHTV3~F(vaieh7XN1W;zZ1qk3*_T*%;$Msc|*U_ZG6u#GUMr;r^p8piQ1w^ z&s?n^zwJ+YQ|#_evN>9YFz!oj5NG3jLLs11Y28RzYX^(vk(PO!4_Hi63I_gqUr|dl zg4eU-BclBY+kFig`La_IlL0JBe)`>Y3m2oI{G|S}Jli$x`Enkh1R~yIg0_d$n|$oE z4e)Ck>OBZRxa+^n4)gZBNy`}1b<|!a*=xUPF_%WEMGJ}wQcJDdYLfc zb7LJr|d z(QzpamdIE7YLzEvYgxpVumL1KYt~iZ!9>g52bo+-6YD;+!9u^Bz|JQ=*3w9#2ddF* zBG=gOJK^2VX(&+NAYd{lH*600(4=uUPU=U7y-I}R6f=Wkt|X=?sWWH!AZuo-Yra$TIG_sw7iqG!r#FloHosmgYiq8l?F={7e|J! zceEF+MDkisW*;krgoLLEAwrxGDYMIN2NEv3gJhe*t$u#iSA4E^HmjBQefbePJ5YV9 zg*sG_3WP{OWR5FiH*RDk8OvyxWrWfwVTt2Iv2nXA|5y%P8GIS=Ye1d> z-M^2=bjng?e25b)diCU1=-IL#?N|)GPAKFYa_k!{3hn!B`^{=JtRbj)RHc(R?x2Yp?@Jo?fRz#G^knVTS+AH_WbNLy7E7@@qSriq>^TB{6-yU*Y zYC>zNR=Il{D&7@+b}p_{n?K{SvgA|A@V&aA{U?cIeH)2h@`4R$@?Yk&8(hC4#632+_S4&!NuFBP&DUFc3{3A_tKEjM}0$Y%5&4$?{q@jx0Q)?=}q$FWps; zMIHg9^UUz2fBmdcNAAYS%lyt$o=HQ@K>w?RQ1I9<_p7hQqO?yzfA5Q5Y?H$Z2#zFZ ze;w5PtVn-ZX?+58NW5Ga0U|@A;BU#-rXQcIKX7uAw-^-VHfyjP?lLj?X@e!i`^#h^ zEZ0MA|NgAR5o@e}L`z#I3sS{jJJKDu=GFB|JX{ZHgF`Z%r&P;$SN0JJ zus#=lt_sW~uicRpKVRLZ!Bvs@90j`f8Ns=7;UyZ1L0rW(3^Fv9Kc-oQ5q7_MTHiJ`eeBdMp?-I)O5m#0=XFZB0q=%_`| zuZHm21@Ejme#>0;FLWvhi!zCEbuKHkfl*61Jl`_rh0^g3Mx1Hb@5n^ozH|GhKLK+V zN$eemwTP$BU{0HyzRKSqMQlSB#1bhuHSa)sw}r#43p6+ zlL)2do9s!I8W`x_m{KrF=d{tEncrFyCx09g(v3wNzuQej2Yj7C`SK_-5GbkJhFdh9>n z&?Y6|CWu(Xd4U)I()NXV5#b5gHi{g2XwsjFYb1)GnyhJy=RU$1N`2IVHIt#Vl%19P zVl#_Di|u$XsnEUBf)=OMl>a4VHS7Ryu9t2I=~KT>&NWj^rnU>)=QCpa*gKEJUYn-< zmCFF|>Dn5gn?{bhYw%hNms!_+VVNtPGY^VYz0QTO{z_4jhr88gpDhNucdl#fu7-yI zZ!%CT493CdC|T(efD{6vYES}Kt#ATdorA5sffBe)Or$^XAhgnH!-Ty>nVi;%)SqI^ zhO|V*6Na_yW&_)E+CIqFch zLc?xXu6wQs;V}&6*35D&_U2gxQ_bDhF^gm_avLo)Q8bv|vCxPGJ|(jD%%|X@X5_9V zTlS{qlY@FVSgV=2p`91Vs6$olDo@7I-WNGR&!9|nmoARX4rLD3*r|%5kou~u>%>m9IBncQ&G&o53 zf5p*&kth_bavmapxR#=c;ZK0(Ng?5T?gQW~EmuWwZmR$S8k%PyRjV_dK`2qEhqy0= z$Qh(pho+q|J2Tcr*@W}UvVm;)t5hk~={qbT0HFUtE!P#Yo2rvUf+^2U<_kKN=?j0R z-2#j&1AJ{BM<4BM4*)bAqTPbO?`Vg{?tSj+fYC>|x-8Y}Pn1&_GpaI3NIem5@+`m? zvUglI-rDMsUs$j&wo-|3ek3dAOYsJgy;Fd3`qLqN*XEpfk>v=&OCl7!l88ayL<}le z?+S~OW8=HrS4c@fy!-C$TiG-5E)63R4itu1KK4MyCD@)<+Pt?njYDEjkJ(oV#4(-r zJRcnj92C#)>!Ep0n}TLoLIk)7XZNZC`n5hFp|lhXO9r9M^{Y5NNec`ldclk=^d$V( zs)6+DRRvzotrB~)Jr<8!7LHKhscZjwO_0n}JJSKw3<`wDj{(GiGp+MynZA~2`!hUW z>dUS1GBd>Gy&?p)OZQUOtuNk~Yrl z77ai8F5sjY}(w z&^-f%6VTfM)q)qcz+L71YhN<+2Q3@wWkCb3>->nrd5ru<5iHO5oMV<2)(O-Vse4XJ>jYQ!w3Qf%AH%ZrW& z$bJ2r7&a}#kDttOUcnT_TwtH#LROstmC0ln3>gK*P+uR>nFN7X$4^5sjwoB$mHK+Ud+&k2wfLF0tSiWVbf+km>m$nrk5h=wMIPTz;>he<9k zi9sE`T*~5+TMfsui|gr8>YlsT8O~t?R>MhW)g`^8gvS)fq{#BVVtwv|4g&p*FoGDG z2WFwYhl))Gc++iEoSct<%2FN*-C8c_P>3dhX z5mrxM>>1VqgK98tAMRbjzDyQk>pWOjFBM{{F_i#)2vbT>SWywgLC$2FDJ+7a-h~pc zwmf>}NyR-wgR^I-I<*|GW>)#wfkrQX_tec@?o(-%S>JMG(Mel$rGt}GsYJkyk)Imn z-D-6mpzCL4mc$c-$J5+7c{touOs`iAxGe_l_9lg~v@}oH4oL)D*T;qA!y_Z-42~;D zJx<*`SaMq8-VqTh*^8A)YRt19VIJK%(pUJlw?gEN2eAa4_Lg|qOE=SO*q(hl=&S{& zWBROBktMjIW@?&o_qwBFh`{slmlB{$8e7hTz@+(iuWz9m%0 zKdI5z_gYAMC9h;O4a*d~mgF5L?a>%d&VWLR40(fUoKDfHfaw=)mJYgT1Tc>sj_$Vk z2h4t!0@Sea~wG*wk4u1nE)WIk9(H&1K2-^e-@FMSVi&y(XkPMj#k zca{gHa-KRP_sAn{ttIyi(9$ z3Uo0n0$eeK77(yzQ(`RPrj^GEBCoFjn*S0YG#zd)(R$reN0*Khh-J3i7^rk9F0L2H zp3TV2zA*qgjH;7td>OLDIJm|0Jf83+50LbDiSDEd9-Fk~bOd^_58SgDe6#ibj?|W} z9exkPhj&D(;b0k}^z`-h-)w<~Y86>sAyLvE1-f5$&entcd_Cy&04qI)*FNJ36p-){ z8mjN#38EkO`@aF5)?}rmmIjNK0L8s|S62$<`HaRMD!D7`nc!Y{A|owLVxHspM?t@~ z%gSPA4#5oI1dbAzkMv7*#PX?-Bm$as;?%Ps{uDPNLz1v&W}PZc9z3J4$jDtFK0C<# zpj~6v`!(YMHB0+75q&@_B>nMwkP(PMrr~!g2*tyQZdjY7FAjmu5jQ7vhWp$72cwRB zb>+}iIgL=29l;MDdO=(|!iA(R9rnJ8IDki19}Sx2tc;X9BIt2C9{gIaWA6vj9B57{ zdFa$WlB$y40=o_qxdK<{Rv}()$0H_0;X86=0GPD7O8f#aXg~r2WS5MA#6IvxI8iXH z4+Th>7>X>@-ocmSDIrX-YwGAf=Mp14YbH6-=paOBoeF0=n7N8BhDPRn4f^}nKAZ?|HdQv=9|Q?f_jh}K)I`e!i2nBC(bJIBpwheuxiHq zz-d@ozkcloo-v@HiL5(P8PkpPx?b_Rf~*c`9XLm^XzJ)CSmZSnLXUZw?l7u;khpzx z)$`lyYFw}i{S(xLx9@f;_=(lVQKTU{I3PxQ_TFzf04`6+aeLd{1JaDekr?GHR^o4{ zKl1R>%uXXedJ76YFmWVjf4-m_+(t{ID~g z(LL!uVYz?R!!y3`p%k50#O0l{_gb5#4_PMplCfj(do!s%baDbcXxfyaoo!LQ2}nhz zkDfJjZcJqie6irPN!IK$?eYYj2E0&}bS$(B(pKG-vy;jA?=roWlrq_v{K4+(8RHChNLR1fH*u80>fBMgnhjH-F-P zU*)z?b=X#SYhaNic7e~8XJzn?QkNHY%Up)`{_!qD+$f>lVOlze?4{e}zjxIh8p>74 zkLrv0t;r5oBXmk$WR6zyMKIANeAZAp_YrFX~icL;ZczUE5{?pMzql;Rx>kfRz>%DdmK;Q!v(B`v+JBp zCcX)xjQm-oolguGzAK%(7HGtwsyqxoj(DHg(*HTh{djtcwu6ZK-LK^^I(9DA4Gj^h zuiskCoN%{O{-FGdDkPeSVYjW--{$zMEa#@W3JcC&CA6{va!?8(sc`Jh1E-dG&s?AZd&B-NbApUxIg&c zbfUIKwtkkAJ^A3F7sH*ol_E32Aae7kJzbL8d>Pc@-S@3+?jE%X(L>Nq zK$(oo{xvz|qvwS)1NQF95X{tRnXOcO?kC$pPuzS;3>06fN~z-$ljG)_n95F6vwi?+ zWN6yyltHtPmw!@nCoE}4MX?!twcF(|C$#uU5vlJT(bBHu1IT+aj~atoF+~z1cUa6j zDT;x(# zb$m?}oJITJJ3x}c5;RzS(i#D@5B)mi`_Dfo3R@b|(}j!)EDeN4Fe%P3C^z{475QC3 zoqL^?EiK7%v2Pbwjhb&Pj%o>XFLsHn8ttODw$4P3h~{)F>~C|g1T(vbWTYqb?RmOv#&5fUp+I-zl>SB1dUw8P&j~ctO!bWwF~YE06V7X ze9__{tJA+9D42;M%Fj0*F8OJ=t}fiEFy4*y%jBLJv4>KZg3T|QoK6MEZQ=`O#Xmq@ zQZ*L`qI>nCZDoENktmfK!90<038GZ{|6yan#=B!a`e1QTsfw~d>ZbD)rt|ErecKo5 z>2UtggMJlydgEE?#ZIBBEt@vNeP*kioO>%4HJA5c_t#MM?e+%>;IPsC{ih^w{Ky(T z&9tKw$u6!LU))R(naQbnF+gkuZEtCrZjEwS4XZBUcIK3cy0K*W0tBun z>H+eHog%%dK@VC^j^RubC(O|cJ=GLDC`KzMcDTs>McGOV-jGrik5 zq{xaK40NcnT%k`255M^bg-0ueyrA$hIG?49Q?R!v3DlZj0;@JKqi~$>awswr`E7kJ z?rvqV{RA%7AYh5`TNz5X7z$QVc!c?b=QwRl3QIOxc7BJL`}L_dJ4669=RgDj*a%!n@aJ0_Q-p$ZFOXt0ZrU?s%Yec;E%4B`D1F?8X&Iyveeh z-JM+{8zg&)c5&asEQ`WA3CbO!+>l2b;&ol9H8e?1%b4&rJ zY{jaPmz6GEI77XkBck~=R>!Y9I#XWm%MN$RsDQV%{alG#5HO4A<>YgCeh5pE39t|* z;=J3Z5~ztrkzgXa8$$hTrXe9oY_EwLem}Oj;`WLB^h_7FhpLZJ4F*foa)&zs!v-!F z`0kH5Ds?h+1WQ|67}iE&$XuVSN(_}MzUvv&da=19g-Jpf(o5;5oCO6FMklaX?L?Z*; z_s6$sf2_Z`IYU?|H`OUmeRMdskl#A*g{^;ZylZgK&Yii=xVN1%qdquLkat5+UW7|l zhqL_9a{rFND~b6z$C(*8-}|Q@VX#7DdDuVT%OqR)?wbU>h3V3bnjwZ-H|tFlwE9siwwhso4~Rwuq`n%G;E)hn#uz>lq(3 z_sLH~LSu%?JWiFzjtu*hR5t$SGr-kM3bI;-ZA&>;T!zv>NlDzmj(_2JX7nns5;9(5 zEO$E$yo4l5wUm*`tW!k)g6=u;1=BMPg~2?vAdpqW+?U^xK6{#BF$C6Nw!jcZa{o_~ zhj3-+g?JD@UmXd9Vv88iSt}CsWG*9E6Mb?q4whi`4sJ2&mpz*Lfgvx3WK4G+l!6%e zrKoVW1U~*P40)6bdkFE^t4O0RWSs06uKzsi-o<17D&jJP=gNiY{ohC3;1Lu9fYSWW znDmj(r?6h{;T|s(ETG?H;IfM%Sc3@cCGH6zG0proHaD4%_PR{wSe7BH91erieQpO( zMOm=3E;=^#j0p;X^f5BVoRp8iQc?`Oj8OPJt3qg9&0%r|UFz9$|4t5MxTwdEmBbN` zf{^?TZNQ_(Pm?pk6oIRR_?`z9uOm!(2;=Zr5RbW#cWCl6-66Sxv2@uj-kI+y64l;g zHXHPR6A$nr-%dGt$zNjzd;51ZOp073mSWZPBS0abWZnPI6b-a@qW2mL*V26eG3~KcJjExc$E_Sp_UM8D@i%4 zfy76f*Qrt5pRZdTJQSl^#7V4C#Bc{7R}W~>KQ0uCGsTY1g#0l7+&+%8GT95{GTzRw z^Yw6=%N9hZ+x+Sk>au(CQRT#Vp!ejsy5!j$ITqrN zamVqgZNJtS9KGD0VgERzo>tb9*Ji8zai}FZ-wttuwf)%$a`ljQRy9Z;Jhi#Z6pn_* zaRUM+KHuXi8d6v9Ov8fWetw9EScFMQq8FPV$3~8N=*j@0lvlRD|KO5+){}Q+E(|Oz zjZ#Zfi=w@hqF!3%r+8Ztg%73fTn6#L<8b48z_q10*LBTUyL6Jq!$+SSe z#9>70_`oeeDU(3E5r|=kKV?4fmQeZV(fB;3?Pr(5AUnHp6V71Oi20MaCxxGMGl)13 zDsF6F5$ZmVpGXp5fieE26|+tT%9Rl+24)An=rq%fKI6Y;+5REUaNh`Cpt zT>u_`VzvWJk8?moqnx>Bfx|*)0+MGoS@gD*+kxvzEUCx(T4YOevu6I5pJHn98#ws# z-B@0JX(EIomm`Y8#*>2XGReRgenoDt)Qb|?p3U!BE1fjca_U`%r7G%Cw#VFDTHT%M z5ZhNPeeb*#?90Fes}E4RY2mJtpp+`t!@v8cAaU6|3P&v|pkFF5*G$DJj9F{`E%$ts zQCi6d1;um~mcjfx-9HjUdds*-1xRe~NapA3Kki-FyYHf$(S78)V7scRRANE=mV|EZ z^CJu*Hcu?FK+J9lQBvWbp2%U5;%|wf#PMjbpid}n-lUZci<38w9FKBt7e*!F3LHnh zsf6~PM`PLsTQPzuC;4SQTmBsV=t=X(L$Fa|xTKZ_HWwo!_;1;5H%D0j;awBZ z5Vf~Qb93VnoS_Jx= zt|XHdJSF$jV@_Sj%7~Q9&XcjKyj5O3{iGw{i33==(-jqZ0*MTE(4Rv6T&f9Q6MAt6 z9xoG_gL3wri7)?h%zQ%%)@xE^%_aDz$&ovO z>%SJV%b|M3mW8JHu@4@L2Z8PjoDNy^*s{94loIayo;xcX*mFuwq*FL}`aEwGH>T?x&rN|9`teO>Yh_dhWHA5cP7?YGyu%}7%MX{Gq0saDB7Vse6=B@zxmM&D zjGK`bAmA7&H{e~gJHo?%ELdaHp7+e=bCVP*E$bpqZ3Wwv>nU=2Qw=}O3AKbfwuHaE zOnQC&pvH&5v+VCtyv7uMSG7lBa&3QeYiG#rc)L*FkN&1YofREj#se-TX2BOGA?syk zg0D3iIu5ouT}xbPlr*Jts!S29e5(#;fWBPRAW0LGOq!*F0=Gj@u6QH`JxR7PXe_~S zgml&f1H*YosOZuplXR0SZ2yEATeKYdzuQ5X}&-Btc?Y4vb8zkj2y1G!A7=odL z9*|@&;4eS@s1A*={s9YcwYy_#k2so9TvC>TgLz5aJJf8aQjskyqlvE5O+UNNOvltN zwY0}wB-Dm`!nd@34;JlsTo^-WWH#V#*TX(0OeDwmlU zze{(%LB^{mNK=iiYf1Qcok#n5t1~k;$6}n!Ot41*Pr;ZY7!y`UX1oCGHJxIflbaQ& z{)N*QPyLSNkmDFrbr21X-#B#!#w*2u7WQlc#^8=G-#*U2PwUyRoxLU#OYbOpRf#gC zi9s_@ID3oJgsNEYD)zyA4R8m-sgG3JV;Q*Z{W>1Kp*%+BPm=15dX2?nN|->dX-t{s za_()1S&N|C8Z_7rg;1u1dEy6Y-0O6=5n{XF?wVNL-!#?SZdd)_u?+wwFUMVej`PP3 z=^CetUQ$QDyG{o|y=b{Lia(Rwho@|9MKGyzlCwP>M@2oI4UhYemmc2_xGWLVFGd?^ z9wwJG6uWS_=U!Jzlq{p*vWfVJrUmFYAhKi5a)3id%AN<5vY96^eq*3g-lpyx_sYIaBrum0*1cJq-`IvLx?q3R z`iRS9vJ}{$3()ma3=|G#Vdgy%)B$(xC3p{RSS|0`MtWE{4sctOls-(vLR=E{^i0`XEk?eEt zh#O`4&E2%#@0^Sp$;IAP_<=WJw;vRPaEb-}_z=d<(Zg-wuC==|a%hYFfMc#&F-sW!^~Az#_h6}iEzY){Fyk!FTmJ_m7Yx1|Z%qWks-ZJCO?$_2A3w{6i7-Ug8>BJmj zRU9MFm3pd9#-m$eJFc=Tkd1R1t>OG*V0 zFgrsd#rLn#TshtC54w5Nk;m4d9I=!q)ps^Q*B2QnsgI$xqJl)Re5n$a4Ai$Izk+El(@oV28J!j_#su>%Fg=+Gc9Hw zD@CUFjSd}5hwkl!wnZl<29F)!e{peSB<6*9J!_oyf491tTql@=aZ^w^p{=6Ocx~2h-mJdclPDT?cv!Xd z{>Y>PKNhOQ1e+m9p3Qv*~sx|Vi|Fd*%n;2T08rP zf&LR~w#oz35|3DXf&(X$&HO}7!Q4&qNcuSY-qf5SVl{srUI~;vzUzf}&q)P5M|u>V z=92PGf7aSn$VF%E4UK@`YcWn#ra2DJRA3f4&>x@T_8&+q>26XE4(cyRtC#_K z*X63W*j24axKR=c1&#g$XiXqtT2$vtof@)G%Ew^i#KWeLkRHMxZogucnkJsKMo?SQ^U|EXu1?Rbg1@uMv>Qq) z$h@n2)v7Fsb>;c^*Y#tIo6PQ6&edVy<6HjEL386UZFgpQ5VI#+WBb2qYh=8Y;e6ziflNq^xg3NO_4I4H)LNC_*(j7 z+Y2Iu0}_B~!&HJmSj-p1g%vBM026`PHJ}Pf z{qSd)?dWK3Z3S)JwlTRiH+mIo4CsGXmI0`YDI_&w)2c#noXDuUN?Vomtv{1_J!pEt z$Q`e7jZ;Lb9;w-ix5;VsHiw4Rpz%p{fQj0;i8PJFgNswocJ4ciA zg=r1;H;*{rzwwssvnN;9IzED%o3sJvr6F+tps}<1__0Zu#fxisOlfQ7n99T8k6KTH+4Lo5?fVxD z=+AKskc@2z#_3e^>`te*#}pqa=Y;I7?Rx$V)j))2+nqer`0n_%G4&!sr~mci`0e{wJ@hH{ZQoaoui6D&Vy~JwD~teYnQ&qEcNv z#{&Acz>9AZ$$kW~Yx}fxP@C8F(~F8b&f1Hy|DnWG>3C3n^$+L=!Vc5Ff!HC&@ve(k zQsXFSs@B!+uyUdZ&3XU6mPx~74z^3K+Ee8=)T!-$BpBg3nwJ-wGy1|#nHIns+?EQ0 zPxpf0V|zmZp%#6N?PHa^y-+D+>Pz5a4w=)f)a4^rrsRA+yQbyQV-DN7l#3uJKsP)z zPT}hlI^Ua&)ij6o>NDkBd`=E4byvzF54I15!G}rq*q5AZC{qLRl^sWtin@A6|7Zc= zNP)rdI+zz9uPE`lG&iOYzZh5y6gRRLM*9|;dd~&qkHuqOj(3rfSB^$9kr0?@5P`yg zlhXj8Lmc!Kf5_P`J+Hlhg@EH+o(!K6Y;A7d%5Sht_#5~cbyx`6z3@^-;=MiyE{^ex zEkBuOxw#9-_U>L`{E#q+-);P z=f{BQWdN={rS^u(4t=>_T3TYb5(lp7*m*1}1a+pQ|Jk5*?k}hdflLmm&+5et*e=54 z)3b(7Ad6c#*kQpb(*rT0Z0sof`#Zrv5~t9!K`m7qb`Y}07vH_E@T`3hc3^-i#v&+; z@3x(JY}c2b7aK$BE*GG5b6PR=q0wo|R#a|nl_G!f_gYp`iBz6l-s60#@fjMizp zA%Ilf=xps+Eog5rLE^QGgF4kSRNkcn*?VwfVr~<4F zjSO>Zht+22{oN&-ShN*zz%05-l9O}Q3JO2*=eU~;geFuTp&9gmtK~iqDLbpBHaG}8 zeQsk4c3 z{WpGmUA=|N@OXTz8XhiC1OF!SaTa!jc?pz1tKyCC@al0`q`Vi^jw5(0(GIseakIC5th+25KFfG5v_x(k| z5d$!Ydd)5?<3-Gx#`S{M^PFt*R8h)eNQ_!s!I`(nScqDzv&L)Ut-bQk#7DiQ8~`$? zbe4B2r5ALGFl>uIaO(Y4%9PSkl0z5A}9dOy#b8eE?w;r?` zi(NNTmlp!}%lcyVX5hcQCDItAVuL|+`TV1!T=kb>`th0gv-%@iIiM8bL{xx@M^%k{ zFv(@h9hwg}ys#D4Ml+7iAUx+e(!uJu)w0Y#qYb9)E?ws^BjB=<74Zdki_yDGz0HWi zu|-1Z{6m7tAbfqAeB}G==o+!u;f|8&GMgJOCK+QU2-h1(lzZ@d+C0M2E*-XtbQER< z9_*&Z@-3|E5%EW-NT>Iba$Y$HFaS>~M!ls2fN`+fg?Zo6aZQ`0Kq>jAe}$dAd{e$6 zbn5OA(mM5W+$q##g0{s%=#s@?a`#hKAD_I{gq363nL8O<1ul-Th!bXKC+V#pSeXm{ zCMiRE1=3FQQ4xO&(K_jT#%Skx7uekQih9fJdvoGkx2Dvl8o+nF2Qa*F+1LzdJ?9QC z7bN8wv`-KP5RUjdH~(U z#ke3%u7>dWdSIY=sw})Ft{NQ@a31}7U8zrrxbZfUIoyn%&!LHbyi8_o)TRzxFJ=?k z-kEUr`0vqocBQV;Hn^vEBDUXGyTBG%fTSU2J6!jzxZkf$da&YtUwpLt(Hmyx>fU%E zE;0q^`CFHl2fC~0slo;Zi@L+BZ$XN@FW7_)GrBs_2@q3uC4!g|zv8|vbiyYfle*Hm zImatJq9y0`f=w)I{#n)6rdZLExjoLR@3kTSuVH-ReS|>9&}nh59``b4RLX{h%eO{w z?M#UmmV)RqIscis^D>(gFP9De55`4BRumFx!U2Eu`^9(ebORnwp}88&5Yl_&!qNYh z4tscvpsmOww_cKf{&W6ks6uI5=VHnoTZ^%WU2@bg9`lj#KoO;U&$n0H_~~kn~e0O=wF-uf5Hg-KPqp& z2W^`Fp&d%x`wRbgU4ccY(}O6|u!A26pC9IC>_DCDAP-3{j?mgHJq1Y6Tf349B&A}q zwMnyzF57}14PX2b*AE3c@pqi38$ktScFmv)0eVoadgFoW*K8!XBGKg7hN)MMs0Mw#bsb)H(k3FSLFmtfVMRJhp0=1oGtO z#rYQXf`)e}7fFa`Zt%PU2`HjxPVeH>!7u*xSO{Nxp~j1#eEwhhT7vyXLE}kLad1ax z>E3t}PEidr*A%K?Z~}PK>%!~Qo<7^{E^ARY#ezd?$xqly~&T+(;$%V6#cNvKHH7!6T_8%*{Q12i4iwUr=ANBwX#$* zQcG$cdUAlaZ+4oKyPr09iK+;Gv0J}Vff>D&YP>V*@_o!~PyH7O7AfEx3^tuAq94I) z4%~1xvRP>GntNn+SqozpW^07J9wjU}?N+vL`a51xXldN!(}U4n`l1|JmT~r@%GWtQ zzc@#^XoYxQMm6w*Z>#{?V;phgE;6RPel=vK)ePL?(pyM30uH-2F?@68vBhv&e#8Af zWO?5=^t&8R`5!N_VRJ1aYrd}TUoZO2&Lemg@pPS0RXa}ft9M-7ZQnDRy{j-sS&8N)$5j(ShYOnWg(u6g84m?iw;`!Xt+F>QlRV<+pr zf)$Gyncz}R|9}8+v5}sPm1xjIuGJ*TiW}sqL+R~j(%A`g`{H8jVS3Z;`LuUOAzjG5D z@J@H8f);d6!{KmHLsxaNo-7`d{j4Dc&Oj6u(Nce&dJ* zB(x}?uvagZr9rtC>~k}=#~+`)(zR4NfbOknmYcwK7@l_ZLAW$FDUlhu=Jp076JSIy znCS$J^ygG4M8xq+8GUc^Zi3Hz;@`i0e7u&oC3r%KYcbS#Wut4}5+F?BlXrU-MQ>&mL zJm@kjljU$1XHrfT_~7V;>+cA!Ptq?I;Lb5iw|P0Nt|3o?>-_%kCJ~zghfIN3PP_tpA2t!E`-YIgB$VaTlo2$6v%ox zCKMwByq`d8dF|poPIo0WJUQE=n^Dw3GnAI)5bE$jef=h3YrF}~4!#ZJ4eyRsD3wDz6?pC?FDAs!6#V zc{J~0(w5J-6YG3u-eT0+%x4d1qX zF~Mc%a$R%DmlX8bl6^i{Nc3Q8sz5cjaw5{J}L6p69I*ObFq{ z?c0~T+El)5AjyWNZAiai zvPQ=3o<81HLa<&7%HfiJ5_@BQIOD`e(LukBiZYa=j0e`QMH_TunQVJtA{Naaa-P|p zSp|W;aX_I?!6zU{Nl78V$8QJMOnRsVf%&w>@)ZNE8~6v@BG9SsgNl+%0d+F--fB72 zXjjU5Xthhk9k+8bAb#=Pn!8F>4!f$hBNUouP>n^}mT`c(wwQ(VH^co(1!O0KF-NXT zvzSVK-cWvutPOU-5Al#st|A4$7>`B!lUWm3n`HdUv%1DO*$ox> zmvyH44VM1;U$$B98!1H#9jU7r1btzHJn1rH7)7h!pE!QHl!oYr{kuwxcrlTBOLD}> zt;O#g5`^z5CH62tKdN8EWW@jFb!RFvz;IhJ3;QJ79;3K63c3!^(-L!gfoGcNB)Q3y zbM(d>YQ$hwY<|H)m-xbn;{xF2Pm!B+QYbyThbLn3v9~a%t19K} zr4Y=UCzqT0g?~w45EFT)C^h0h44Qzb%~P6pQ7sNfMRO%-SEf~HhO7Knl5(`H7R}}b z->k0zN9hAD%SLgve9S#g7YCRK9zl1C4E^zP0X-!a6q;ZA_@y&L3%sKW%hBv+^ z=2Ey*Hk>_|X6XLKA}NG=<|(K)az4F6`aqR7+tX;{7q$BU-6u9iN<2u}=B~7B6*6*I z2&K;PX)|q}&FN2I5x{j?H5A{<07JN+kyAENR)0ovvCFHwJFPH`fMpilY`P|h&HR{d zNj(-#_NLBvZX8^8_lhIUZ_@rwtnY>>##vte^_`m0i-4V4(y{VWkH8s?Sm2|>%KKs% zDtTDYX$3r=eQ_+#VKuqV*JR5$n*C8YhD={7cI774%9xzv2)-@VNfjL3z_442CCc{$ z>#N!JP(OZ`Y21b&f%hiXI@cZNc~HhlrEtY}_3Dhrsw8C>Yi*$%f}mcLp=1Y7?qv(5 z>H3*xKS}?BAg7Xs%eV86b`0NYyf_&fDLOIh%bnHmH{X!3zqK4c!}WaJ>o2F=Su-yB2uYY8&`e?;^h7SZW5b|_Bu-yKe zham-&aQ*<|jaKH+;wasHpMYJ6b@3*mj(0lXvhe-}=!XHrCW+r!Q7TKjyW?S59ok49 z$)I4?AFbqR&^k{%7JSvZj^L?T&E=?H#k?QIVo32{wQ^HN9M`g^Yt~wt`WkGZ1$W<( zhGx=e4`yd}7L@F$7r7U>@fBGxWL~4K-7*FDQrTF`M;@M=llAA8sq6zr(2E*)k)ym%pN{Jn8EmJ|}ytU1;-B>m~-WH3VvYQOPM37+;a z?(Tu2UY0m0>P1Jeq_Vinr#$kzyL&*CT_M^FNgPzEKHT37FS~LGA24-(XF6;2ix!j( z&$6E4wVna`4VFsLSY7}s*x-w&1;>l%9e5F3-6&lG+wf>(+=_sRR-QKBy;8Hcxha0~ zDX`yX-{HxOp5nK{Z(3wO9=3V8uhP+cGgjvR3&9{PNn<+9vRvjw*f#e_c#Hqxu(9I` zJ2A$$O8Syf3iA%KQoM~M@+c)Tw}Z|c_ea3O_e`#J!{AQAt7awe8}dkKtG4LvV)opF zhj~l)Gos>tOMTu)gGhowA38eFfy5B~i!!~`{umlmM{VX*F!j+%b^L0$e1e`K4@GXKKP?6tuS|Wc+vk9N~ z732&u)=xqSj=ELe*w{5BEBAp9+Y-3v^23Q#o1w#gI2F>t;g5R5*m8 zUt#X!{c5*0EQ9YYce2C@V2onttE0=e|8PrK~|c%uxDMmEy@Q-6u<%p83YV=9Z?So*e+etE&Z!pIA@QbKy@ZdiZ~)ZR zYj?Jn>2@DD&HSR&U-RSZJK8s7v7TaNf~|Q}zn7~SJ7wNAcUylYU42?5UA><&(BCA6 z8J{Kn$;VQ;XI0Ea(1Bl^YqQ{~4>V3Dx{3BvqIGWqQyJU{U;(Ahb~{-MlEokSB0M2^-iEFafWP>{q?qquQUs%`vHG`8veo!-iFSNHxT-(jH+yuKzc@@GE*1%Jq) z)(@@!Q9#D0Z2HzDqBmC&c+Rp?Tk>8<;NDBub_FaF?p5&>%b4&dfPq2U++n}}6X=zco+Eyb+w!wTuEY6$#1Z98*`FOe@)`8;t7 z!|ni?(ozuu^SO_}d|W1K@#Xl#LjhwrMP`ZE5@&%vGJTQwKfVSyO#+<01h1n-o!pi& z8dbDO-5vA&EDnZ%hPN0_yWbvVzc||XI6R!iti_Ik*+S3kU>4>7KgI%)ka%$a&dF7y zVyif|=en~+{bl*m^*WIK^XwOxIwey&!fO$P5rD~d_7PEp1ZWZZUsC>`po4IIbTd9gZJgtm>hjei&1h3lc6eA> z`=7ZXQhsNj|G6%ZZS?<45WEtyi2pyo!bN8OnP)fy;r@c~f8PBb^StEo_hg^^_P18XRR*QED)1i5|uJ({hl&pm5MDG7=B31X)3Cs{W$KJ(QUVUs|GbCbTMf} zOx^%=ORuc?&y?+9u!?V^m3FO6c0(41rLrV~vP=Cv)h7%^CL%x)3EM!ASpoaA}0|shjD@Amc9g`ioty?#7mO*iD^e#Z==~0wS&! zbGnjKchG5y*?DC^#O<&FWaDC)%3aN&c3XF4qD7xIX4p(Umq$Dq^4}+ajsV1-s(l$h z`E1;14x_ZkxZ#$+og~MQF**jgd@`Jw1$%>r#~~07hw>H*<=%ocgWIb9Xi*XVF`I#D zMopQ(ZAl}~Z3gc%Y}8GKS=?!1d&y+)`M!Q`$p>w>ALeutGU1uZ+UmCpi|v=vbOcR* zNRn0pZmcBqOPy(bhJA7b!oS&edhnNR zF?V%-t54_GCsdu1#0+;k4I#Ht|J^k#c@ezUp7+e-T~^b#_^b5l^d|sST56%5UQqZ2 zCedvGtfO3em66iTRI~7vhAGIZWWoU+5gY?An;tXzQPZzl$nKv3yVK$)FCem{^E~_L$erdnhs=8 z*)JB;`*e<;g8m`{D?=in<$>vc_oa%CRJu>vJ5mn8Lp8c&e}9T(2!!kPe-3qKuL39l zXG%pbgB}NWaKO3WoA^%R#1jNbv98q@Ste*dk@rwNww+xtFt-(jq zWY8XgZr=`sTWG zK=##ZLNxClD|lwDM6hMZ639c7J7C%sui3IgvBmn|x7UXPb;+UK_Tqxw(i>KOMW)7T zI+4~IFm?VdX9k@69Oj0`Fvc!x!;(1ukg_Kc#_heX@Gi^!RN-=kTivN$(8QORWV}4)!TJMcsu(R^+lbMtn_H z#?fUfSIp}PZyJn#islW7=1)tMH2H!bQMoMB#co=YENW&@NLd%+A+blH9*CW+88MM~ zC2!`G7v(>!TRPG^d>IprohIPYRTc+$1a?k0Ny(yVDyzy`z<>c?&6CAo-T)hQH4>fs zZJ@p7*Whnn$9!LV?9H_ymk3HK@^|cQ0tR zN9AUAFq>G-!M0Ka8J&iLSb6$J-6P&WxzNb;GI^{`lCuB0@w3WMA0t)O38G*G z8E)wLd8$5{p6%6lMk={`6O`65YnT8`5A;9Y0SQl8?Yyc+QsA*B99fj@!uo=`AoZK- zqH)ZJ?#$QbpJ8_bH@|kZwbP=w>q7+Hd{C>5f}`5*{UnSSEE}j@rwxQJM|9hrprZWq zc$QTasiFM6+8tMoBFKW0HE5FV;VKZ{Ljc_^+ISt0cT^PQuLnFnk<^WRlssoXB2gMO zGjMaenb{QWcql_2=eTdbQ$zkR2}~iqy+x|=B_bi6ps617;VyC4HpJ9BL+FxO8=n!X z|EM8{p2P`mtN(^s$Df`NdgmJq(-2vTyp4Leu`MH2Rw2jzHO;PjL;g^7vF^0!9VK#{Fx-Xcg*l*XDyCN$CCf_~865Yz} zLJ2NP52H(N;8SW-LNP4IIrn)sp0?ZNdi~~Kd#U$e_VO^dl>_0dZysWvI(-C&s@zCCsSQGt_yb$KFyR23Wg`wA@Bm8 zxlTqcCA?9*`gbMz$OBr&Oe6GgISTJ)%ZDSH=t!%+m(QOjcK?OLaG*T+`uYlP95c08 ze7w8hx5+6nO)kz&;Vq8ox1bkbD(>zECbcJ7MXMTHGf>%tG4N}`0hQ>|a?ljYl z5i1cfCMGL}ApzuBHEoGIxcGOD`f&Adi#u(*kyF6h`o;asa{-A{H5P zUuqPcH9@fBz8r9aLtbk8^spP}>!mA}R_@T-e(E6I$iA$%P=p6CwZy+Yi^FP6_61UWu z@yhiSF}ugA!gfI|G_u~1G$*LgS0<==BU;DO&tixO=W-3Zl~j(gW2_h#Z<6R{dg9LY zGe)ubUxA|cy$+tLY}{nLh|SX}SuuZ78Jg&*4&^D8D}I6#=_~2W-n2ZX&Srv`=Nj*^ z8WUl9OXSnB5ELX&WoZ-NTTCP~hmtYTqgn|((lePK6L;Z?0X-kb8N=UX%xTu0*?o~( zjI+C}f~}t|uZ_1mT@%ZMASx;!+7@$vL}CZ6Y0#@xkn+9%-4)mVmO_(vZhZSY?Q1ic zu3(zu?XBF$I5e+fvhXB*P4Ay;7BGy<3C?Q69IKlxZo4V)?T-YR0zBNv#IMORMD~q8 zw-mMZ&F-4lWl4RZosW#ib2lK_<#8ya`{d$p39HPc4+?5bdunQOEE+I+{=7Ai@Vf6; z0|N83c>f}R?X!KJtJ*jJhH3AOfy5HVVoQbo_Zf3h+&uP~Bu)~+%?WvU_Ogj;3spMq z+fb0Ry4n6CUA2cw#?H}3{#-ppBG`KMX^NyX%?YV5uON|Y`-`@|MyWg~CzM>0;>}E9 zQ{hDHpy;7!e4|;7imcU+e{MMoKe**-Hofa-vD#iNeXy#(XLBqJ5KP3l{FHx&VHl4; z?EK$7<{#*Pga*HLK>nu`hq`G7)a)bHNr)-#pV}!l69m7|N^6r4eDS2e$4$=sOyB&| z9{s1}3k1b>%zFokrGJfYKrr%W9B@>KI##bT%Rs_{sZB6hKWzL{S*A)6ve=MXtUniD zQiLK^5{nbHd8lDx(%F^utVrX2Fh1nP-vjkWb_6sdA?EwKOq*ZN?d^d?+?rx0_|v{{nHY@R&4amQJduKt~UW%i+^?_9ocsfp_*SttD4!*=B!^z0dPj1 z-rmO?dl2XTXi4HRKmK^rSUDnW#_KWJ#ojDUu=*#ro{|Vy^gSde@BUOq0=a~BEZRe} z7KK~(+OJ|)^PYgLOy(KI!TlqWhj?K3P$or+kX``m^vj4q|#U>aX=0Q_~+i!BC@7KOf~y z7=GSQc!6_PWOk>+&irGc61{g46uWiP@(m49)nfAlEJa_Z+j$KAd*Q}MQDt*q+k!-i zFo*>gKM{0}H-rYAx3Qf$s?7Ca=Im||7(etmPa*EIDK9np6#I@9?Wij@LF6;?bJTY; z$`4bFnqMkz%!RJaLpKMFX^)oIbu-R#@KG15pi@{dE%NrOM{_+(Mv1xNHlBa?ejn8@ zx?z2Q{J5J_9S)PwDb8GbdJ{X_)bf(vsj1H1f*TwMPv+8hp7ZfN`c+|5 zqo58{GzXo%RQ#|950FODUMFrcaB*O!CY9uQI>B^@!H}u#8ag&O0Sw~_YCS4Y)`Xx= z9#dB|)saLVtqEzx!>Iu6mJzDvq#d2z5g7){sH`rZHghy6BlvChUPIbI4CHE;MpiPk zFtc%)P6Aih=1?pVWPT{r-BM$?M))lDlccC>uqjdn%vNPVV$ch+o2qf0TE)SDm8Dfg z#B4yILMxpOg2p=LsOLsRu7K2QDI37bd|0NM)s=7h?4 zA*KX((?gkm-31Gf_^vvzcOJ!!dJS-*)-Obh*ZUNinM^BpUE=1xO5!yLdY#MPK*_i7 z$fM9isqc(>u4PmLBt-wqN5gFj0IYa3b8&#hWhVJ76_4EfT}pv#?>{x%?j##AO>+JZJ4si;H|X=BPX=M9t=P@^;rt zN&Bb?GcuyGFRstsZi>t}14KFBFdNaXYn)tm7EH=3E!RG^!|l9V@DSI2ku%}U#`I-} zhb}&tpbfgliW$FW4FYGZl#Wd#ZlR2A<5H{mbO^weB50RtI)atb9}F_WbQ;%UI*s>o z`D6)1XbdbhvtP13(O28vwsE&BZD&W3%HbbVuCdy9j+ok$*vA!tH;u4!p3$#B6K%m% zJoKgw!`%t&KXZ-eIYoI+*1~XjHRV#E4P$z^4*UF479DlJXjiT%pG4tLvE z$4s7WW(VveVDjB89nDCyv9U!yZ)`FfEVi1Q67Al3P9^o=@vfTNafj$=Veu~;tI(N& z2m1y7{@Qn9>s@p5s*V{qjQ>Lv$mV{q8e35j%V%*<>&1;@-492X;BLETgb}%4#`~w6 zaqWBM)U7K8AATZo{ipKJ|3$aV|E#f7yn_K9z=1qIt`q*vu-Ydffks_)3-J$N;Rsq; znV!{46FbyWS6Nq6%S)TkMhO3Oxmm*3Y-e3oAxIo%#Y8pv>c(gru1WysXcaVTjRvh z*KX~wu%|e!n$C$8UW5SPuRoF_g6%MnFv}@!oN{JTidJ9Ca9y7M76eEoTD&)+G2`Cn zts;MBFWVzIcud1*R39IMqv=|s>pJHtbxQl!gcHUawvT(1VesWq-8uA7902Bd<9V<@ zY|x~prd3N(xw29dQ)DJR&`Rcjv61kX~H-mSzm%C6I_c)>jPm9J;g*xB6mJ34U@#7 z!i4|yPJWR)ds8#y)o(x%DRK!6$mktNFT7dFyuoddBztL(OP95Jfr&DNnk9{~u=mFR z@?#<}fZQz{3t&{XCq4qS+LLE$_tZK))VLv@_QxF5GQ~oa-U0n)e6a-^U8Vh;95GWP zBVAr52BjoCZP8FO`;B58sNi90OG!W-nl})Y?g9yMkFn}vU44Aw@-qO{t5?g${APRA zY)%YR-988&v$wiXj0-Gxq#hq%x(hnbiT~6#G{uo+dqMo#L6i~d+I>JP-sgaP&^qST z{Twt%xXGQB!obAO4~Tql{#m!l%0qH`j0LUbx$~A==%~EzDl*fU`U)yfJW_wUw4hF; ze-;o8jo&M5Hp}w?pwA4jlE`yJ3?@EfO1yvVZVe#QQua8OC9_vlKN_KEZH{`? zm80`ycVkFhpgiRiX70fHc+DHs^9S?6jB<`lP@`oxXSqfHT#W>%JBbS)f7kV*a9?<8 z*znkF8jogh4jJlyp#u>Y3}1T8rxo^HmU1ak_1s>N;?D5q0W{}~DV z2sQs{KYl}7{TU(thTisfSK&Y7*8j5J&xMR6B4a-VVc-Sdyo`*^eJJW>&M&Nt@t-YO&JER2XST05n>f~4HG83sY{l35X%wcsU_tV^@`c035 zS8_6Pckg(<^_C-g|7n_-aPn!%rDnZeS0(Ag)w)$j+NJ3Wh%qluDqI||7hZV5-1PY0 zFP+2VRKm8VBeW;H-Q#zQ!QV6X3McalI#s_@ZFph8rIWw7YHEi3oQUJ00%miZDa6F^qS+f6a-uH%;L*V|X)BUAna%p9iKEv6*|HoOWNxM%&T zuEMD+N@X{Dt>sRSRjq&a#wUWhAbuIn_EZIi%lPvaeO#0-LUkDq0KD zb(<9H?;jdEzXS!00|xF=t%BZkgBM=R6>7YWEaQ~}#@AMq1vDCw zd;yY&W(;qWz@e+brsY`Mhv^(0y8bI?K8Kv`ED`({q$0r2xyeZfp7IJ_?Pl zCJnu?v47CbS@NN$oIh#xD|hN06)NG8$L$pTDhs*BN(S@VW95s1R#n$?tA{9r-V1tk zFc?g?$?+xza0>R8bW$XJz~xdu565%j+PoMj9Ihy-tHZ^(F~*krLasp*zrUD&@+&VZ zD+)HE`*3Hvz5Wx0S%%hB0GHx7YeD3j4?Sf?g1TBJcGHhn?2tt(E2{X0Nl*l{(&wa& zUe;B+G&qaq;y`atK2nf_6L4?HXQ-h9>Ux5095iTd5;?&Q<1Iy7iRWJj2?broi(P_s zeMi3Bv`P3_FFC`5@^fLiJ^QUrUGt)zoBa*FjH|?S?i5ZouOdjd9T$)YR7bk63U?g7 zkJ~#~b(Qu<;k^k2a~i(|r4z}+9H*MZR(Uh|8j)dV(ojO*Z?KGX@}PG#6APkSA1H(c zR-VSC+6uGw;`g0fkC1$DZ>|-Y5-ut$GMJ$?fmUpNH3#FQG!T;KmpbTF zqQWMi6g#Be?)l1Y@60*D^Bl*CZ4)vL_eV&SHMyO+LV)a4c9}!&Y=%@^TbnNq?Y88s zTzgnF@+l}p)L;1Y1+vRRLe5uI`t9-zXun1gJNcv+@{Er2nZLNFJy~o@+ECc7i60Pf zWf3~xXWzIBQ`x0nw@WjM$QriqZrF@HLl9h1oWkQ+Gm>=ueX^b8-ke5mps*LwCa?2a z1I)T?y9)L4GH=LbiJ?XjNEu=j`EudcL!fm5_y$)z>LcT6x48?Y0~9rM-qlQZv8}Lx z3E5Ecq~xEwF|s@s+8v08M~@KQH8<$WMDJ-81WvXnY1j(x_gps^t*xh$uBj5pU*Ni?Snv&Ol&n=09dUmE8PPILkr^;loF(-FA zYVlupj%gaHd<~F7)vENaUXYI;IQhnf1&%kv`BS0+2VnBmJ1<wgblf>_*oEMIl`1aUw2|`FPo7X_qpCKfM zem%+VX(TM5@)J&?Hr}6S%S}cB-smnL$5Nyz%bY$sxca+VG_4?!NLQSog$zQMVR8;X zmF54;ZyyjZ#%un*#4#ixUd(a5tnHk!<4cgn1;1CF-vL*1>2kF9Rz7j;=9uP?lzDo? zd({V7_75J6Z4MP6oaqr+SGl~cb6&M^)8=F46nJ^R+40ejpJM8k1NZuJPp9;k8 zIT>2{w8Y$YXcR$G-}q#_igZj(jo>vHAS6l@|A*-EI%lBt7q3aw58DuUE+8FQQp$g7 z>-CGgPMAA24i7KnWEW1h(3!E)2D48&0siRkr6dx}p2uHd9|UjA?j4VFxWQGLq7acx z1jiHdA(WR|z-p@`;qtNjwYK0v=W~R8H2xMRHEr8#Mr$+ZxAaS##+6S(yvfok>)e=XpqB?HM4yHFW~RRgETC;{ zZobE)@Ar(p6=|M!$y?civF42kUwpdBuM1VgKkveDE(eWQ5s&=P56o}CdwBDN?J9cz z&|=A+cwm7YatlHiexAM)tU#t8sQOoL4(R^;?ahp}ElD>u<58gtKmuW4&T_%#uSPK43nOWD$Cqgb%rO_%b7C z2`uTn59SDV>le6-eEz9~TNShxe81ELrg1CNF;2%se+QxhE zR84a{Fb2`#OF4{yDM27{>C0}IOP6jEh;F@!CcU==gS@l@Uml#~Lee<0K=hs=PT){E zvdn*gVr_u~BHr@>YJv^N`_TOz38|?oU#!Y9I!#_d0%4#5Urtf{S3imOd>^#^`yIan zWU7-~r@x#17oW>B9>hOXLZzP!=wKYEF#DlAm90~(7}p6I`dB{I_Y`sB{DtyRtnWGt zrBk%_iYLV-z(R?noHMXao*BUZ&bG6;7=k#l1c>Owz2vwUpQK7cswD1cSCDuDPH6}c zAbt}CZM)w&99cY1eS|($0-S0zAAY{^U;hToFJKRNeqL-3fy)am!QNbsc40}zMDtlh4?Id!O3005^fy;-vA>kth`aikT<=aaPljv= zt%-?=LNrf9LqjP5vFegvCe$n*%Z$Coxp|kLW{J9j1ci0<6rMgWam;vzP4Hx<19^*d zdwY9SIVY5`Y<^&3VnW1a!+fA*8N=eGtzBjd`Y+ft@h{$x${{B>D3cv@rULxe=_egG z?;Wj{6K?w1pEPMYj8#fvuF5EMz-_8EVHmKBbt(!e2;}(ExeG+Xv>xim!|vT<$Di%a z8wQ+6fF}}QqLY5x`L>oEBH9O7nhFaG0W(*{N+Vmo+!g>0m?R-3MW&?%G_}0G{>jdL zg+27jiVsV8ltx_$os~eK=l9vXYWMF?vA+th-m#W$uPHR?lnQ3IL5DQ+Uyte7pv|Fq zT-ulzZF$+WI46kB_Xdr9eC-J!a602*r#w&jl{Mkh2&$1(ZjA)O@t)xpajY?z^>AHy_qb#JIo}_33jCe ziY2a}t|;%wVjqenZeD-BVF3x4p#VC=v!oG=H%d%jASf%&>`1&x49 zA~8-Th|&+QhA#cI`uNmTK!=jpzFG>H%w_MOmnAD*lO@~ld+u+!`Dos=Nj+Pt(kFe8 z%w#tC#%$5UK9hLVi%!&O&%4TTByhY-4`TUJsH`ef#>Lj!Bv+IfY5~Ut@p$6E>6S_@ zhS~?(+S+r-8yrsaMcrrcsb5D#F!J5kqU;K1X3{2z0idS@3f?YnH3R&2xnltHoj+`Z zK&mW1zvFS$h1taYB*Ee2bvr~>t>f9oM3q+vn`kE9Q677&OJr(HpimDflInd~n|t;B zouaS@FYc7CHb|y1;k^rQEM(y4*Bocp%9pwu2^i$G*#wXgpMXSAREv_UD1Xi8wj2;p z83`6zP(GUp5SF>`B|xKib9^@f(5@lZ;f!@U%J|6TFs>*+qj}?-Ew3m?xJ5nt-UzY` ziD1jUPG5{~jp0{%h!2=#qX9#=crsdHM6!Cpi(J=EMN6yqTaJ$Jxr_ApbWYT8i z@{|B}S~QOd=3EI@fG4M(rJ?J({5jtaz$?pi?ZS+PYl|L+LOu5Ef311K_qmeDdieCG z*Vhf^jusvY_XjMF8B1|EM>@|Bjdb2z;JA_Cx^mlWvc~7EbSksp+AJqCyC#?{?dcH;|(?bwiP7iljJ3=GuB(FW{!=3rt| zB}>|HxXKN%9+FrlgYnKJDnK%JQP}Y<`wn0hTPT~&t*8)IVX%~PyfCr2)hv_kdrPDDN}?`k*NcJET?2a7T&G{6 zY~sHo-{Ai+FBsLDkrcTKzWt?L9a-c^i(;8DDr{>AV?LffXP87nVE7p2089riAazUt z|1SX}jM42~)gFiaSgt;=3xORY_z1;fc(De99BFq_jCR)oPeFu`3fyD?$G|K9aEZ7p zSKkVzVO2bHXe~OjNAv-xYq7ONXK-j;nY^9=^WX}CtV`xUmj8b+SHD@f{|{#IhdDm8 z7;pB=HedPx)_Z9N^W|TE@F=V{DZA!KW=vyzDFD;aUI}MW$6O2kR zTED!+KYbZ&*4vN34mWsXPxB21TSh9 zSgR|NR!ygXrwOS!11uxaDy+ZQ%BnLpXv^Atqt{qAm>NSSi8)CDVZ|(?$UCE^e7TJo zHRN$9!1#IOVaV#kK|{cG0F$n6n{|!ybyQX)95~&KM5D@+)dxGX;P?X`JT`WA zSMa8WlYl>PvHI{Jm`3RRqxR8gv)t{Cek;Q&zXuC<}iwG0i1Q$nV5{f#@=M2wj2<()JWo6KoO^5 zIW&g!B@nlBnl7e9y(s3!(B2aAEy4OiUn7X9`T>>#YJjy_S9uOWe(&+eA=a-#ixn)Y zsr7qvb8{0-2#-TRm||$C33oy^$MTwe+n!1Q*k_gxRCB9#&@on3T9Z20b@&Km11a8N z7&dPYyDV0M$Aj2@$3t2~0Q`b`_lAl(jBlf|gw7+DvY(wf-&blob+EttC4`QP+pvJEYD+&$Bj*6{ zp*Gz~s4(+gw(NF{ilnGZfaufa<%RexQ6 zf9h*ov3W0jXAHm9*QTadGaK{$tzbwzDM^A>9uE)iz2zWlXGBB4P(XqyuG6!zv5AQtA_1RJls*>^J1S345Ay(?hy@}u z(%rxyJi!?P+JPkLvrhV8?;aksB}yVVby_fL1~U2=WqH8B2iU2{{oc=H;Rfm%r;D7DmKd zjAyH9>zs6H8#pf-d2oyO!$oG@;9=io>&ei}i(hXu;SWNx_;p~ne*#>UJKsM(6Fb}k z+*7{IXY}0GyWID8{G`B%5Zjr)5;bE-+AassyzhAI%%pL;RM^<3jS#uBp;cqSIRnn7 zVO`)wqf}KyxOD+hk#$d)1g0_Mu3M}Ux1UsepX(dbxJ+An`5 zZ=$!=ET7FSZ|2tpnWrQIgP-#uZ-n5l69FYj76)6MK0_Cby zo76Gs%8W2kRqT1Mw0bQvHEHA4Kzm~Zo5$9OFQ9`Aq7~`ua|eoi@@pJhG_P%J?hU7v zall#Y3inATv=0e!2Dib=$w|xe=;d_S{(KUEqr|P&kJe7(WEnGJ{8MN<3HXP;dV|OS zC~S{Yo3$0;;&ZNmfY%%VyvWGZd19SxZq01gRdDp7vMfndAf<_PW2CnD& z!XT#U`o=0?_N~b#At6agOG|Lu=mqB_Y+J*qC7O{_bBv4@Bff8qS|}eV0L;(um>4$g zx9_VTkJRj}&sGDIY82e5?oCR?Z+Xwi$Os%0I+wY3=`OCAgV_Df;LBCT*q&Tnp^-AF zXE!>{HF`n;`X?apUG%p7-#wTgTAg7I2@%J}#8}S&Y7mG@JoA82^gdq^{+~@7CJmAF zgdC#Cn19Ed3=in6`5AM(=vUFbSA*ijP2pb68C@#M4~v{jv0Nx z694&A)f!B@vi&*Y#EY{z-?$(Qe#$v^dd@642OUJ?(quBymu{q?T!~vx|xTn*iS;7Wf(*`?X5djjd;3788}brLyu14 zP-N?QUL!oZWz1UM-~tB;0`aT`;sCxg0EY^EyJT%T3mS`c`0P!W121yV!icX9JZ!y_ z8Sgmi2h2wXKf!m~+HCi|ud~;2isF!yVyAVw}T+Jb!WL=PNy=ZlB8fjP~TgnZYjb*9jbNvmZc zIU0636{tZ6C4lhoX+!8-is<&kFJ7n5qAo+0I94Y`CWo&@drG}!Wr@lSJ|TX0+2vH5 zsdRTE6d2?#cNeU0%~9N4m`i~ZQLMCbKda;}qWir;69gaw)@zZiQv0gxSvL98bQP%)=ZX9ujAkF{r z%csSiSUz2wi=qW*>=wOGC{Hi6>-N#7ysE+Z9syQ-dkvYOe-f+s5fyygZ%_Y@tx*Yu16gyCbQp+&(XTJ=No3tcfr`) zH0+2f)l^hAFLKk)e9Q5zWQHSIS6DH(_BbSkwBcS&hIaeWCyxtpVbikl`(#Dy}iSO^*mi81#SS*3~d#hrB=Dj zmerN6g0Si_`>8(bb*si^QD8-u`kygRR2>yvq#g?D)LDZ&O)W zxzJ|fux}MvyU1E2*AT&ZK$G4#$Y)?tdr5lTwOiC!WWh|Z(2wPv){;cr{gu_ytidOT(xh&a$}V=B65LLj27Hb z#omgJQCx`FPKy`#Yme5^{LM-0vrwTYbLR2u=&Z$P=_ez|OL8Sr9S^xiyfxE{ zE|@767_~?#AW@ud^jzYN)_xF>`64|#9M5I{_2xbp?Rmwfo9#S1X*MX+Dwq}W;^buF zTa6Wy;%t*iwws)xQlX-3f-G?P!0z?#VSQTvVF3{J=H1|^n+EPUcR;ODmeVTPJCdkv ziN)$_r|C9me!e!PI?tmv@r$j9Hl*QEgY3kayND_!tS&wNTqgj3}cL9!G>x^Mz8^mm>ib`i+uv`FjL26Xqc!|`n zjXcXmZ5r#p-H_EpQA)1Wzu0Z%i4#Y~7d}6=5m!1Yz_O6>A3sHIRacV6D!&2K?i=np}y6h>tQ*8Gx((cA;O_GqIFBE&tz3(Hd}P^%DjK(>-+9? zQ~9yF^nUbi8ikv1`8^o{Q`Cg~F^tFjNI73gdXEp*`g{ zI>qs$tl&6-_)-^m&hui@F5bMhk?dNG)C+UHAs`2#Gj|bq{l&Je`=DvlAEO3i=}l`-ec>)Y~>%Uw;T?p9`0|8cJ76l5sW9W50YalJe6^aquYiHQUK#BN{{!Q@o?h zI7SYG*FhGqtI{P8B!~hoS8Wbq&xuJRis?dZ0C@(MtFGC~r#C_?blfF>o_6AZHcB33Ebb*p} z*zfyq53W&n<7qHb4pdLTL6(Ivm1e6V=oM0qtfQYaq+BC+PwzqL+%_)ThFsF$CbX75 zqX|S>9a29m+1*nwDS6Q=fR^5)|NVZJMkV}ij|C9%IeQ|FxlpMD(jbJtBu4c7q+tkq zG%Uq$%?}7X(>t6?ye=CCy5(f4NaVys+)YDyzZFuZBDJFz>2|{#yD{b~U4dae?*a%kl|-*nZ4;yWlBDnfKpa&SN`y#~ugb+Jwre3i8^f1Z2cup>|iTd@D!v Apptheus: Socket creation & Metrics endpoints expose + +== Apptainer Container Creation == + + activate Apptainer #Green + Apptainer -> Apptheus: SO_PEERCRED verification request via socket + Apptheus -> Apptheus: Retrieve the caller PID info \nand verifies caller info against\n --trust.path + + alt #Pink Failure + Apptheus ->x Apptainer: Connection refused + else #Cyan Success + Apptheus -> Apptainer: Connection established + Apptainer -> Apptainer: Save the socket connection + + Group Cgroup manipulation + Apptheus -> Cgroup: Create sub-group under \nmetric-gateway group + Apptheus -> Cgroup: Request to move caller process \ninto the newly created sub-group + note left: the caller \nactually is \nstarter (starter-suid) + Apptheus -> Apptheus: Launch a monitor goroutine + loop every 500ms (configurable) + Apptheus -> Cgroup: Check whether there are any processes running + Apptheus <- Cgroup: PIDs data + Apptheus -> Apptheus: If no processes are live, monitor routine \nwill exit (close the socket connection) + Apptheus -> Cgroup: Retrieve sub-group stats + Apptheus <- Cgroup: Stats data + Apptheus -> Apptheus: Push metrics into storage + end + end + end + + +== Apptainer Container Cleanup == + + Apptainer -> Apptainer: Close the saved socket connection + deactivate Apptainer + + deactivate Apptheus +@enduml diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7086478 --- /dev/null +++ b/go.mod @@ -0,0 +1,46 @@ +module github.com/jasonyangshadow/apptheus + +go 1.21 + +require ( + github.com/alecthomas/kingpin/v2 v2.3.2 + github.com/go-kit/log v0.2.1 + github.com/golang/protobuf v1.5.3 + github.com/golang/snappy v0.0.4 + github.com/opencontainers/runc v1.1.9 + github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 + github.com/prometheus/common v0.44.0 + github.com/prometheus/exporter-toolkit v0.10.0 + golang.org/x/sys v0.11.0 + toolman.org/net/peercred v0.6.1 +) + +require ( + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cilium/ebpf v0.7.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/godbus/dbus/v5 v5.0.6 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/moby/sys/mountinfo v0.5.0 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect + golang.org/x/crypto v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a19eaec --- /dev/null +++ b/go.sum @@ -0,0 +1,124 @@ +github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM= +github.com/opencontainers/runc v1.1.9/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/exporter-toolkit v0.10.0 h1:yOAzZTi4M22ZzVxD+fhy1URTuNRj/36uQJJ5S8IPza8= +github.com/prometheus/exporter-toolkit v0.10.0/go.mod h1:+sVFzuvV5JDyw+Ih6p3zFxZNVnKQa3x5qPmDSiPu4ZY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +toolman.org/net/peercred v0.6.1 h1:xAjw6yxNJRO2asnmqMPfbzOwKpb1wUJF3iKoTvqH0zk= +toolman.org/net/peercred v0.6.1/go.mod h1:soGaSNwoDm9E75fpgElzOOMDapKLnrwWeDdMUKbsUmo= diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go new file mode 100644 index 0000000..6d754e1 --- /dev/null +++ b/internal/cgroup/cgroup.go @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 +package cgroup + +import ( + "bytes" + "fmt" + + "github.com/jasonyangshadow/apptheus/internal/cgroup/parser" + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/manager" + "github.com/opencontainers/runc/libcontainer/configs" +) + +const gateway = "metric_gateway" + +type CGroup struct { + cgroups.Manager +} + +func NewCGroup(path string) (*CGroup, error) { + cg := &configs.Cgroup{Resources: &configs.Resources{}} + cg.Path = fmt.Sprintf("/%s/%s", gateway, path) + mgr, err := manager.New(cg) + if err != nil { + return nil, err + } + return &CGroup{Manager: mgr}, nil +} + +func (c *CGroup) HasProcess() (bool, error) { + pids, err := c.GetPids() + return len(pids) != 0, err +} + +func (c *CGroup) CreateStats() ([]parser.StatFunc, error) { + stat, err := c.Manager.GetStats() + if err != nil { + return nil, err + } + + statManager := &parser.StatManager{Stats: stat} + statManager.WithCPU().WithMemory().WithMemorySwap().WithMemoryKernel().WithPid() + return statManager.All(), nil +} + +func (c *CGroup) Marshal(buffer *bytes.Buffer) (*bytes.Buffer, error) { + stats, err := c.CreateStats() + if err != nil { + return nil, err + } + + // write stats + for _, stat := range stats { + key, val := stat() + fmt.Fprintf(buffer, "%s %f\n", key, val) + } + + return buffer, nil +} diff --git a/internal/cgroup/parser/parser.go b/internal/cgroup/parser/parser.go new file mode 100644 index 0000000..dc8c978 --- /dev/null +++ b/internal/cgroup/parser/parser.go @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package parser + +import ( + "bytes" + + "github.com/opencontainers/runc/libcontainer/cgroups" +) + +type Marshal interface { + Marshal(*bytes.Buffer) (*bytes.Buffer, error) +} + +type StatManager struct { + funcs []StatFunc + *cgroups.Stats +} + +func (s *StatManager) add(fc StatFunc) { + s.funcs = append(s.funcs, fc) +} + +func (s *StatManager) WithCPU() *StatManager { + s.add(func() (string, float64) { + return "cpu_usage", float64(s.CpuStats.CpuUsage.TotalUsage) + }) + return s +} + +func (s *StatManager) WithMemory() *StatManager { + s.add(func() (string, float64) { + return "memory_usage", float64(s.MemoryStats.Usage.Usage) + }) + return s +} + +func (s *StatManager) WithMemorySwap() *StatManager { + s.add(func() (string, float64) { + return "memory_swap_usage", float64(s.MemoryStats.SwapUsage.Usage) + }) + return s +} + +func (s *StatManager) WithMemoryKernel() *StatManager { + s.add(func() (string, float64) { + return "memory_kernel_usage", float64(s.MemoryStats.KernelUsage.Usage) + }) + return s +} + +func (s *StatManager) WithPid() *StatManager { + s.add(func() (string, float64) { + return "pid_usage", float64(s.PidsStats.Current) + }) + return s +} + +func (s *StatManager) All() []StatFunc { + return s.funcs +} + +type StatFunc func() (string, float64) + +type Stat interface { + CreateStats() ([]StatFunc, error) +} + +type ContainerInfo struct { + FullPath string + Pid uint64 + Exe string + ID string +} diff --git a/internal/monitor/instance.go b/internal/monitor/instance.go new file mode 100644 index 0000000..be74be6 --- /dev/null +++ b/internal/monitor/instance.go @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 +package monitor + +import ( + "bytes" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/jasonyangshadow/apptheus/internal/cgroup" + "github.com/jasonyangshadow/apptheus/internal/cgroup/parser" + "github.com/jasonyangshadow/apptheus/internal/push" + "github.com/jasonyangshadow/apptheus/storage" +) + +type Instance struct { + *cgroup.CGroup + ticker *time.Ticker + + ErrCh chan error + Done chan struct{} +} + +func New(ticker *time.Ticker) *Instance { + ins := &Instance{} + ins.ticker = ticker + ins.ErrCh = make(chan error, 1) + ins.Done = make(chan struct{}, 1) + return ins +} + +func (i *Instance) Start(container *parser.ContainerInfo, ms storage.MetricStore, logger log.Logger) { + defer i.ticker.Stop() + + c, err := cgroup.NewCGroup(container.ID) + if err != nil { + level.Error(logger).Log("msg", "while validating cgroup info", "err", err, "container id", container.ID) + i.ErrCh <- err + return + } + i.CGroup = c + + err = i.Apply(int(container.Pid)) + if err != nil { + level.Error(logger).Log("msg", "while adding proc to cgroup info", "err", err, "container id", container.ID) + i.ErrCh <- err + return + } + + defer i.Destroy() + + var buffer bytes.Buffer + labels := make(map[string]string) + labels["job"] = container.ID + + for range i.ticker.C { + ok, err := i.HasProcess() + if err != nil { + level.Error(logger).Log("msg", "while verifying if there are any processes inside current cgroup", "err", err, "container id", container.ID) + i.ErrCh <- err + return + } + + // No processes left in the current cgroup + if !ok { + level.Info(logger).Log("msg", "no processes in current cgroup, exit", "container id", container.ID) + // also need to remove the related job metrics + ms.SubmitWriteRequest(storage.WriteRequest{ + Labels: labels, + Timestamp: time.Now(), + }) + i.Done <- struct{}{} + return + } + + buffer.Reset() + buffer, err := i.Marshal(&buffer) + if err != nil { + level.Error(logger).Log("msg", "while marshing the stat info", "err", err, "container id", container.ID) + i.ErrCh <- err + return + } + data := buffer.Bytes() + + // send request to pushgate + err = push.Push(ms, data, labels) + if err != nil { + level.Error(logger).Log("msg", "while pushing data to pushgateway", "err", err, "container id", container.ID) + i.ErrCh <- err + return + } + } +} diff --git a/internal/network/wrapper.go b/internal/network/wrapper.go new file mode 100644 index 0000000..d2b065e --- /dev/null +++ b/internal/network/wrapper.go @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 +package network + +import ( + "errors" + "fmt" + "net" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/jasonyangshadow/apptheus/internal/cgroup/parser" + "github.com/jasonyangshadow/apptheus/internal/monitor" + "github.com/jasonyangshadow/apptheus/storage" + "github.com/prometheus/exporter-toolkit/web" + "golang.org/x/sys/unix" + "toolman.org/net/peercred" +) + +type ServerOption struct { + Server *http.Server + WebConfig *web.FlagConfig + MetricStore storage.MetricStore + Logger log.Logger + SocketPath string + TrustedPath string + Interval *time.Ticker + ErrCh chan error +} + +type WrappedInstance struct { + *parser.ContainerInfo + *monitor.Instance + net.Conn + Err error +} + +type WrappedListener struct { + *peercred.Listener + TrustedPath string + Option *ServerOption + ErrCh chan *WrappedInstance + DoneCh chan *WrappedInstance +} + +func (l *WrappedListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + pid := conn.(*peercred.Conn).Ucred.Pid + + dirfd, err := unix.Open(fmt.Sprintf("/proc/%d", pid), unix.O_DIRECTORY, 0) + if err != nil { + return nil, err + } + + pidfd, err := unix.PidfdOpen(int(pid), 0) + if err != nil { + if !errors.Is(err, errors.ErrUnsupported) { + return nil, err + } + level.Warn(l.Option.Logger).Log("alert", "host kernel does not support pidfd_open, silently ignored") + } + + if err == nil { + if err = unix.PidfdSendSignal(pidfd, 0, nil, 0); err != nil { + return nil, err + } + } + + buf := make([]byte, 4096) + n, err := unix.Readlinkat(dirfd, "exe", buf) + if err != nil { + return nil, err + } + link := string(buf[:n]) + + exe := filepath.Base(link) + verify := false + for _, path := range strings.Split(l.TrustedPath, ";") { + if strings.TrimSpace(link) == strings.TrimSpace(path) { + verify = true + } + } + + if !verify { + if conn != nil { + conn.Close() + } + level.Error(l.Option.Logger).Log("msg", fmt.Sprintf("%s is not trusted, connection rejected", link)) + return conn, nil + } + + // container and monitor instance info + container := &parser.ContainerInfo{ + FullPath: link, + Pid: uint64(pid), + Exe: exe, + ID: fmt.Sprintf("%s_%d", exe, pid), + } + instance := monitor.New(l.Option.Interval) + + // save the container info for further usage + wrappedInstance := &WrappedInstance{ + ContainerInfo: container, + Instance: instance, + Conn: conn, + } + + // fire monitor thread + go instance.Start(container, l.Option.MetricStore, l.Option.Logger) + + // fire a goroutine to retrieve the error or done message + go func() { + select { + case err := <-instance.ErrCh: + wrappedInstance.Err = err + l.ErrCh <- wrappedInstance + return + case <-instance.Done: + l.DoneCh <- wrappedInstance + return + } + }() + + level.Info(l.Option.Logger).Log("msg", "New connection established", "container id", container.ID, "container pid", container.Pid, "container full path", container.FullPath) + + return conn, nil +} diff --git a/internal/push/push.go b/internal/push/push.go new file mode 100644 index 0000000..fe3cade --- /dev/null +++ b/internal/push/push.go @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 +package push + +import ( + "bytes" + "errors" + "time" + + "github.com/jasonyangshadow/apptheus/storage" + "github.com/prometheus/common/expfmt" +) + +func Push(ms storage.MetricStore, data []byte, labels map[string]string) error { + if _, ok := labels["job"]; !ok { + return errors.New("job should be set in labels") + } + + var parser expfmt.TextParser + metricFamilies, err := parser.TextToMetricFamilies(bytes.NewReader(data)) + if err != nil { + return err + } + + errCh := make(chan error, 1) + ms.SubmitWriteRequest(storage.WriteRequest{ + Labels: labels, + Timestamp: time.Now(), + MetricFamilies: metricFamilies, + Replace: false, + Done: errCh, + }) + + for err := range errCh { + return err + } + return nil +} diff --git a/internal/util/util.go b/internal/util/util.go new file mode 100644 index 0000000..00dba9c --- /dev/null +++ b/internal/util/util.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 +package util + +import ( + "os/user" +) + +func IsRoot() (bool, error) { + u, err := user.Current() + if err != nil { + return false, err + } + + return u.Username == "root", nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..59108db --- /dev/null +++ b/main.go @@ -0,0 +1,335 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +// Copyright 2014 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/signal" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/alecthomas/kingpin/v2" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/golang/snappy" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/promlog" + "github.com/prometheus/common/route" + "github.com/prometheus/common/version" + "github.com/prometheus/exporter-toolkit/web" + webflag "github.com/prometheus/exporter-toolkit/web/kingpinflag" + + dto "github.com/prometheus/client_model/go" + promlogflag "github.com/prometheus/common/promlog/flag" + + "github.com/jasonyangshadow/apptheus/internal/network" + "github.com/jasonyangshadow/apptheus/internal/util" + "github.com/jasonyangshadow/apptheus/storage" + "toolman.org/net/peercred" +) + +const ( + VERSION = "0.1.0" +) + +func init() { + prometheus.MustRegister(version.NewCollector("apptheus")) +} + +// logFunc in an adaptor to plug gokit logging into promhttp.HandlerOpts. +type logFunc func(...interface{}) error + +func (lf logFunc) Println(v ...interface{}) { + lf("msg", fmt.Sprintln(v...)) +} + +func main() { + var ( + app = kingpin.New(filepath.Base(os.Args[0]), "The Apptheus") + webConfig = webflag.AddFlags(app, ":9091") + metricsPath = app.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String() + externalURL = app.Flag("web.external-url", "The URL under which the Apptheus is externally reachable.").Default("").URL() + routePrefix = app.Flag("web.route-prefix", "Prefix for the internal routes of web endpoints. Defaults to the path of --web.external-url.").Default("").String() + persistenceFile = app.Flag("persistence.file", "File to persist metrics. If empty, metrics are only kept in memory.").Default("").String() + persistenceInterval = app.Flag("persistence.interval", "The minimum interval at which to write out the persistence file.").Default("5m").Duration() + promlogConfig = promlog.Config{} + socketPath = app.Flag("socket.path", "Socket path for communication.").Default("/run/apptheus/gateway.sock").String() + trustedPath = app.Flag("trust.path", "Multiple trusted apptainer starter paths, use ';' to separate multiple entries").Default("").String() + monitorInterval = app.Flag("monitor.inverval", "The internval for sending system status.").Default("0.5s").Duration() + ) + promlogflag.AddFlags(app, &promlogConfig) + version.Version = VERSION + app.Version(version.Print("apptheus")) + app.HelpFlag.Short('h') + kingpin.MustParse(app.Parse(os.Args[1:])) + logger := promlog.New(&promlogConfig) + + *routePrefix = computeRoutePrefix(*routePrefix, *externalURL) + level.Info(logger).Log("msg", "starting apptheus", "version", version.Info()) + + // verify the caller is root or not + isRoot, err := util.IsRoot() + if err != nil { + level.Error(logger).Log("msg", "Could not verify the caller", "err", err) + os.Exit(-1) + } + + if !isRoot { + level.Info(logger).Log("msg", "Please launch using root user", "version", version.Info()) + os.Exit(-1) + } + + // flags is used to show command line flags on the status page. + // Kingpin default flags are excluded as they would be confusing. + flags := map[string]string{} + boilerplateFlags := kingpin.New("", "").Version("") + for _, f := range app.Model().Flags { + if boilerplateFlags.GetFlag(f.Name) == nil { + flags[f.Name] = f.Value.String() + } + } + + ms := storage.NewDiskMetricStore(*persistenceFile, *persistenceInterval, prometheus.DefaultGatherer, logger) + + // Create a Gatherer combining the DefaultGatherer and the metrics from the metric store. + g := prometheus.Gatherers{ + prometheus.DefaultGatherer, + prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { return ms.GetMetricFamilies(), nil }), + } + + // server error channel + errCh := make(chan error, 2) + + // verification server route + verifyRoute := route.New() + vmux := http.NewServeMux() + vmux.Handle("/", decodeRequest(verifyRoute)) + verifyServer := &http.Server{Handler: vmux, ReadHeaderTimeout: time.Second} + + // create necessary parent folder for socket path + parentFolder := path.Dir(*socketPath) + if _, err := os.Stat(parentFolder); os.IsNotExist(err) { + err := os.MkdirAll(parentFolder, 0o755) + if err != nil { + level.Error(logger).Log("msg", "Failed to create parent folder", "err", err) + } + } + + verificationOption := &network.ServerOption{ + Server: verifyServer, + WebConfig: webConfig, + MetricStore: ms, + Logger: logger, + SocketPath: *socketPath, + TrustedPath: *trustedPath, + Interval: time.NewTicker(*monitorInterval), + ErrCh: errCh, + } + go startVerificationServer(verificationOption) + + // metrics server + metricsRoute := route.New() + mmux := http.NewServeMux() + metricsRoute.Get( + path.Join(*routePrefix, *metricsPath), + promhttp.HandlerFor(g, promhttp.HandlerOpts{ + ErrorLog: logFunc(level.Error(logger).Log), + }).ServeHTTP, + ) + mmux.Handle("/", decodeRequest(metricsRoute)) + metricServer := &http.Server{Handler: mmux, ReadHeaderTimeout: time.Second} + + metricOption := &network.ServerOption{ + Server: metricServer, + WebConfig: webConfig, + MetricStore: ms, + Logger: logger, + ErrCh: errCh, + } + go startMetricsServer(metricOption) + + err = shutdownServerOnQuit(*socketPath, []*network.ServerOption{verificationOption, metricOption}, ms, errCh, logger) + if err != nil { + level.Error(logger).Log("msg", "Failed to clean up the server", "err", err) + } +} + +func decodeRequest(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() // Make sure the underlying io.Reader is closed. + switch contentEncoding := r.Header.Get("Content-Encoding"); strings.ToLower(contentEncoding) { + case "gzip": + gr, err := gzip.NewReader(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer gr.Close() + r.Body = gr + case "snappy": + r.Body = io.NopCloser(snappy.NewReader(r.Body)) + default: + // Do nothing. + } + + h.ServeHTTP(w, r) + }) +} + +// computeRoutePrefix returns the effective route prefix based on the +// provided flag values for --web.route-prefix and +// --web.external-url. With prefix empty, the path of externalURL is +// used instead. A prefix "/" results in an empty returned prefix. Any +// non-empty prefix is normalized to start, but not to end, with "/". +func computeRoutePrefix(prefix string, externalURL *url.URL) string { + if prefix == "" { + prefix = externalURL.Path + } + + if prefix == "/" { + prefix = "" + } + + if prefix != "" { + prefix = "/" + strings.Trim(prefix, "/") + } + + return prefix +} + +// shutdownServerOnQuit shutdowns the provided server upon closing the provided +// quitCh or upon receiving a SIGINT or SIGTERM. +func shutdownServerOnQuit(socketPath string, options []*network.ServerOption, ms *storage.DiskMetricStore, errCh <-chan error, logger log.Logger) error { + notifier := make(chan os.Signal, 1) + signal.Notify(notifier, os.Interrupt, syscall.SIGTERM) + + select { + case <-notifier: + level.Info(logger).Log("msg", "received SIGINT/SIGTERM; exiting gracefully...") + break + case err := <-errCh: + level.Warn(logger).Log("msg", "received error when launching server, exiting gracefully...", "err", err) + break + } + + defer os.Remove(socketPath) + + var retErr error + for _, option := range options { + err := option.Server.Shutdown(context.Background()) + if err != nil { + level.Error(logger).Log("msg", "unable to shutdown the server", "err", err) + retErr = errors.Join(retErr, err) + } + } + + err := ms.Shutdown() + if err != nil { + level.Error(logger).Log("msg", "unable to shutdown the storage service", "err", err) + retErr = errors.Join(retErr, err) + } + return retErr +} + +// startVerificationServer starts a verification server listening the unix socket +// it is also responsible for authentication via pid. +func startVerificationServer(option *network.ServerOption) { + level.Info(option.Logger).Log("msg", "Start verification server") + unixListener, err := peercred.Listen(context.Background(), option.SocketPath) + if err != nil { + level.Error(option.Logger).Log("msg", "Could not create local unix socket", "err", err) + option.ErrCh <- err + return + } + + // chmod socketPath + err = os.Chmod(option.SocketPath, 0o777) + if err != nil { + level.Error(option.Logger).Log("msg", "Could not chmod local unix socket", "err", err) + option.ErrCh <- err + return + } + + listener := network.WrappedListener{ + Listener: unixListener, + TrustedPath: option.TrustedPath, + Option: option, + ErrCh: make(chan *network.WrappedInstance, 1), + DoneCh: make(chan *network.WrappedInstance, 1), + } + + quitCh := make(chan struct{}, 1) + + go func() { + err = web.Serve(&listener, option.Server, option.WebConfig, option.Logger) + if err != nil { + if errors.Is(err, http.ErrServerClosed) { + level.Info(option.Logger).Log("msg", "Verification server stopped") + } else { + level.Error(option.Logger).Log("msg", "Verification server stopped with error", "err", err) + option.ErrCh <- err + } + } + + quitCh <- struct{}{} + }() + + for { + select { + case <-quitCh: + // stop all loop + return + case wrappedInstance := <-listener.ErrCh: + level.Error(option.Logger).Log("msg", "Container monitor instance receieved error", "container id", wrappedInstance.ContainerInfo.ID, "err", wrappedInstance.Err) + // server side closes the connection in case client side misses (in theory client will close the connection first) + if wrappedInstance.Conn != nil { + wrappedInstance.Conn.Close() + } + case wrappedInstance := <-listener.DoneCh: + level.Info(option.Logger).Log("msg", "Container monitor instance completed, will close the connection", "container id", wrappedInstance.ContainerInfo.ID) + // server side closes the connection in case client side misses (in theory client will close the connection first) + if wrappedInstance.Conn != nil { + wrappedInstance.Conn.Close() + } + } + } +} + +// startMetricsServer starts the `/metrics` endpoints, exposing metrics +func startMetricsServer(option *network.ServerOption) { + level.Info(option.Logger).Log("msg", "Start metrics server") + err := web.ListenAndServe(option.Server, option.WebConfig, option.Logger) + if err != nil { + if errors.Is(err, http.ErrServerClosed) { + level.Info(option.Logger).Log("msg", "Metrics server stopped") + } else { + level.Error(option.Logger).Log("msg", "Metrics server stopped", "err", err) + option.ErrCh <- err + } + } +} diff --git a/storage/diskmetricstore.go b/storage/diskmetricstore.go new file mode 100644 index 0000000..9b1abb7 --- /dev/null +++ b/storage/diskmetricstore.go @@ -0,0 +1,609 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +// Copyright 2014 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "encoding/gob" + "errors" + "fmt" + "os" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + + //nolint:staticcheck // Ignore SA1019. Dependencies use the deprecated package, so we have to, too. + "github.com/golang/protobuf/proto" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + + dto "github.com/prometheus/client_model/go" +) + +const ( + pushMetricName = "push_time_seconds" + pushMetricHelp = "Last Unix time when changing this group in the Pushgateway succeeded." + pushFailedMetricName = "push_failure_time_seconds" + pushFailedMetricHelp = "Last Unix time when changing this group in the Pushgateway failed." + writeQueueCapacity = 1000 +) + +var errTimestamp = errors.New("pushed metrics must not have timestamps") + +// DiskMetricStore is an implementation of MetricStore that persists metrics to +// disk. +type DiskMetricStore struct { + lock sync.RWMutex // Protects metricFamilies. + writeQueue chan WriteRequest + drain chan struct{} + done chan error + metricGroups GroupingKeyToMetricGroup + persistenceFile string + predefinedHelp map[string]string + logger log.Logger +} + +type mfStat struct { + pos int // Where in the result slice is the MetricFamily? + copied bool // Has the MetricFamily already been copied? +} + +// NewDiskMetricStore returns a DiskMetricStore ready to use. To cleanly shut it +// down and free resources, the Shutdown() method has to be called. +// +// If persistenceFile is the empty string, no persisting to disk will +// happen. Otherwise, a file of that name is used for persisting metrics to +// disk. If the file already exists, metrics are read from it as part of the +// start-up. Persisting is happening upon shutdown and after every write action, +// but the latter will only happen persistenceDuration after the previous +// persisting. +// +// If a non-nil Gatherer is provided, the help strings of metrics gathered by it +// will be used as standard. Pushed metrics with deviating help strings will be +// adjusted to avoid inconsistent expositions. +func NewDiskMetricStore( + persistenceFile string, + persistenceInterval time.Duration, + gatherPredefinedHelpFrom prometheus.Gatherer, + logger log.Logger, +) *DiskMetricStore { + // TODO: Do that outside of the constructor to allow the HTTP server to + // serve /-/healthy and /-/ready earlier. + dms := &DiskMetricStore{ + writeQueue: make(chan WriteRequest, writeQueueCapacity), + drain: make(chan struct{}), + done: make(chan error), + metricGroups: GroupingKeyToMetricGroup{}, + persistenceFile: persistenceFile, + logger: logger, + } + if err := dms.restore(); err != nil { + level.Error(logger).Log("msg", "could not load persisted metrics", "err", err) + } + if helpStrings, err := extractPredefinedHelpStrings(gatherPredefinedHelpFrom); err == nil { + dms.predefinedHelp = helpStrings + } else { + level.Error(logger).Log("msg", "could not gather metrics for predefined help strings", "err", err) + } + + go dms.loop(persistenceInterval) + return dms +} + +// SubmitWriteRequest implements the MetricStore interface. +func (dms *DiskMetricStore) SubmitWriteRequest(req WriteRequest) { + dms.writeQueue <- req +} + +// Shutdown implements the MetricStore interface. +func (dms *DiskMetricStore) Shutdown() error { + close(dms.drain) + return <-dms.done +} + +// Healthy implements the MetricStore interface. +func (dms *DiskMetricStore) Healthy() error { + // By taking the lock we check that there is no deadlock. + dms.lock.Lock() + defer dms.lock.Unlock() + + // A pushgateway that cannot be written to should not be + // considered as healthy. + if len(dms.writeQueue) == cap(dms.writeQueue) { + return fmt.Errorf("write queue is full") + } + + return nil +} + +// Ready implements the MetricStore interface. +func (dms *DiskMetricStore) Ready() error { + return dms.Healthy() +} + +// GetMetricFamilies implements the MetricStore interface. +func (dms *DiskMetricStore) GetMetricFamilies() []*dto.MetricFamily { + dms.lock.RLock() + defer dms.lock.RUnlock() + + result := []*dto.MetricFamily{} + mfStatByName := map[string]mfStat{} + + for _, group := range dms.metricGroups { + for name, tmf := range group.Metrics { + mf := tmf.GetMetricFamily() + if mf == nil { + level.Warn(dms.logger).Log("msg", "storage corruption detected, consider wiping the persistence file") + continue + } + stat, exists := mfStatByName[name] + if exists { + existingMF := result[stat.pos] + if !stat.copied { + mfStatByName[name] = mfStat{ + pos: stat.pos, + copied: true, + } + existingMF = copyMetricFamily(existingMF) + result[stat.pos] = existingMF + } + if mf.GetHelp() != existingMF.GetHelp() { + level.Info(dms.logger).Log("msg", "metric families inconsistent help strings", "err", "Metric families have inconsistent help strings. The latter will have priority. This is bad. Fix your pushed metrics!", "new", mf, "old", existingMF) + } + // Type inconsistency cannot be fixed here. We will detect it during + // gathering anyway, so no reason to log anything here. + existingMF.Metric = append(existingMF.Metric, mf.Metric...) + } else { + copied := false + if help, ok := dms.predefinedHelp[name]; ok && mf.GetHelp() != help { + level.Info(dms.logger).Log("msg", "metric families overlap", "err", "Metric family has the same name as a metric family used by the Pushgateway itself but it has a different help string. Changing it to the standard help string. This is bad. Fix your pushed metrics!", "metric_family", mf, "standard_help", help) + mf = copyMetricFamily(mf) + copied = true + mf.Help = proto.String(help) + } + mfStatByName[name] = mfStat{ + pos: len(result), + copied: copied, + } + result = append(result, mf) + } + } + } + return result +} + +// GetMetricFamiliesMap implements the MetricStore interface. +func (dms *DiskMetricStore) GetMetricFamiliesMap() GroupingKeyToMetricGroup { + dms.lock.RLock() + defer dms.lock.RUnlock() + groupsCopy := make(GroupingKeyToMetricGroup, len(dms.metricGroups)) + for k, g := range dms.metricGroups { + metricsCopy := make(NameToTimestampedMetricFamilyMap, len(g.Metrics)) + groupsCopy[k] = MetricGroup{Labels: g.Labels, Metrics: metricsCopy} + for n, tmf := range g.Metrics { + metricsCopy[n] = tmf + } + } + return groupsCopy +} + +func (dms *DiskMetricStore) loop(persistenceInterval time.Duration) { + lastPersist := time.Now() + persistScheduled := false + lastWrite := time.Time{} + persistDone := make(chan time.Time) + var persistTimer *time.Timer + + checkPersist := func() { + if dms.persistenceFile != "" && !persistScheduled && lastWrite.After(lastPersist) { + persistTimer = time.AfterFunc( + persistenceInterval-lastWrite.Sub(lastPersist), + func() { + persistStarted := time.Now() + if err := dms.persist(); err != nil { + level.Error(dms.logger).Log("msg", "error persisting metrics", "err", err) + } else { + level.Info(dms.logger).Log("msg", "metrics persisted", "file", dms.persistenceFile) + } + persistDone <- persistStarted + }, + ) + persistScheduled = true + } + } + + for { + select { + case wr := <-dms.writeQueue: + lastWrite = time.Now() + if dms.checkWriteRequest(wr) { + dms.processWriteRequest(wr) + } else { + dms.setPushFailedTimestamp(wr) + } + if wr.Done != nil { + close(wr.Done) + } + checkPersist() + case lastPersist = <-persistDone: + persistScheduled = false + checkPersist() // In case something has been written in the meantime. + case <-dms.drain: + // Prevent a scheduled persist from firing later. + if persistTimer != nil { + persistTimer.Stop() + } + // Now draining... + for { + select { + case wr := <-dms.writeQueue: + if dms.checkWriteRequest(wr) { + dms.processWriteRequest(wr) + } else { + dms.setPushFailedTimestamp(wr) + } + default: + dms.done <- dms.persist() + return + } + } + } + } +} + +func (dms *DiskMetricStore) processWriteRequest(wr WriteRequest) { + dms.lock.Lock() + defer dms.lock.Unlock() + + key := groupingKeyFor(wr.Labels) + + if wr.MetricFamilies == nil { + // No MetricFamilies means delete request. Delete the whole + // metric group, and we are done here. + delete(dms.metricGroups, key) + return + } + // Otherwise, it's an update. + group, ok := dms.metricGroups[key] + if !ok { + group = MetricGroup{ + Labels: wr.Labels, + Metrics: NameToTimestampedMetricFamilyMap{}, + } + dms.metricGroups[key] = group + } else if wr.Replace { + // For replace, we have to delete all metric families in the + // group except pre-existing push timestamps. + for name := range group.Metrics { + if name != pushMetricName && name != pushFailedMetricName { + delete(group.Metrics, name) + } + } + } + wr.MetricFamilies[pushMetricName] = newPushTimestampGauge(wr.Labels, wr.Timestamp) + // Only add a zero push-failed metric if none is there yet, so that a + // previously added fail timestamp is retained. + if _, ok := group.Metrics[pushFailedMetricName]; !ok { + wr.MetricFamilies[pushFailedMetricName] = newPushFailedTimestampGauge(wr.Labels, time.Time{}) + } + for name, mf := range wr.MetricFamilies { + group.Metrics[name] = TimestampedMetricFamily{ + Timestamp: wr.Timestamp, + GobbableMetricFamily: (*GobbableMetricFamily)(mf), + } + } +} + +func (dms *DiskMetricStore) setPushFailedTimestamp(wr WriteRequest) { + dms.lock.Lock() + defer dms.lock.Unlock() + + key := groupingKeyFor(wr.Labels) + + group, ok := dms.metricGroups[key] + if !ok { + group = MetricGroup{ + Labels: wr.Labels, + Metrics: NameToTimestampedMetricFamilyMap{}, + } + dms.metricGroups[key] = group + } + + group.Metrics[pushFailedMetricName] = TimestampedMetricFamily{ + Timestamp: wr.Timestamp, + GobbableMetricFamily: (*GobbableMetricFamily)(newPushFailedTimestampGauge(wr.Labels, wr.Timestamp)), + } + // Only add a zero push metric if none is there yet, so that a + // previously added push timestamp is retained. + if _, ok := group.Metrics[pushMetricName]; !ok { + group.Metrics[pushMetricName] = TimestampedMetricFamily{ + Timestamp: wr.Timestamp, + GobbableMetricFamily: (*GobbableMetricFamily)(newPushTimestampGauge(wr.Labels, time.Time{})), + } + } +} + +// checkWriteRequest return if applying the provided WriteRequest will result in +// a consistent state of metrics. The dms is not modified by the check. However, +// the WriteRequest _will_ be sanitized: the MetricFamilies are ensured to +// contain the grouping Labels after the check. If false is returned, the +// causing error is written to the Done channel of the WriteRequest. +// +// Special case: If the WriteRequest has no Done channel set, the (expensive) +// consistency check is skipped. The WriteRequest is still sanitized, and the +// presence of timestamps still results in returning false. +func (dms *DiskMetricStore) checkWriteRequest(wr WriteRequest) bool { + if wr.MetricFamilies == nil { + // Delete request cannot create inconsistencies, and nothing has + // to be sanitized. + return true + } + + var err error + defer func() { + if err != nil && wr.Done != nil { + wr.Done <- err + } + }() + + if timestampsPresent(wr.MetricFamilies) { + err = errTimestamp + return false + } + for _, mf := range wr.MetricFamilies { + sanitizeLabels(mf, wr.Labels) + } + + // Without Done channel, don't do the expensive consistency check. + if wr.Done == nil { + return true + } + + // Construct a test dms, acting on a copy of the metrics, to test the + // WriteRequest with. + tdms := &DiskMetricStore{ + metricGroups: dms.GetMetricFamiliesMap(), + predefinedHelp: dms.predefinedHelp, + logger: log.NewNopLogger(), + } + tdms.processWriteRequest(wr) + + // Construct a test Gatherer to check if consistent gathering is possible. + tg := prometheus.Gatherers{ + prometheus.DefaultGatherer, + prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { + return tdms.GetMetricFamilies(), nil + }), + } + if _, err = tg.Gather(); err != nil { + return false + } + return true +} + +func (dms *DiskMetricStore) persist() error { + // Check (again) if persistence is configured because some code paths + // will call this method even if it is not. + if dms.persistenceFile == "" { + return nil + } + f, err := os.CreateTemp( + path.Dir(dms.persistenceFile), + path.Base(dms.persistenceFile)+".in_progress.", + ) + if err != nil { + return err + } + inProgressFileName := f.Name() + e := gob.NewEncoder(f) + + dms.lock.RLock() + err = e.Encode(dms.metricGroups) + dms.lock.RUnlock() + if err != nil { + f.Close() + os.Remove(inProgressFileName) + return err + } + if err := f.Close(); err != nil { + os.Remove(inProgressFileName) + return err + } + return os.Rename(inProgressFileName, dms.persistenceFile) +} + +func (dms *DiskMetricStore) restore() error { + if dms.persistenceFile == "" { + return nil + } + f, err := os.Open(dms.persistenceFile) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + defer f.Close() + d := gob.NewDecoder(f) + + return d.Decode(&dms.metricGroups) +} + +func copyMetricFamily(mf *dto.MetricFamily) *dto.MetricFamily { + return &dto.MetricFamily{ + Name: mf.Name, + Help: mf.Help, + Type: mf.Type, + Metric: append([]*dto.Metric{}, mf.Metric...), + } +} + +// groupingKeyFor creates a grouping key from the provided map of grouping +// labels. The grouping key is created by joining all label names and values +// together with model.SeparatorByte as a separator. The label names are sorted +// lexicographically before joining. In that way, the grouping key is both +// reproducible and unique. +func groupingKeyFor(labels map[string]string) string { + if len(labels) == 0 { // Super fast path. + return "" + } + + labelNames := make([]string, 0, len(labels)) + for labelName := range labels { + labelNames = append(labelNames, labelName) + } + sort.Strings(labelNames) + + sb := strings.Builder{} + for i, labelName := range labelNames { + sb.WriteString(labelName) + sb.WriteByte(model.SeparatorByte) + sb.WriteString(labels[labelName]) + if i+1 < len(labels) { // No separator at the end. + sb.WriteByte(model.SeparatorByte) + } + } + return sb.String() +} + +// extractPredefinedHelpStrings extracts all the HELP strings from the provided +// gatherer so that the DiskMetricStore can fix deviations in pushed metrics. +func extractPredefinedHelpStrings(g prometheus.Gatherer) (map[string]string, error) { + if g == nil { + return nil, nil + } + mfs, err := g.Gather() + if err != nil { + return nil, err + } + result := map[string]string{} + for _, mf := range mfs { + result[mf.GetName()] = mf.GetHelp() + } + return result, nil +} + +func newPushTimestampGauge(groupingLabels map[string]string, t time.Time) *dto.MetricFamily { + return newTimestampGauge(pushMetricName, pushMetricHelp, groupingLabels, t) +} + +func newPushFailedTimestampGauge(groupingLabels map[string]string, t time.Time) *dto.MetricFamily { + return newTimestampGauge(pushFailedMetricName, pushFailedMetricHelp, groupingLabels, t) +} + +func newTimestampGauge(name, help string, groupingLabels map[string]string, t time.Time) *dto.MetricFamily { + var ts float64 + if !t.IsZero() { + ts = float64(t.UnixNano()) / 1e9 + } + mf := &dto.MetricFamily{ + Name: proto.String(name), + Help: proto.String(help), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Gauge: &dto.Gauge{ + Value: proto.Float64(ts), + }, + }, + }, + } + sanitizeLabels(mf, groupingLabels) + return mf +} + +// sanitizeLabels ensures that all the labels in groupingLabels and the +// `instance` label are present in the MetricFamily. The label values from +// groupingLabels are set in each Metric, no matter what. After that, if the +// 'instance' label is not present at all in a Metric, it will be created (with +// an empty string as value). +// +// Finally, sanitizeLabels sorts the label pairs of all metrics. +func sanitizeLabels(mf *dto.MetricFamily, groupingLabels map[string]string) { + gLabelsNotYetDone := make(map[string]string, len(groupingLabels)) + +metric: + for _, m := range mf.GetMetric() { + for ln, lv := range groupingLabels { + gLabelsNotYetDone[ln] = lv + } + hasInstanceLabel := false + for _, lp := range m.GetLabel() { + ln := lp.GetName() + if lv, ok := gLabelsNotYetDone[ln]; ok { + lp.Value = proto.String(lv) + delete(gLabelsNotYetDone, ln) + } + if ln == string(model.InstanceLabel) { + hasInstanceLabel = true + } + if len(gLabelsNotYetDone) == 0 && hasInstanceLabel { + sort.Sort(labelPairs(m.Label)) + continue metric + } + } + for ln, lv := range gLabelsNotYetDone { + m.Label = append(m.Label, &dto.LabelPair{ + Name: proto.String(ln), + Value: proto.String(lv), + }) + if ln == string(model.InstanceLabel) { + hasInstanceLabel = true + } + delete(gLabelsNotYetDone, ln) // To prepare map for next metric. + } + if !hasInstanceLabel { + m.Label = append(m.Label, &dto.LabelPair{ + Name: proto.String(string(model.InstanceLabel)), + Value: proto.String(""), + }) + } + sort.Sort(labelPairs(m.Label)) + } +} + +// Checks if any timestamps have been specified. +func timestampsPresent(metricFamilies map[string]*dto.MetricFamily) bool { + for _, mf := range metricFamilies { + for _, m := range mf.GetMetric() { + if m.TimestampMs != nil { + return true + } + } + } + return false +} + +// labelPairs implements sort.Interface. It provides a sortable version of a +// slice of dto.LabelPair pointers. +type labelPairs []*dto.LabelPair + +func (s labelPairs) Len() int { + return len(s) +} + +func (s labelPairs) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s labelPairs) Less(i, j int) bool { + return s[i].GetName() < s[j].GetName() +} diff --git a/storage/diskmetricstore_test.go b/storage/diskmetricstore_test.go new file mode 100644 index 0000000..40d1478 --- /dev/null +++ b/storage/diskmetricstore_test.go @@ -0,0 +1,1571 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +// Copyright 2014 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:testpackage +package storage + +import ( + "errors" + "fmt" + "math" + "os" + "path" + "sort" + "testing" + "time" + + "github.com/go-kit/log" + //nolint:staticcheck // Ignore SA1019. Dependencies use the deprecated package, so we have to, too. + "github.com/golang/protobuf/proto" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + + dto "github.com/prometheus/client_model/go" + + "github.com/jasonyangshadow/apptheus/testutil" +) + +var ( + logger = log.NewNopLogger() + // Example metric families. Keep labels sorted lexicographically! + mf1a = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(-3e3), + }, + }, + }, + } + mf1b = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + }, + } + mf1c = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance1"), + }, + { + Name: proto.String("job"), + Value: proto.String("job2"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + }, + } + mf1d = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job3"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + }, + } + mf1e = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + }, + } + // mf1acd is merged from mf1a, mf1c, mf1d. + mf1acd = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(-3e3), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance1"), + }, + { + Name: proto.String("job"), + Value: proto.String("job2"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job3"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + }, + } + // mf1be is merged from mf1b and mf1e, with added empty instance label for mf1e. + mf1be = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + }, + } + // mf1ts is mf1a with a timestamp set. + mf1ts = &dto.MetricFamily{ + Name: proto.String("mf1"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(-3e3), + }, + TimestampMs: proto.Int64(103948), + }, + }, + } + mf2 = &dto.MetricFamily{ + Name: proto.String("mf2"), + Help: proto.String("doc string 2"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("basename"), + Value: proto.String("basevalue2"), + }, + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + { + Name: proto.String("labelname"), + Value: proto.String("val2"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(math.Inf(+1)), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + { + Name: proto.String("labelname"), + Value: proto.String("val1"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(math.Inf(-1)), + }, + }, + }, + } + mf3 = &dto.MetricFamily{ + Name: proto.String("mf3"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance1"), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(42), + }, + }, + }, + } + mf4 = &dto.MetricFamily{ + Name: proto.String("mf4"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance2"), + }, + { + Name: proto.String("job"), + Value: proto.String("job3"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(3.4345), + }, + }, + }, + } + mf5 = &dto.MetricFamily{ + Name: proto.String("mf5"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance5"), + }, + { + Name: proto.String("job"), + Value: proto.String("job5"), + }, + }, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(0), + SampleSum: proto.Float64(0), + }, + }, + }, + } + mfh1 = &dto.MetricFamily{ + Name: proto.String("mf_help"), + Help: proto.String("Help string for mfh1."), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(3948.838), + }, + }, + }, + } + mfh2 = &dto.MetricFamily{ + Name: proto.String("mf_help"), + Help: proto.String("Help string for mfh2."), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job2"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(83), + }, + }, + }, + } + // Both mfh metrics with mfh1's help string. + mfh12 = &dto.MetricFamily{ + Name: proto.String("mf_help"), + Help: proto.String("Help string for mfh1."), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(3948.838), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job2"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(83), + }, + }, + }, + } + // Both mfh metrics with mfh2's help string. + mfh21 = &dto.MetricFamily{ + Name: proto.String("mf_help"), + Help: proto.String("Help string for mfh2."), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(3948.838), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job2"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(83), + }, + }, + }, + } + // mfgg is the usual go_goroutines gauge but with a different help text. + mfgg = &dto.MetricFamily{ + Name: proto.String("go_goroutines"), + Help: proto.String("Inconsistent doc string, fixed version in mfggFixed."), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(5), + }, + }, + }, + } + // mfgc is the usual go_goroutines metric but mistyped as counter. + mfgc = &dto.MetricFamily{ + Name: proto.String("go_goroutines"), + Help: proto.String("Number of goroutines that currently exist."), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(5), + }, + }, + }, + } + mfggFixed = &dto.MetricFamily{ + Name: proto.String("go_goroutines"), + Help: proto.String("Number of goroutines that currently exist."), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String(""), + }, + { + Name: proto.String("job"), + Value: proto.String("job1"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(5), + }, + }, + }, + } + mfUnlabelled = &dto.MetricFamily{ + Name: proto.String("mf_unlabelled"), + Help: proto.String("Metric with no labels to check sanitizeLabels."), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{}, + Gauge: &dto.Gauge{ + Value: proto.Float64(42), + }, + }, + }, + } +) + +func addGroup( + mg GroupingKeyToMetricGroup, + groupingLabels map[string]string, + metrics NameToTimestampedMetricFamilyMap, +) { + mg[groupingKeyFor(groupingLabels)] = MetricGroup{ + Labels: groupingLabels, + Metrics: metrics, + } +} + +func TestGetMetricFamilies(t *testing.T) { + testTime := time.Now() + + mg := GroupingKeyToMetricGroup{} + addGroup( + mg, + map[string]string{ + "job": "job1", + "instance": "instance1", + }, + NameToTimestampedMetricFamilyMap{ + "mf2": TimestampedMetricFamily{ + Timestamp: testTime, + GobbableMetricFamily: (*GobbableMetricFamily)(mf2), + }, + }, + ) + addGroup( + mg, + map[string]string{ + "job": "job1", + "instance": "instance2", + }, + NameToTimestampedMetricFamilyMap{ + "mf1": TimestampedMetricFamily{ + Timestamp: testTime, + GobbableMetricFamily: (*GobbableMetricFamily)(mf1a), + }, + "mf3": TimestampedMetricFamily{ + Timestamp: testTime, + GobbableMetricFamily: (*GobbableMetricFamily)(mf3), + }, + }, + ) + addGroup( + mg, + map[string]string{ + "job": "job2", + "instance": "instance1", + }, + NameToTimestampedMetricFamilyMap{ + "mf1": TimestampedMetricFamily{ + Timestamp: testTime, + GobbableMetricFamily: (*GobbableMetricFamily)(mf1c), + }, + }, + ) + addGroup( + mg, + map[string]string{ + "job": "job3", + "instance": "instance1", + }, + NameToTimestampedMetricFamilyMap{}, + ) + addGroup( + mg, + map[string]string{ + "job": "job3", + "instance": "instance2", + }, + NameToTimestampedMetricFamilyMap{ + "mf4": TimestampedMetricFamily{ + Timestamp: testTime, + GobbableMetricFamily: (*GobbableMetricFamily)(mf4), + }, + "mf1": TimestampedMetricFamily{ + Timestamp: testTime, + GobbableMetricFamily: (*GobbableMetricFamily)(mf1d), + }, + }, + ) + addGroup( + mg, + map[string]string{ + "job": "job4", + }, + NameToTimestampedMetricFamilyMap{}, + ) + + dms := &DiskMetricStore{metricGroups: mg} + + if err := checkMetricFamilies(dms, mf1acd, mf2, mf3, mf4); err != nil { + t.Error(err) + } +} + +func TestAddDeletePersistRestore(t *testing.T) { + tempDir, err := os.MkdirTemp("", "diskmetricstore.TestAddDeletePersistRestore.") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + fileName := path.Join(tempDir, "persistence") + dms := NewDiskMetricStore(fileName, 100*time.Millisecond, nil, logger) + + // Submit a single simple metric family. + ts1 := time.Now() + grouping1 := map[string]string{ + "job": "job1", + "instance": "instance1", + } + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts1, + MetricFamilies: testutil.MetricFamiliesMap(mf3), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp := newPushTimestampGauge(grouping1, ts1) + pushFailedTimestamp := newPushFailedTimestampGauge(grouping1, time.Time{}) + if err := checkMetricFamilies( + dms, mf3, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Submit two metric families for a different instance. + ts2 := ts1.Add(time.Second) + grouping2 := map[string]string{ + "job": "job1", + "instance": "instance2", + } + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping2, + Timestamp: ts2, + MetricFamilies: testutil.MetricFamiliesMap(mf1b, mf2), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp.Metric = append( + pushTimestamp.Metric, newPushTimestampGauge(grouping2, ts2).Metric[0], + ) + pushFailedTimestamp.Metric = append( + pushFailedTimestamp.Metric, newPushFailedTimestampGauge(grouping2, time.Time{}).Metric[0], + ) + if err := checkMetricFamilies( + dms, mf1b, mf2, mf3, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + // Submit a metric family with the same name for the same job/instance again. + // Should overwrite the previous metric family for the same job/instance + ts3 := ts2.Add(time.Second) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping2, + Timestamp: ts3, + MetricFamilies: testutil.MetricFamiliesMap(mf1a), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp.Metric[1] = newPushTimestampGauge(grouping2, ts3).Metric[0] + if err := checkMetricFamilies( + dms, mf1a, mf2, mf3, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Add a new group by job, with a summary without any observations yet. + ts4 := ts3.Add(time.Second) + grouping4 := map[string]string{ + "job": "job5", + } + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping4, + Timestamp: ts4, + MetricFamilies: testutil.MetricFamiliesMap(mf5), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp.Metric = append( + pushTimestamp.Metric, newPushTimestampGauge(grouping4, ts4).Metric[0], + ) + pushFailedTimestamp.Metric = append( + pushFailedTimestamp.Metric, newPushFailedTimestampGauge(grouping4, time.Time{}).Metric[0], + ) + if err := checkMetricFamilies( + dms, mf1a, mf2, mf3, mf5, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Shutdown the dms. + if err := dms.Shutdown(); err != nil { + t.Fatal(err) + } + + // Load it again. + dms = NewDiskMetricStore(fileName, 100*time.Millisecond, nil, logger) + if err := checkMetricFamilies( + dms, mf1a, mf2, mf3, mf5, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + // Spot-check timestamp. + tmf := dms.metricGroups[groupingKeyFor(map[string]string{ + "job": "job1", + "instance": "instance2", + })].Metrics["mf1"] + if expected, got := ts3, tmf.Timestamp; !expected.Equal(got) { + t.Errorf("Expected timestamp %v, got %v.", expected, got) + } + + // Delete two groups. + dms.SubmitWriteRequest(WriteRequest{ + Labels: map[string]string{ + "job": "job1", + "instance": "instance1", + }, + }) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: map[string]string{ + "job": "job5", + }, + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp = newPushTimestampGauge(grouping2, ts3) + pushFailedTimestamp = newPushFailedTimestampGauge(grouping2, time.Time{}) + if err := checkMetricFamilies( + dms, mf1a, mf2, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Submit another one. + ts5 := ts4.Add(time.Second) + grouping5 := map[string]string{ + "job": "job3", + "instance": "instance2", + } + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping5, + Timestamp: ts5, + MetricFamilies: testutil.MetricFamiliesMap(mf4), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp.Metric = append( + pushTimestamp.Metric, newPushTimestampGauge(grouping5, ts5).Metric[0], + ) + pushFailedTimestamp.Metric = append( + pushFailedTimestamp.Metric, newPushFailedTimestampGauge(grouping5, time.Time{}).Metric[0], + ) + if err := checkMetricFamilies( + dms, mf1a, mf2, mf4, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Delete a job does not remove anything because there is no suitable + // grouping. + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: map[string]string{ + "job": "job1", + }, + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + if err := checkMetricFamilies( + dms, mf1a, mf2, mf4, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Delete another group. + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping5, + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp = newPushTimestampGauge(grouping2, ts3) + pushFailedTimestamp = newPushFailedTimestampGauge(grouping2, time.Time{}) + if err := checkMetricFamilies( + dms, mf1a, mf2, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + // Check that no empty map entry for job3 was left behind. + if _, stillExists := dms.metricGroups[groupingKeyFor(grouping5)]; stillExists { + t.Error("An instance map for 'job3' still exists.") + } + + // Shutdown the dms again, directly after a number of write request + // (to check draining). + for i := 0; i < 10; i++ { + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping5, + Timestamp: ts5, + MetricFamilies: testutil.MetricFamiliesMap(mf4), + }) + } + grouping6 := map[string]string{ + "job": "job4", + "instance": "instance1", + } + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping6, + Timestamp: ts5, + MetricFamilies: testutil.MetricFamiliesMap(mfUnlabelled), + }) + if err := dms.Shutdown(); err != nil { + t.Fatal(err) + } + pushTimestamp.Metric = append( + pushTimestamp.Metric, + newPushTimestampGauge(grouping5, ts5).Metric[0], + newPushTimestampGauge(grouping6, ts5).Metric[0], + ) + pushFailedTimestamp.Metric = append( + pushFailedTimestamp.Metric, + newPushFailedTimestampGauge(grouping5, time.Time{}).Metric[0], + newPushFailedTimestampGauge(grouping6, time.Time{}).Metric[0], + ) + mfLabelled := proto.Clone(mfUnlabelled).(*dto.MetricFamily) + // SanitizeLabels should add these labels to the unlabelled metric. + mfLabelled.Metric[0].Label = []*dto.LabelPair{ + { + Name: proto.String("instance"), + Value: proto.String("instance1"), + }, + { + Name: proto.String("job"), + Value: proto.String("job4"), + }, + } + if err := checkMetricFamilies( + dms, mf1a, mf2, mf4, mfLabelled, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } +} + +func TestNoPersistence(t *testing.T) { + dms := NewDiskMetricStore("", 100*time.Millisecond, nil, logger) + + ts1 := time.Now() + grouping1 := map[string]string{ + "job": "job1", + "instance": "instance1", + } + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts1, + MetricFamilies: testutil.MetricFamiliesMap(mf3), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp := newPushTimestampGauge(grouping1, ts1) + pushFailedTimestamp := newPushFailedTimestampGauge(grouping1, time.Time{}) + if err := checkMetricFamilies( + dms, mf3, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + if err := dms.Shutdown(); err != nil { + t.Fatal(err) + } + + dms = NewDiskMetricStore("", 100*time.Millisecond, nil, logger) + if err := checkMetricFamilies(dms); err != nil { + t.Error(err) + } + + if err := dms.Ready(); err != nil { + t.Error(err) + } + + if err := dms.Healthy(); err != nil { + t.Error(err) + } +} + +func TestRejectTimestamps(t *testing.T) { + dms := NewDiskMetricStore("", 100*time.Millisecond, nil, logger) + + ts1 := time.Now() + grouping1 := map[string]string{ + "job": "job1", + "instance": "instance2", + } + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts1, + MetricFamilies: testutil.MetricFamiliesMap(mf1ts), + Done: errCh, + }) + var err error + for err = range errCh { + if !errors.Is(err, errTimestamp) { + t.Errorf("Expected error %q, got %q.", errTimestamp, err) + } + } + if err == nil { + t.Error("Expected error on pushing metric with timestamp.") + } + pushTimestamp := newPushTimestampGauge(grouping1, time.Time{}) + pushFailedTimestamp := newPushFailedTimestampGauge(grouping1, ts1) + if err := checkMetricFamilies( + dms, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + if err := dms.Shutdown(); err != nil { + t.Fatal(err) + } +} + +func TestRejectInconsistentPush(t *testing.T) { + dms := NewDiskMetricStore("", 100*time.Millisecond, nil, logger) + + ts1 := time.Now() + grouping1 := map[string]string{ + "job": "job1", + } + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts1, + MetricFamilies: testutil.MetricFamiliesMap(mfgc), + Done: errCh, + }) + err := <-errCh + if err == nil { + t.Error("Expected error pushing inconsistent go_goroutines metric.") + } + pushTimestamp := newPushTimestampGauge(grouping1, time.Time{}) + pushFailedTimestamp := newPushFailedTimestampGauge(grouping1, ts1) + if err := checkMetricFamilies( + dms, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + ts2 := ts1.Add(time.Second) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts2, + MetricFamilies: testutil.MetricFamiliesMap(mf1a), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp = newPushTimestampGauge(grouping1, ts2) + if err := checkMetricFamilies( + dms, mf1a, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + ts3 := ts2.Add(time.Second) + grouping3 := map[string]string{ + "job": "job1", + "instance": "instance2", + } + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping3, + Timestamp: ts3, + MetricFamilies: testutil.MetricFamiliesMap(mf1b), + Done: errCh, + }) + err = <-errCh + if err == nil { + t.Error("Expected error pushing duplicate mf1 metric.") + } + pushTimestamp.Metric = append( + pushTimestamp.Metric, newPushTimestampGauge(grouping3, time.Time{}).Metric[0], + ) + pushFailedTimestamp.Metric = append( + pushFailedTimestamp.Metric, newPushFailedTimestampGauge(grouping3, ts3).Metric[0], + ) + if err := checkMetricFamilies( + dms, mf1a, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + if err := dms.Shutdown(); err != nil { + t.Fatal(err) + } +} + +func TestSanitizeLabels(t *testing.T) { + dms := NewDiskMetricStore("", 100*time.Millisecond, nil, logger) + + // Push mf1c with the grouping matching mf1b, mf1b should end up in storage. + ts1 := time.Now() + grouping1 := map[string]string{ + "job": "job1", + "instance": "instance2", + } + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts1, + MetricFamilies: testutil.MetricFamiliesMap(mf1c), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp := newPushTimestampGauge(grouping1, ts1) + pushFailedTimestamp := newPushFailedTimestampGauge(grouping1, time.Time{}) + if err := checkMetricFamilies( + dms, mf1b, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Push mf1e, missing the instance label. Again, mf1b should end up in storage. + ts2 := ts1.Add(1) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts2, + MetricFamilies: testutil.MetricFamiliesMap(mf1e), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp = newPushTimestampGauge(grouping1, ts2) + if err := checkMetricFamilies( + dms, mf1b, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Push mf1e, missing the instance label, into a grouping without the + // instance label. The result in the storage should have an empty + // instance label. + ts3 := ts2.Add(1) + grouping3 := map[string]string{ + "job": "job1", + } + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping3, + Timestamp: ts3, + MetricFamilies: testutil.MetricFamiliesMap(mf1e), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp.Metric = append( + pushTimestamp.Metric, newPushTimestampGauge(grouping3, ts3).Metric[0], + ) + pushFailedTimestamp.Metric = append( + pushFailedTimestamp.Metric, newPushFailedTimestampGauge(grouping3, time.Time{}).Metric[0], + ) + if err := checkMetricFamilies( + dms, mf1be, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } +} + +func TestReplace(t *testing.T) { + dms := NewDiskMetricStore("", 100*time.Millisecond, nil, logger) + + // First do an invalid push to set pushFailedTimestamp and to later + // verify that it is retained and not replaced. + ts1 := time.Now() + grouping1 := map[string]string{ + "job": "job1", + } + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts1, + MetricFamilies: testutil.MetricFamiliesMap(mf1ts), + Done: errCh, + }) + var err error + for err = range errCh { + if !errors.Is(err, errTimestamp) { + t.Errorf("Expected error %q, got %q.", errTimestamp, err) + } + } + if err == nil { + t.Error("Expected error on pushing metric with timestamp.") + } + pushTimestamp := newPushTimestampGauge(grouping1, time.Time{}) + pushFailedTimestamp := newPushFailedTimestampGauge(grouping1, ts1) + if err := checkMetricFamilies( + dms, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Now a valid update in replace mode. It doesn't replace anything, but + // it already tests that the push-failed timestamp is retained. + ts2 := ts1.Add(time.Second) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts2, + MetricFamilies: testutil.MetricFamiliesMap(mf1a), + Done: errCh, + Replace: true, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp = newPushTimestampGauge(grouping1, ts2) + if err := checkMetricFamilies( + dms, mf1a, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Now push something else in replace mode that should replace mf1. + ts3 := ts2.Add(time.Second) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts3, + MetricFamilies: testutil.MetricFamiliesMap(mf2), + Done: errCh, + Replace: true, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp = newPushTimestampGauge(grouping1, ts3) + if err := checkMetricFamilies( + dms, mf2, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Another invalid push in replace mode, which should only update the + // push-failed timestamp. + ts4 := ts3.Add(time.Second) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts4, + MetricFamilies: testutil.MetricFamiliesMap(mf1ts), + Done: errCh, + Replace: true, + }) + err = nil + for err = range errCh { + if !errors.Is(err, errTimestamp) { + t.Errorf("Expected error %q, got %q.", errTimestamp, err) + } + } + if err == nil { + t.Error("Expected error on pushing metric with timestamp.") + } + pushFailedTimestamp = newPushFailedTimestampGauge(grouping1, ts4) + if err := checkMetricFamilies( + dms, mf2, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Push an empty map (rather than a nil map) in replace mode. Should + // delete everything except the push timestamps. + ts5 := ts4.Add(time.Second) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: grouping1, + Timestamp: ts5, + MetricFamilies: testutil.MetricFamiliesMap(), + Done: errCh, + Replace: true, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp = newPushTimestampGauge(grouping1, ts5) + if err := checkMetricFamilies( + dms, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + if err := dms.Shutdown(); err != nil { + t.Fatal(err) + } +} + +func TestGetMetricFamiliesMap(t *testing.T) { + tempDir, err := os.MkdirTemp("", "diskmetricstore.TestGetMetricFamiliesMap.") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + fileName := path.Join(tempDir, "persistence") + + dms := NewDiskMetricStore(fileName, 100*time.Millisecond, nil, logger) + + labels1 := map[string]string{ + "job": "job1", + "instance": "instance1", + } + + labels2 := map[string]string{ + "job": "job1", + "instance": "instance2", + } + + gk1 := groupingKeyFor(labels1) + gk2 := groupingKeyFor(labels2) + + // Submit a single simple metric family. + ts1 := time.Now() + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: labels1, + Timestamp: ts1, + MetricFamilies: testutil.MetricFamiliesMap(mf3), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + pushTimestamp := newPushTimestampGauge(labels1, ts1) + pushFailedTimestamp := newPushFailedTimestampGauge(labels1, time.Time{}) + if err := checkMetricFamilies( + dms, mf3, + pushTimestamp, pushFailedTimestamp, + ); err != nil { + t.Error(err) + } + + // Submit two metric families for a different instance. + ts2 := ts1.Add(time.Second) + errCh = make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: labels2, + Timestamp: ts2, + MetricFamilies: testutil.MetricFamiliesMap(mf1b, mf2), + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + + // expectedMFMap is a multi-layered map that maps the labelset + // fingerprints to the corresponding metric family string + // representations. This is for test assertion purposes. + expectedMFMap := map[string]map[string]string{ + gk1: { + "mf3": mf3.String(), + pushMetricName: pushTimestamp.String(), + pushFailedMetricName: pushFailedTimestamp.String(), + }, + gk2: { + "mf1": mf1b.String(), + "mf2": mf2.String(), + pushMetricName: newPushTimestampGauge(labels2, ts2).String(), + pushFailedMetricName: newPushFailedTimestampGauge(labels2, time.Time{}).String(), + }, + } + + if err := checkMetricFamilyGroups(dms, expectedMFMap); err != nil { + t.Error(err) + } +} + +func TestHelpStringFix(t *testing.T) { + dms := NewDiskMetricStore("", 100*time.Millisecond, prometheus.DefaultGatherer, logger) + + ts1 := time.Now() + errCh := make(chan error, 1) + dms.SubmitWriteRequest(WriteRequest{ + Labels: map[string]string{ + "job": "job1", + }, + Timestamp: ts1, + MetricFamilies: map[string]*dto.MetricFamily{ + "go_goroutines": mfgg, + "mf_help": mfh1, + }, + }) + dms.SubmitWriteRequest(WriteRequest{ + Labels: map[string]string{ + "job": "job2", + }, + Timestamp: ts1, + MetricFamilies: map[string]*dto.MetricFamily{ + "mf_help": mfh2, + }, + Done: errCh, + }) + for err := range errCh { + t.Fatal("Unexpected error:", err) + } + + // Either we have settled on the mfh1 help string or the mfh2 help string. + gotMFs := dms.GetMetricFamilies() + if len(gotMFs) != 4 { + t.Fatalf("expected 4 metric families, got %d", len(gotMFs)) + } + gotMFsAsStrings := make([]string, len(gotMFs)) + for i, mf := range gotMFs { + sort.Sort(metricSorter(mf.GetMetric())) + gotMFsAsStrings[i] = mf.String() + } + sort.Strings(gotMFsAsStrings) + gotGG := gotMFsAsStrings[0] + got12 := gotMFsAsStrings[1] + expectedGG := mfggFixed.String() + expected12 := mfh12.String() + expected21 := mfh21.String() + + if gotGG != expectedGG { + t.Errorf( + "help strings weren't properly adjusted, got '%s', expected '%s'", + gotGG, expectedGG, + ) + } + if got12 != expected12 && got12 != expected21 { + t.Errorf( + "help strings weren't properly adjusted, got '%s' which is neither '%s' nor '%s'", + got12, expected12, expected21, + ) + } + + if err := dms.Shutdown(); err != nil { + t.Fatal(err) + } +} + +func TestGroupingKeyForLabels(t *testing.T) { + sep := string([]byte{model.SeparatorByte}) + scenarios := []struct { + in map[string]string + out string + }{ + { + in: map[string]string{}, + out: "", + }, + { + in: map[string]string{"foo": "bar"}, + out: "foo" + sep + "bar", + }, + { + in: map[string]string{"foo": "bar", "dings": "bums"}, + out: "dings" + sep + "bums" + sep + "foo" + sep + "bar", + }, + } + + for _, s := range scenarios { + if want, got := s.out, groupingKeyFor(s.in); want != got { + t.Errorf("Want grouping key %q for labels %v, got %q.", want, s.in, got) + } + } +} + +func checkMetricFamilies(dms *DiskMetricStore, expectedMFs ...*dto.MetricFamily) error { + gotMFs := dms.GetMetricFamilies() + if expected, got := len(expectedMFs), len(gotMFs); expected != got { + return fmt.Errorf("expected %d metric families, got %d", expected, got) + } + + expectedMFsAsStrings := make([]string, len(expectedMFs)) + for i, mf := range expectedMFs { + sort.Sort(metricSorter(mf.Metric)) + expectedMFsAsStrings[i] = mf.String() + } + sort.Strings(expectedMFsAsStrings) + + gotMFsAsStrings := make([]string, len(gotMFs)) + for i, mf := range gotMFs { + sort.Sort(metricSorter(mf.GetMetric())) + gotMFsAsStrings[i] = mf.String() + } + sort.Strings(gotMFsAsStrings) + + for i, got := range gotMFsAsStrings { + expected := expectedMFsAsStrings[i] + if expected != got { + return fmt.Errorf("expected metric family '%s', got '%s'", expected, got) + } + } + return nil +} + +func checkMetricFamilyGroups(dms *DiskMetricStore, expectedMFMap map[string]map[string]string) error { + mfMap := dms.GetMetricFamiliesMap() + + if expected, got := len(expectedMFMap), len(mfMap); expected != got { + return fmt.Errorf("expected %d metric families in map, but got %d", expected, got) + } + + for k, v := range mfMap { + if innerMap, ok := expectedMFMap[k]; ok { + if len(innerMap) != len(v.Metrics) { + return fmt.Errorf("expected %d metric entries for grouping key %s in map, but got %d", + len(innerMap), k, len(v.Metrics)) + } + for metricName, metricString := range innerMap { + if v.Metrics[metricName].GetMetricFamily().String() != metricString { + return fmt.Errorf("expected metric %s to be present for key %s", metricString, metricName) + } + } + } else { + return fmt.Errorf("expected grouping key %s to be present in metric families map", k) + } + } + return nil +} + +type metricSorter []*dto.Metric + +func (s metricSorter) Len() int { + return len(s) +} + +func (s metricSorter) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s metricSorter) Less(i, j int) bool { + for n, lp := range s[i].Label { + vi := lp.GetValue() + vj := s[j].Label[n].GetValue() + if vi != vj { + return vi < vj + } + } + return true +} diff --git a/storage/interface.go b/storage/interface.go new file mode 100644 index 0000000..ba7e92c --- /dev/null +++ b/storage/interface.go @@ -0,0 +1,177 @@ +//nolint:goheader +// Copyright 2014 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "sort" + "time" + + //nolint:staticcheck // Ignore SA1019. Dependencies use the deprecated package, so we have to, too. + "github.com/golang/protobuf/proto" + + dto "github.com/prometheus/client_model/go" +) + +// MetricStore is the interface to the storage layer for metrics. All its +// methods must be safe to be called concurrently. +type MetricStore interface { + // SubmitWriteRequest submits a WriteRequest for processing. There is no + // guarantee when a request will be processed, but it is guaranteed that + // the requests are processed in the order of submission. + SubmitWriteRequest(req WriteRequest) + // GetMetricFamilies returns all the currently saved MetricFamilies. The + // returned MetricFamilies are guaranteed to not be modified by the + // MetricStore anymore. However, they may still be read somewhere else, + // so the caller is not allowed to modify the returned MetricFamilies. + // If different groups have saved MetricFamilies of the same name, they + // are all merged into one MetricFamily by concatenating the contained + // Metrics. Inconsistent help strings are logged, and one of the + // versions will "win". Inconsistent types and inconsistent or duplicate + // label sets will go undetected. + GetMetricFamilies() []*dto.MetricFamily + // GetMetricFamiliesMap returns a map grouping-key -> MetricGroup. The + // MetricFamily pointed to by the Metrics map in each MetricGroup is + // guaranteed to not be modified by the MetricStore anymore. However, + // they may still be read somewhere else, so the caller is not allowed + // to modify it. Otherwise, the returned nested map can be seen as a + // deep copy of the internal state of the MetricStore and completely + // owned by the caller. + GetMetricFamiliesMap() GroupingKeyToMetricGroup + // Shutdown must only be called after the caller has made sure that + // SubmitWriteRequests is not called anymore. (If it is called later, + // the request might get submitted, but not processed anymore.) The + // Shutdown method waits for the write request queue to empty, then it + // persists the content of the MetricStore (if supported by the + // implementation). Also, all internal goroutines are stopped. This + // method blocks until all of that is complete. If an error is + // encountered, it is returned (whereupon the MetricStorage is in an + // undefinded state). If nil is returned, the MetricStore cannot be + // "restarted" again, but it can still be used for read operations. + Shutdown() error + // Healthy returns nil if the MetricStore is currently working as + // expected. Otherwise, a non-nil error is returned. + Healthy() error + // Ready returns nil if the MetricStore is ready to be used (all files + // are opened and checkpoints have been restored). Otherwise, a non-nil + // error is returned. + Ready() error +} + +// WriteRequest is a request to change the MetricStore, i.e. to process it, a +// write lock has to be acquired. +// +// If MetricFamilies is nil, this is a request to delete metrics that share the +// given Labels as a grouping key. Otherwise, this is a request to update the +// MetricStore with the MetricFamilies. +// +// If Replace is true, the MetricFamilies will completely replace the metrics +// with the same grouping key. Otherwise, only those MetricFamilies with the +// same name as new MetricFamilies will be replaced. +// +// The key in MetricFamilies is the name of the mapped metric family. +// +// When the WriteRequest is processed, the metrics in MetricFamilies will be +// sanitized to have the same job and other labels as those in the Labels +// fields. Also, if there is no instance label, an instance label with an empty +// value will be set. This implies that the MetricFamilies in the WriteRequest +// may be modified be the MetricStore during processing of the WriteRequest! +// +// The Timestamp field marks the time the request was received from the +// network. It is not related to the TimestampMs field in the Metric proto +// message. In fact, WriteRequests containing any Metrics with a TimestampMs set +// are invalid and will be rejected. +// +// The Done channel may be nil. If it is not nil, it will be closed once the +// write request is processed. Any errors occurring during processing are sent to +// the channel before closing it. +type WriteRequest struct { + Labels map[string]string + Timestamp time.Time + MetricFamilies map[string]*dto.MetricFamily + Replace bool + Done chan error +} + +// GroupingKeyToMetricGroup is the first level of the metric store, keyed by +// grouping key. +type GroupingKeyToMetricGroup map[string]MetricGroup + +// MetricGroup adds the grouping labels to a NameToTimestampedMetricFamilyMap. +type MetricGroup struct { + Labels map[string]string + Metrics NameToTimestampedMetricFamilyMap +} + +// SortedLabels returns the label names of the grouping labels sorted +// lexicographically but with the "job" label always first. This method exists +// for presentation purposes, see template.html. +func (mg MetricGroup) SortedLabels() []string { + lns := make([]string, 1, len(mg.Labels)) + lns[0] = "job" + for ln := range mg.Labels { + if ln != "job" { + lns = append(lns, ln) + } + } + sort.Strings(lns[1:]) + return lns +} + +// LastPushSuccess returns false if the automatically added metric for the +// timestamp of the last failed push has a value larger than the value of the +// automatically added metric for the timestamp of the last successful push. In +// all other cases, it returns true (including the case that one or both of +// those metrics are missing for some reason.) +func (mg MetricGroup) LastPushSuccess() bool { + fail := mg.Metrics[pushFailedMetricName].GobbableMetricFamily + if fail == nil { + return true + } + success := mg.Metrics[pushMetricName].GobbableMetricFamily + if success == nil { + return true + } + return (*dto.MetricFamily)(fail).GetMetric()[0].GetGauge().GetValue() <= (*dto.MetricFamily)(success).GetMetric()[0].GetGauge().GetValue() +} + +// NameToTimestampedMetricFamilyMap is the second level of the metric store, +// keyed by metric name. +type NameToTimestampedMetricFamilyMap map[string]TimestampedMetricFamily + +// TimestampedMetricFamily adds the push timestamp to a gobbable version of the +// MetricFamily-DTO. +type TimestampedMetricFamily struct { + Timestamp time.Time + GobbableMetricFamily *GobbableMetricFamily +} + +// GetMetricFamily returns the normal GetMetricFamily DTO (without the gob additions). +func (tmf TimestampedMetricFamily) GetMetricFamily() *dto.MetricFamily { + return (*dto.MetricFamily)(tmf.GobbableMetricFamily) +} + +// GobbableMetricFamily is a dto.MetricFamily that implements GobDecoder and +// GobEncoder. +type GobbableMetricFamily dto.MetricFamily + +// GobDecode implements gob.GobDecoder. +func (gmf *GobbableMetricFamily) GobDecode(b []byte) error { + return proto.Unmarshal(b, (*dto.MetricFamily)(gmf)) +} + +// GobEncode implements gob.GobEncoder. +func (gmf *GobbableMetricFamily) GobEncode() ([]byte, error) { + return proto.Marshal((*dto.MetricFamily)(gmf)) +} diff --git a/testutil/main/main.go b/testutil/main/main.go new file mode 100644 index 0000000..2511fb4 --- /dev/null +++ b/testutil/main/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "net" + "os" + "time" +) + +func main() { + conn, err := net.Dial("unix", "/run/apptheus/gateway.sock") + if err != nil { + fmt.Printf("connection error: %v", err) + os.Exit(-1) + } + + defer conn.Close() + + for { + sum := uint64(0) + for i := 0; i < 10000000; i++ { + sum = sum + uint64(i) + } + uid := os.Getuid() + pid := os.Getpid() + fmt.Printf("calculation completed, my uid: %d, my pid: %d\n", uid, pid) + time.Sleep(time.Second * 5) + } +} diff --git a/testutil/metric_families.go b/testutil/metric_families.go new file mode 100644 index 0000000..2e379d3 --- /dev/null +++ b/testutil/metric_families.go @@ -0,0 +1,41 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testutil + +import ( + //nolint:staticcheck // Ignore SA1019. Dependencies use the deprecated package, so we have to, too. + "github.com/golang/protobuf/proto" + + dto "github.com/prometheus/client_model/go" +) + +// MetricFamiliesMap creates the map needed in the MetricFamilies field of a +// WriteRequest from the provided reference metric families. While doing so, it +// creates deep copies of the metric families so that modifications that might +// happen during processing of the WriteRequest will not affect the reference +// metric families. +func MetricFamiliesMap(mfs ...*dto.MetricFamily) map[string]*dto.MetricFamily { + m := map[string]*dto.MetricFamily{} + for _, mf := range mfs { + buf, err := proto.Marshal(mf) + if err != nil { + panic(err) + } + mfCopy := &dto.MetricFamily{} + if err := proto.Unmarshal(buf, mfCopy); err != nil { + panic(err) + } + m[mf.GetName()] = mfCopy + } + return m +}