From b1aab5cae7a80e9699c4e7ed131fc3ddcac41513 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 12 Jun 2023 11:18:06 +0100 Subject: [PATCH 01/29] Revert "Pin to Ubuntu 18.04" This reverts commit c9994b48dda2b951781afc078c7c96c82136a796. Ubuntu 18 is no longer available in GH Actions, and also we want to see the failing tests due to the OpenSSL changes. --- .github/workflows/unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index dea79cf1..f7f89548 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: unit-test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: From d590639a7891c4d746ca5f7a19a500d427009199 Mon Sep 17 00:00:00 2001 From: Sae126V Date: Tue, 28 Mar 2023 14:34:34 +0000 Subject: [PATCH 02/29] Update exceptions to catch the timeout errors --- ssm/agents.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ssm/agents.py b/ssm/agents.py index 46e401d7..3d6ef885 100644 --- a/ssm/agents.py +++ b/ssm/agents.py @@ -37,7 +37,8 @@ from stomp.exception import NotConnectedException try: - from argo_ams_library import AmsConnectionException + from argo_ams_library import (AmsConnectionException, AmsTimeoutException, + AmsBalancerException) except ImportError: # ImportError is raised when Ssm2 initialised if AMS is requested but lib # not installed. @@ -328,7 +329,9 @@ def run_receiver(protocol, brokers, project, token, cp, log, dn_file): if protocol == Ssm2.STOMP_MESSAGING: ssm.send_ping() - except (NotConnectedException, AmsConnectionException) as error: + except (NotConnectedException, AmsConnectionException, + AmsTimeoutException, AmsBalancerException) as error: + log.warning('Connection lost.') log.debug(error) ssm.shutdown() From 38f3a24d58042e5deaedf13951954a643d174295 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 14:51:35 +0000 Subject: [PATCH 03/29] Bump docker/build-push-action from 4.1.0 to 4.1.1 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4.1.0...v4.1.1) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index af0dbbd8..6190ce31 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -48,7 +48,7 @@ jobs: # Build and push Docker image # https://github.com/docker/build-push-action name: Build and push Docker image - uses: docker/build-push-action@v4.1.0 + uses: docker/build-push-action@v4.1.1 with: # Only push containers to the registry on GitHub pushes, # not pull requests. GitHub won't let a rogue PR create a container From fd0e5d2d59a54b8582eb9e575c9873676274542f Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 29 Jun 2023 08:51:43 +0100 Subject: [PATCH 04/29] Update version numbers for 3.3.0 --- apel-ssm.spec | 2 +- scripts/ssm-build-deb.sh | 2 +- scripts/ssm-build-rpm.sh | 2 +- ssm/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index 08f5b06c..4d95a324 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -4,7 +4,7 @@ %endif Name: apel-ssm -Version: 3.2.1 +Version: 3.3.0 %define releasenumber 1 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index f5a070d6..1a4db00c 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=3.2.1-1 +TAG=3.3.0-1 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index d6767749..e99d3725 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=3.2.1-1 +VERSION=3.3.0-1 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS diff --git a/ssm/__init__.py b/ssm/__init__.py index d5a80aac..74f42008 100644 --- a/ssm/__init__.py +++ b/ssm/__init__.py @@ -19,7 +19,7 @@ import logging import sys -__version__ = (3, 2, 1) +__version__ = (3, 3, 0) LOG_BREAK = '========================================' From 16479e7a67ca8a64d3799d89d9038993f6153f4e Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 29 Jun 2023 10:39:53 +0100 Subject: [PATCH 05/29] Update changelogs for 3.3.0 --- CHANGELOG | 11 +++++++++++ apel-ssm.spec | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 60e3edc0..4a74b5f0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,16 @@ Changelog for ssm ================= +* Thu Jun 29 2023 Adrian Coveney - 3.3.0-1 + - Added destination queue to the log during startup to aid troubleshooting. + - Added check that the config file exists to allow for better error messages. + - Changed dependencies to limit python-ldap below 3.4.0 and python-daemon below 2.3.0. + - Changed rpmbuild config to use less OS-specific dependencies. + - Fixed read timeouts leading to a crash. + - Fixed command line to allow a relative file path for the DNs file. + - Removed the separate logging config file. + - Removed python-daemon as a hard requirement as only needed for receivers. + - Refactored a large amount of code for maintainability and security. + * Wed Apr 28 2021 Adrian Coveney - 3.2.1-1 - Changed certificate and key comparison to allow both RSA and EC keys. - Corrected dependencies to include OpenSSL. diff --git a/apel-ssm.spec b/apel-ssm.spec index 4d95a324..416007ae 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -100,6 +100,17 @@ rm -rf $RPM_BUILD_ROOT %doc %_defaultdocdir/%{name} %changelog +* Thu Jun 29 2023 Adrian Coveney - 3.3.0-1 + - Added destination queue to the log during startup to aid troubleshooting. + - Added check that the config file exists to allow for better error messages. + - Changed dependencies to limit python-ldap below 3.4.0 and python-daemon below 2.3.0. + - Changed rpmbuild config to use less OS-specific dependencies. + - Fixed read timeouts leading to a crash. + - Fixed command line to allow a relative file path for the DNs file. + - Removed the separate logging config file. + - Removed python-daemon as a hard requirement as only needed for receivers. + - Refactored a large amount of code for maintainability and security. + * Wed Apr 28 2021 Adrian Coveney - 3.2.1-1 - Changed certificate and key comparison to allow both RSA and EC keys. - Corrected dependencies to include OpenSSL. From 7e8afc30aa2552d5807a14691dd6a59298535835 Mon Sep 17 00:00:00 2001 From: Adrian Coveney <4836233+tofu-rocketry@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:59:40 +0100 Subject: [PATCH 06/29] Apply suggestions from code review Clarify that it's the program's command line _arguments_ that have been fixed. --- CHANGELOG | 2 +- apel-ssm.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 4a74b5f0..b0a161aa 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,7 +6,7 @@ Changelog for ssm - Changed dependencies to limit python-ldap below 3.4.0 and python-daemon below 2.3.0. - Changed rpmbuild config to use less OS-specific dependencies. - Fixed read timeouts leading to a crash. - - Fixed command line to allow a relative file path for the DNs file. + - Fixed command line arguments to allow a relative file path for the DNs file. - Removed the separate logging config file. - Removed python-daemon as a hard requirement as only needed for receivers. - Refactored a large amount of code for maintainability and security. diff --git a/apel-ssm.spec b/apel-ssm.spec index 416007ae..5be15b28 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -106,7 +106,7 @@ rm -rf $RPM_BUILD_ROOT - Changed dependencies to limit python-ldap below 3.4.0 and python-daemon below 2.3.0. - Changed rpmbuild config to use less OS-specific dependencies. - Fixed read timeouts leading to a crash. - - Fixed command line to allow a relative file path for the DNs file. + - Fixed command line arguments to allow a relative file path for the DNs file. - Removed the separate logging config file. - Removed python-daemon as a hard requirement as only needed for receivers. - Refactored a large amount of code for maintainability and security. From fba8f511a54e6ff7b4064006c6221b8a4d10304e Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:20:33 +0100 Subject: [PATCH 07/29] Fix logo URLs and files have moved --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 47762b75..fb890868 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL ## Acknowledgements - STFC logo - EU flag - EOSC-hub logo + STFC logo + EU flag + EOSC-hub logo SSM is provided by [STFC](https://stfc.ukri.org/), a part of [UK Research and Innovation](https://www.ukri.org/), and is co-funded by the [EOSC-hub](https://www.eosc-hub.eu/) project (Horizon 2020) under Grant number 777536. Licensed under the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0). From ccbcfd42b13b6546ad2bc074a86742e781a99d04 Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:29:22 +0100 Subject: [PATCH 08/29] Remove mention of RHEL 6 from README No longer pacakged for RHEL 6 --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index fb890868..62e3991e 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,7 @@ using the [STOMP protocol](http://stomp.github.io/) or via the ARGO Messaging Se Messages are signed and may be encrypted during transit. Persistent queues should be used to guarantee delivery. -SSM is written in Python. Packages are available for RHEL 6 and 7, and - Ubuntu Trusty. +SSM is written in Python. Packages are available for RHEL 7, and Ubuntu Trusty. For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL/SSM). From 3a870cd360f357b66350e93ea192c59322bcdddf Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:35:15 +0100 Subject: [PATCH 09/29] Remove EOSC-hub acknowledgement and logos --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62e3991e..cf4c6d68 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,10 @@ For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL ## Acknowledgements - STFC logo - EU flag - EOSC-hub logo + STFC logo -SSM is provided by [STFC](https://stfc.ukri.org/), a part of [UK Research and Innovation](https://www.ukri.org/), and is co-funded by the [EOSC-hub](https://www.eosc-hub.eu/) project (Horizon 2020) under Grant number 777536. Licensed under the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0). +SSM is provided by [STFC](https://stfc.ukri.org/), a part of [UK Research and Innovation](https://www.ukri.org/). Licensed under the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0). ## Installing the RPM From cee0d9e18997b0581f94d69304246b9b2056307a Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 13:17:14 +0100 Subject: [PATCH 10/29] Add LDAP version requirment to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf4c6d68..fb647589 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ The Python STOMP library (N.B. versions between 3.1.1 (inclusive) and 5.0.0 The Python AMS library. This is only required if you want to use AMS. See here for details on obtaining an RPM: https://github.com/ARGOeu/argo-ams-library/ -The Python ldap library +The Python ldap library (N.B. versions before 3.4.0 (exclusive) are currently supported) * `yum install python-ldap` Optionally, the Python dirq library (N.B. this is only required if your messages From cb8957ad27ee50a327a0cf928bc4b3438253ff4c Mon Sep 17 00:00:00 2001 From: DanielPerkins7 Date: Tue, 1 Aug 2023 08:19:56 +0000 Subject: [PATCH 11/29] redundant line and comment removed resolve #236 --- ssm/agents.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ssm/agents.py b/ssm/agents.py index 3d6ef885..673a9906 100644 --- a/ssm/agents.py +++ b/ssm/agents.py @@ -141,10 +141,6 @@ def get_ssm_args(protocol, cp, log): elif protocol == Ssm2.AMS_MESSAGING: # Then we are setting up an SSM to connect to a AMS. - # TODO: See if setting use_ssl directly in Ssm2 constructor is ok. - # 'use_ssl' isn't checked when using AMS (SSL is always used), but it - # is needed for the call to the Ssm2 constructor below. - use_ssl = None try: # We only need a hostname, not a port host = cp.get('broker', 'host') From 83e33c783c0bc2c15fb801408a0a7b1f0c3cb02f Mon Sep 17 00:00:00 2001 From: rowan04 Date: Tue, 18 Jul 2023 14:42:17 +0000 Subject: [PATCH 12/29] Update logging in test_crypto - removes old comment - gets rid of logging.getLogger('SSM') --- test/test_crypto.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_crypto.py b/test/test_crypto.py index db13fdbb..0dcb4001 100644 --- a/test/test_crypto.py +++ b/test/test_crypto.py @@ -17,9 +17,7 @@ verify_cert, \ CryptoException -# Set up logging - is this necessary? logging.basicConfig() -log = logging.getLogger('SSM') class TestEncryptUtils(unittest.TestCase): From 59f0f0ae18ddc632e42f758dfb81e146553e565c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 06:42:14 +0000 Subject: [PATCH 13/29] Bump actions/upload-artifact from 3.1.2 to 3.1.3 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3.1.2...v3.1.3) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build-pkgs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-pkgs.yml b/.github/workflows/build-pkgs.yml index a48e35ea..d2b1232b 100644 --- a/.github/workflows/build-pkgs.yml +++ b/.github/workflows/build-pkgs.yml @@ -51,7 +51,7 @@ jobs: run: rpmlint ${{ steps.rpm.outputs.rpm_dir_path }} - name: Upload artifact - uses: actions/upload-artifact@v3.1.2 + uses: actions/upload-artifact@v3.1.3 with: name: Binary and Source RPMs path: | From aad7194f26f56c5c79a1ff39c00406d2d285950f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:08:46 +0000 Subject: [PATCH 14/29] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-pkgs.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/unit-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pkgs.yml b/.github/workflows/build-pkgs.yml index d2b1232b..43589e31 100644 --- a/.github/workflows/build-pkgs.yml +++ b/.github/workflows/build-pkgs.yml @@ -21,7 +21,7 @@ jobs: run: | sudo apt-get update sudo apt-get install rpmlint - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Get all branches and tags so the latest tag can be found for VERSION fetch-depth: 0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 29501c12..37232599 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index f7f89548..cb3048c0 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -11,7 +11,7 @@ jobs: python-version: ['2.x', '3.x'] name: Python ${{ matrix.python-version }} test steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: From 7594e5ca898f5f2f496e8199f8bb0064af891ac3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:11:41 +0000 Subject: [PATCH 15/29] Bump docker/build-push-action from 4.1.1 to 5.0.0 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4.1.1 to 5.0.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4.1.1...v5.0.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6190ce31..4f016aa2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -48,7 +48,7 @@ jobs: # Build and push Docker image # https://github.com/docker/build-push-action name: Build and push Docker image - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v5.0.0 with: # Only push containers to the registry on GitHub pushes, # not pull requests. GitHub won't let a rogue PR create a container From 1d2d622a76fc4b8a2e69a1a1d5645ac2301e8e96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:58:14 +0000 Subject: [PATCH 16/29] Bump docker/metadata-action from 4 to 5 Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 4 to 5. - [Release notes](https://github.com/docker/metadata-action/releases) - [Upgrade guide](https://github.com/docker/metadata-action/blob/master/UPGRADE.md) - [Commits](https://github.com/docker/metadata-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f016aa2..1bfbc292 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -40,7 +40,7 @@ jobs: # https://github.com/docker/metadata-action name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} From 0ec0b316e07b835670f1f3f8b4e7288cf2cd2fb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 08:05:24 +0000 Subject: [PATCH 17/29] Bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1bfbc292..59f06ebc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: # Login against a Docker registry # https://github.com/docker/login-action name: Login to ${{ env.REGISTRY }} - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} From 7b2011589499920f093611345ee59adb18f3ff5c Mon Sep 17 00:00:00 2001 From: DanielPerkins7 Date: Tue, 1 Aug 2023 08:44:13 +0000 Subject: [PATCH 18/29] else case added to issue 242 resolve #242 --- ssm/ssm2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 72099ad8..4d3a0e79 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -392,6 +392,9 @@ def _send_msg_ams(self, text, msgid): argo_response = self._ams.publish(self._dest, message, retry=3) return argo_response['messageIds'][0] + else: + return None + def pull_msg_ams(self): """Pull 1 message from the AMS and acknowledge it.""" From 2cef8c9a7ddabb32afe187a107c431fa84e304ec Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 18 Sep 2023 11:14:46 +0100 Subject: [PATCH 19/29] Add explanatory comment --- ssm/ssm2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 4d3a0e79..51efdfa8 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -393,6 +393,8 @@ def _send_msg_ams(self, text, msgid): argo_response = self._ams.publish(self._dest, message, retry=3) return argo_response['messageIds'][0] else: + # We ignore empty messages as there is no point sending them. + # (STOMP did require empty messages to keep the connection alive.) return None From b248d7a57d88d03776135a319fda64466459dca2 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 18 Sep 2023 11:19:51 +0100 Subject: [PATCH 20/29] Remove extra line --- ssm/ssm2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 51efdfa8..21e225b7 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -397,7 +397,6 @@ def _send_msg_ams(self, text, msgid): # (STOMP did require empty messages to keep the connection alive.) return None - def pull_msg_ams(self): """Pull 1 message from the AMS and acknowledge it.""" if self._protocol != Ssm2.AMS_MESSAGING: From 2034e650dc7a5e58608f355206cb09db2fd2a0a0 Mon Sep 17 00:00:00 2001 From: Nicholas Whyatt Date: Mon, 18 Sep 2023 08:38:12 +0000 Subject: [PATCH 21/29] Minor timeout changes Add timeout to prevent TCP spinlocks --- ssm/ssm2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 21e225b7..9cbc28e2 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -390,7 +390,7 @@ def _send_msg_ams(self, text, msgid): message = AmsMessage(data=to_send, attributes={'empaid': msgid}).dict() - argo_response = self._ams.publish(self._dest, message, retry=3) + argo_response = self._ams.publish(self._dest, message, retry=3, timeout=10) return argo_response['messageIds'][0] else: # We ignore empty messages as there is no point sending them. @@ -415,7 +415,8 @@ def pull_msg_ams(self): for msg_ack_id, msg in self._ams.pull_sub(self._listen, messages_to_pull, - retry=3): + retry=3, + timeout=10): # Get the AMS message id msgid = msg.get_msgid() # Get the SSM dirq id @@ -444,7 +445,7 @@ def pull_msg_ams(self): # it can move the offset for the next subscription pull # (basically acknowledging pulled messages) if ackids: - self._ams.ack_sub(self._listen, ackids, retry=3) + self._ams.ack_sub(self._listen, ackids, retry=3, timeout=10) def send_ping(self): """Perform connection stay-alive steps. From e2fa9f4b336992dbf7506dc7e1e679749f1c89f4 Mon Sep 17 00:00:00 2001 From: RedProkofiev Date: Mon, 18 Sep 2023 10:41:02 +0000 Subject: [PATCH 22/29] Added warning message to LDAP initialisation. --- ssm/brokers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ssm/brokers.py b/ssm/brokers.py index 598824e0..e6013db6 100644 --- a/ssm/brokers.py +++ b/ssm/brokers.py @@ -43,6 +43,8 @@ class StompBrokerGetter(object): def __init__(self, bdii_url): """Set up the LDAP connection and strings which are re-used.""" # Set up the LDAP connection + logging.warning('LDAP is deprecated and will be removed in an upcoming version, ' + + 'migrate to AMS.') log.debug('Connecting to %s...', bdii_url) self._ldap_conn = ldap.initialize(bdii_url) @@ -99,7 +101,7 @@ def _get_broker_details(self, service_type): return broker_details def _broker_in_network(self, broker_id, network): - """Check that a GlueServiceUniqueID is part of a specified netowrk.""" + """Check that a GlueServiceUniqueID is part of a specified network.""" ldap_filter = '(&(GlueServiceDataKey=cluster)(GlueChunkKey=GlueServiceUniqueID=%s))' \ % broker_id attrs = [self._service_data_value_key] From 961c652fea995d054e9327a9f99ef4e97860a100 Mon Sep 17 00:00:00 2001 From: RedProkofiev <87487335+RedProkofiev@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:30:36 +0100 Subject: [PATCH 23/29] Update ssm/brokers.py Change wording of warning message Co-authored-by: Adrian Coveney <4836233+tofu-rocketry@users.noreply.github.com> --- ssm/brokers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ssm/brokers.py b/ssm/brokers.py index e6013db6..28826c34 100644 --- a/ssm/brokers.py +++ b/ssm/brokers.py @@ -43,8 +43,8 @@ class StompBrokerGetter(object): def __init__(self, bdii_url): """Set up the LDAP connection and strings which are re-used.""" # Set up the LDAP connection - logging.warning('LDAP is deprecated and will be removed in an upcoming version, ' + - 'migrate to AMS.') + logging.warning('LDAP is deprecated and will be removed in an upcoming version, ' + 'please set host locally in SSM config.') log.debug('Connecting to %s...', bdii_url) self._ldap_conn = ldap.initialize(bdii_url) From 261945025cec9935dec1cef4e4d6433e0d1d67cb Mon Sep 17 00:00:00 2001 From: gregcorbett Date: Mon, 18 Sep 2023 16:01:22 +0100 Subject: [PATCH 24/29] Cap version of certifi used below 2020.4.5.2 (#264) Cap version of certifi used below 2020.4.5.2 2020.4.5.2 dropped Python 2 support: - https://github.com/certifi/python-certifi/commit/5efdd48f719d9c3c7c8f9a812da2256d088eab78 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 23c85f7a..e2c93c97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Base requirements for ssm argo-ams-library +certifi<2020.4.5.2 # Used by AMS (via requests), 2020.4.5.2 dropped support for Python 2 stomp.py<5.0.0 python-daemon<=2.3.0 # 2.3.1 dropped support for Python 2 python-ldap<3.4.0 # python-ldap-3.4.0 dropped support for Python 2 From 6f7a21c47ff3baf105ed35b63c874be20842651a Mon Sep 17 00:00:00 2001 From: Rose <83274393+RoseECooper@users.noreply.github.com> Date: Mon, 18 Sep 2023 16:22:34 +0100 Subject: [PATCH 25/29] Add sys.exit() to both sender and receiver (#263) * Add sys.exit(1) to both sender and receiver. Return 1 instead of 0 when failing due to an exception * Added a flag to both sender and receiver to return true when an exception is thrown. Moved sys.exit() to the end of the sender and receiver functions. --- ssm/agents.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ssm/agents.py b/ssm/agents.py index 673a9906..07ebeed7 100644 --- a/ssm/agents.py +++ b/ssm/agents.py @@ -250,6 +250,9 @@ def run_sender(protocol, brokers, project, token, cp, log): print('SSM failed to complete successfully. See log file for details.') log.error('Unexpected exception in SSM: %s', e) log.error('Exception type: %s', e.__class__) + sender_failed = True + else: + sender_failed = False try: sender.close_connection() @@ -259,6 +262,8 @@ def run_sender(protocol, brokers, project, token, cp, log): log.info('SSM has shut down.') log.info(LOG_BREAK) + if sender_failed: + sys.exit(1) def run_receiver(protocol, brokers, project, token, cp, log, dn_file): @@ -344,15 +349,24 @@ def run_receiver(protocol, brokers, project, token, cp, log, dn_file): log.info('Received the shutdown signal: %s', e) ssm.shutdown() dc.close() + receiver_failed = True except Exception as e: log.error('Unexpected exception: %s', e) log.error('Exception type: %s', e.__class__) log.error('The SSM will exit.') ssm.shutdown() dc.close() + receiver_failed = True + # Currently won't run the else statement due to the while loop in the reciever + # Leaving here in case of future refactoring, but commented out so the unreachable + # code isn't flagged by tests + # else: + # receiver_failed = False log.info('Receiving SSM has shut down.') log.info(LOG_BREAK) + if receiver_failed: + sys.exit(1) def get_dns(dn_file, log): From 835ac6e25fd2db81308704992ea406bda0e59a4d Mon Sep 17 00:00:00 2001 From: gregcorbett Date: Mon, 18 Sep 2023 16:12:16 +0100 Subject: [PATCH 26/29] Fix copy/paste error in receiver docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fb647589..7cf98716 100644 --- a/README.md +++ b/README.md @@ -208,8 +208,8 @@ add your messages using the `add` method. ``` docker run \ -d --entrypoint ssmreceive \ - -v /path/to/downloaded/config/sender.cfg:/etc/apel/sender.cfg \ - -v /path/to/read/messages:/var/spool/apel/outgoing \ + -v /path/to/downloaded/config/receiver.cfg:/etc/apel/receiver.cfg \ + -v /path/to/read/messages:/var/spool/apel/ \ -v /path/to/dns/file:/etc/apel/dns \ -v /etc/grid-security:/etc/grid-security \ -v /path/to/persistently/log:/var/log/apel \ From dfa3db5cc9bba2df007093a335618b5f8b55ad17 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 26 Sep 2023 09:55:32 +0100 Subject: [PATCH 27/29] Remove Py 2 from GH Action CI run Python 2 is not supported on GH runners --- .github/workflows/unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index cb3048c0..6d58f556 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['2.x', '3.x'] + python-version: ['3.x'] name: Python ${{ matrix.python-version }} test steps: - uses: actions/checkout@v4 From 15fc9a2d48a7e00b2b68020466351133ceb0b524 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 26 Sep 2023 12:34:07 +0100 Subject: [PATCH 28/29] Update pre-commit repo and add a couple of checks --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f80092a2..6305e79c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,7 @@ repos: - id: check-added-large-files - id: check-merge-conflict - id: check-yaml + - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending name: Force line endings to LF @@ -22,8 +23,9 @@ repos: - id: trailing-whitespace - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 + rev: v1.10.0 hooks: - id: python-check-mock-methods - id: python-no-eval - id: python-no-log-warn + - id: python-use-type-annotations From 1204fcc91179564d5816b2dc1c7a7d9595b73da7 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 26 Sep 2023 12:58:10 +0100 Subject: [PATCH 29/29] Add pre-commit CI config Disables autofixes on PRs and reduces autoupdate frequency (from weekly) as at the moment we have to use the versions specified for Py2 compatability. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6305e79c..86c0ad80 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,3 +29,8 @@ repos: - id: python-no-eval - id: python-no-log-warn - id: python-use-type-annotations + +# Pre-commit CI config, see https://pre-commit.ci/ +ci: + autofix_prs: false + autoupdate_schedule: quarterly