From 1cda5d3d15491dab7c6610034ede06f1cfc436d2 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Wed, 28 Feb 2024 15:00:24 -0600 Subject: [PATCH 1/5] Bug 1875338 - Update the clean-up script to manage each package in the product delivery repository I hacked together the clean-up script as fast as I could to mitigate the Nightly channel growing. These changes update the script to mange each Mozilla product package in the product delivery repository. --- README.md | 59 ++-- src/mozilla_linux_pkg_manager/cli.py | 386 +++++++++------------------ 2 files changed, 159 insertions(+), 286 deletions(-) diff --git a/README.md b/README.md index 5aaf572..46a4812 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,7 @@ # mozilla-linux-pkg-manager -`mozilla-releng/mozilla-linux-pkg-manager` is a Python tool for managing Mozilla product packages hosted in Linux software repositories hosted on Google Cloud Platform. -It can be used to clean-up obsolete Firefox Nightly versions. +`mozilla-releng/mozilla-linux-pkg-manager` is a Python tool for managing packages stored in Linux software repositories hosted on Google Cloud Platform. ## Requirements - Python 3.11 or higher @@ -38,22 +37,45 @@ export GOOGLE_CLOUD_PROJECT=[PROJECT_NAME] ### Running `mozilla-linux-pkg-manager` To run `mozilla-linux-pkg-manager`, use Poetry with the following command: ```bash -poetry run mozilla-linux-pkg-manager clean-up [-h] --product PRODUCT --channel CHANNEL --format FORMAT --repository REPOSITORY --region REGION [--retention-days RETENTION_DAYS] [--dry-run] +poetry run mozilla-linux-pkg-manager clean-up [-h] --product PRODUCT --repository REPOSITORY --region REGION --retention-days RETENTION_DAYS [--dry-run] ``` + #### Parameters -- `--product`: Specifies the Mozilla product to manage (e.g. `firefox`, `devedition`, `vpn`). Currently, only `firefox` is supported. -- `--channel`: Specifies the package channel (e.g. `nightly`, `release`, `beta`). Currently, only `nightly` is supported. -- `--format`: The package format (i.e. deb). Currently, only `deb` is supported. -- `--retention-days`: Sets the retention period in days for packages in the nightly channel. This parameter is only supported on the `nightly` channel. +- `--package`: A regular expression matching the name of the packages to clean-up. +- `--retention-days`: Sets the retention period in days for packages that match the `package` regex. - `--dry-run`: Tells the script to do a no-op run and print out a summary of the operations that will be executed. - `--repository`: The repository to perform maintenance operations on. - `--region`: The cloud region the repository is hosted in. -#### Example -To clean up the nightly .deb packages that are older than 7 days: +#### Examples +Clean up firefox and firefox l10n .deb packages that are older than 365 days: +```bash +mozilla-linux-pkg-manager \ +clean-up \ +--package "^firefox(-l10n-.+)?$" \ +--retention-days 365 \ +--repository mozilla \ +--region us +``` + +Clean up firefox-nightly and firefox-nightly l10n .deb packages that are older than a day: +```bash +mozilla-linux-pkg-manager \ +clean-up \ +--package "^firefox-nightly(-l10n-.+)?$" \ +--retention-days 1 \ +--repository mozilla \ +--region us +``` +Clean up firefox-devedition and firefox-devedition l10n .deb packages that are older than 60 days: ```bash -poetry run mozilla-linux-pkg-manager clean-up --product firefox --channel nightly --format deb --retention-days 7 --repository mozilla --region us +mozilla-linux-pkg-manager \ +clean-up \ +--package "^firefox-(devedition|beta)(-l10n-.+)?$" \ +--retention-days 60 \ +--repository mozilla \ +--region us ``` ## Docker @@ -69,9 +91,7 @@ docker run --rm \ -v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/keys/google/key.json:ro \ mozillareleases/mozilla-linux-pkg-manager:0.7.0 \ clean-up \ ---product firefox \ ---channel nightly \ ---format deb \ +--package "^firefox(-l10n-.+)?$" \ --retention-days 3 \ --repository [REPOSITORY] \ --region [REGION] \ @@ -109,12 +129,11 @@ docker run \ -v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/keys/google/key.json:ro \ $IMAGE_NAME \ clean-up \ ---product firefox \ ---channel nightly \ ---format deb \ +--package "^firefox(-l10n-.+)?$" \ --retention-days 3 \ --repository mozilla \ ---region us +--region us \ +--dry-run ``` In this command: @@ -137,9 +156,3 @@ The `mozilla-linux-pkg-manager` package can be packaged into a wheel file for di ### Using the Installed Package After installation, the package can be used from anywhere on your system, provided you are running the Python interpreter where it was installed. -#### Example -To clean up nightly .deb packages that are older than 3 days: - -```bash -mozilla-linux-pkg-manager clean-up --product firefox --channel nightly --format deb --retention-days 3 --repository mozilla --region us -``` diff --git a/src/mozilla_linux_pkg_manager/cli.py b/src/mozilla_linux_pkg_manager/cli.py index 300ba54..8877b18 100644 --- a/src/mozilla_linux_pkg_manager/cli.py +++ b/src/mozilla_linux_pkg_manager/cli.py @@ -2,147 +2,72 @@ import asyncio import logging import os -import random +import json +import re +import time from collections import defaultdict -from collections.abc import Awaitable, Callable, Sequence -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from itertools import islice -from pprint import pformat -from typing import ( - Any, - Optional, - Union, -) -from urllib.parse import urljoin -import aiohttp -import yaml +import requests +import requests.exceptions as requests_exceptions +from google.api_core import exceptions as api_exceptions +from google.api_core import retry_async +from google.auth import exceptions as auth_exceptions from google.cloud import artifactregistry_v1 -from mozilla_version.gecko import GeckoVersion logging.basicConfig( - format="%(asctime)s - mozilla-linux-pkg-manager - %(levelname)s - %(message)s", + format="%(asctime)s - %(funcName)s - %(message)s", level=logging.INFO, ) +RETRYABLE_TYPES = ( + api_exceptions.TooManyRequests, # 429 + api_exceptions.InternalServerError, # 500 + api_exceptions.BadGateway, # 502 + api_exceptions.ServiceUnavailable, # 503 + api_exceptions.GatewayTimeout, # 504 + ConnectionError, + requests.ConnectionError, + requests_exceptions.ChunkedEncodingError, + requests_exceptions.Timeout, +) -def calculate_sleep_time( - attempt, delay_factor=5.0, randomization_factor=0.5, max_delay=120 -): - """Calculate the sleep time between retries, in seconds. - - Based off of `taskcluster.utils.calculateSleepTime`, but with kwargs instead - of constant `delay_factor`/`randomization_factor`/`max_delay`. The taskcluster - function generally slept for less than a second, which didn't always get - past server issues. - Args: - attempt (int): the retry attempt number - delay_factor (float, optional): a multiplier for the delay time. Defaults to 5. - randomization_factor (float, optional): a randomization multiplier for the - delay time. Defaults to .5. - max_delay (float, optional): the max delay to sleep. Defaults to 120 (seconds). - Returns: - float: the time to sleep, in seconds. - """ - if attempt <= 0: - return 0 - - # We subtract one to get exponents: 1, 2, 3, 4, 5, .. - delay = float(2 ** (attempt - 1)) * float(delay_factor) - # Apply randomization factor. Only increase the delay here. - delay = delay * (randomization_factor * random.random() + 1) - # Always limit with a maximum delay - return min(delay, max_delay) - - -async def retry_async( - func: Callable[..., Awaitable[Any]], - attempts: int = 5, - sleeptime_callback: Callable[..., Any] = calculate_sleep_time, - retry_exceptions: Union[ - type[BaseException], tuple[type[BaseException], ...] - ] = Exception, - args: Sequence[Any] = (), - kwargs: Optional[dict[str, Any]] = None, - sleeptime_kwargs: Optional[dict[str, Any]] = None, -) -> Any: - """Retry ``func``, where ``func`` is an awaitable. - - Args: - func (function): an awaitable function. - attempts (int, optional): the number of attempts to make. Default is 5. - sleeptime_callback (function, optional): the function to use to determine - how long to sleep after each attempt. Defaults to ``calculateSleepTime``. - retry_exceptions (list or exception, optional): the exception(s) to retry on. - Defaults to ``Exception``. - args (list, optional): the args to pass to ``func``. Defaults to () - kwargs (dict, optional): the kwargs to pass to ``func``. Defaults to - {}. - sleeptime_kwargs (dict, optional): the kwargs to pass to ``sleeptime_callback``. - If None, use {}. Defaults to None. - Returns: - object: the value from a successful ``function`` call - Raises: - Exception: the exception from a failed ``function`` call, either outside - of the retry_exceptions, or one of those if we pass the max - ``attempts``. - """ - kwargs = kwargs or {} - attempt = 1 - while True: - try: - return await func(*args, **kwargs) - except retry_exceptions: - attempt += 1 - check_number_of_attempts(attempt, attempts, func, "retry_async") - await asyncio.sleep( - define_sleep_time( - sleeptime_kwargs, sleeptime_callback, attempt, func, "retry_async" - ) - ) +# Some retriable errors don't have their own custom exception in api_core. +ADDITIONAL_RETRYABLE_STATUS_CODES = (408,) -def check_number_of_attempts( - attempt: int, attempts: int, func: Callable[..., Any], retry_function_name: str -) -> None: - if attempt > attempts: - logging.warning(f"{retry_function_name}: {func.__name__}: too many retries!") - raise - - -def define_sleep_time( - sleeptime_kwargs: Optional[dict[str, Any]], - sleeptime_callback: Callable[..., int], - attempt: int, - func: Callable[..., Any], - retry_function_name: str, -) -> float: - sleeptime_kwargs = sleeptime_kwargs or {} - sleep_time = sleeptime_callback(attempt, **sleeptime_kwargs) - logging.debug( - "{}: {}: sleeping {} seconds before retry".format( - retry_function_name, func.__name__, sleep_time - ) - ) - return sleep_time +def should_retry(exc): + """Predicate for determining when to retry.""" + if isinstance(exc, RETRYABLE_TYPES): + return True + elif isinstance(exc, api_exceptions.GoogleAPICallError): + return exc.code in ADDITIONAL_RETRYABLE_STATUS_CODES + elif isinstance(exc, auth_exceptions.TransportError): + return should_retry(exc.args[0]) + else: + return False + + +ASYNC_RETRY = retry_async.AsyncRetry(predicate=should_retry) -async def batch_delete_versions(versions, args): +async def batch_delete_versions(targets, args): client = artifactregistry_v1.ArtifactRegistryAsyncClient() - for package in versions: - batches = batched(versions[package], 50) + for package in targets: + batches = batched(targets[package], 50) for batch in batches: logging.info( - f"Deleting {format(len(batch), ',')} expired package versions for {package}." + f"Deleting {format(len(batch), ',')} expired package versions of {package}" ) - if len(batch) < 5: - logging.info(f"versions:\n{pformat(batch)}") request = artifactregistry_v1.BatchDeleteVersionsRequest( parent=package, names=batch, validate_only=args.dry_run, ) - operation = await client.batch_delete_versions(request=request) + operation = await client.batch_delete_versions( + request=request, retry=ASYNC_RETRY + ) await operation.result() @@ -152,7 +77,9 @@ async def get_repository(args): get_repository_request = artifactregistry_v1.GetRepositoryRequest( name=parent, ) - repository = await client.get_repository(request=get_repository_request) + repository = await client.get_repository( + request=get_repository_request, retry=ASYNC_RETRY + ) return repository @@ -168,121 +95,76 @@ def batched(iterable, n): batch = tuple(islice(it, n)) -async def fetch_url(url): - async with aiohttp.ClientSession() as session: - async with session.get(url) as response: - if response.status != 200: - raise Exception(f"Failed to fetch data: HTTP Status {response.status}") - - content = await response.text() - return content - - -def parse_key_value_block(block): - package = {} - for line in block.split("\n"): - if line: - key, value = line.split(": ", 1) - package[key.strip()] = value.strip() - if key == "Version": - version, postfix = value.split("~") - package["Gecko-Version"] = GeckoVersion.parse(version) - if package["Gecko-Version"].is_nightly: - package["Build-ID"] = postfix - package["Moz-Build-Date"] = datetime.strptime( - package["Build-ID"], "%Y%m%d%H%M%S" - ) - else: - package["Build-Number"] = postfix[len("build") :] - return package - - -async def delete_nightly_versions(args): - url = f"https://{args.region}-apt.pkg.dev/projects/{os.environ['GOOGLE_CLOUD_PROJECT']}/dists/{args.repository}" - normalized_url = f"{url}/" if not url.endswith("/") else url - release_url = urljoin(normalized_url, "Release") - logging.info(f"Fetching raw_release_data at {url}") - raw_release_data = await retry_async( - fetch_url, - args=[release_url], - attempts=3, +async def list_packages(repository): + client = artifactregistry_v1.ArtifactRegistryAsyncClient() + request = artifactregistry_v1.ListPackagesRequest( + parent=repository.name, + page_size=1000, ) - parsed_release_data = yaml.safe_load(raw_release_data) - logging.info(f"parsed_release_data:\n{pformat(parsed_release_data)}") - architectures = parsed_release_data["Architectures"].split() - logging.info(f"architectures: {pformat(architectures)}") - package_data_promises = [] - for architecture in architectures: - pkg_url = f"{normalized_url}main/binary-{architecture}/Packages" - package_data_promises.append( - retry_async( - fetch_url, - args=[pkg_url], - attempts=3, - ) - ) - logging.info("Downloading package data...") - package_data_results = await asyncio.gather(*package_data_promises) - logging.info("Package data downloaded") - logging.info("Unpacking package data") - package_data = [] - for architecture, package_data_result in zip(architectures, package_data_results): - logging.info(f"Parsing {architecture} data") - parsed_package_data = [ - yaml.safe_load(raw_package_data) - for raw_package_data in package_data_result.split("\n\n") - ] - package_data.extend(parsed_package_data) - logging.info("Package data unpacked") - logging.info("Loading nightly package data") - firefox_package_data = [] - for package in package_data: - if package and "firefox" in package["Package"]: - version, postfix = package["Version"].split("~") - package["Gecko-Version"] = GeckoVersion.parse(version) - if package["Gecko-Version"].is_nightly: - package["Build-ID"] = postfix - package["Moz-Build-Date"] = datetime.strptime( - package["Build-ID"], "%Y%m%d%H%M%S" - ) - else: - package["Build-Number"] = postfix[len("build") :] - firefox_package_data.append(package) - nightly_package_data = [ - package - for package in firefox_package_data - if package and package["Gecko-Version"].is_nightly - ] - if not nightly_package_data: - logging.info("No nightly_package_data, nothing to do!") + packages = await client.list_packages(request=request, retry=ASYNC_RETRY) + return packages + + +async def list_versions(package): + client = artifactregistry_v1.ArtifactRegistryAsyncClient() + request = artifactregistry_v1.ListVersionsRequest( + parent=package.name, + page_size=1000, + ) + versions = await client.list_versions(request=request, retry=ASYNC_RETRY) + return versions + + +async def clean_up(args): + logging.info("Pinging repository...") + repository = await get_repository(args) + logging.info(f"Found repository: {repository.name}") + packages = await list_packages(repository) + now = datetime.now(UTC) + targets = defaultdict(set) + pattern = re.compile(args.package) + unique_expired_versions = set() + + start = time.time() + + async for package in packages: + name = os.path.basename(package.name) + if pattern.match(name): + logging.info(f"Looking for expired package versions of {name}...") + versions = await list_versions(package) + async for version in versions: + if now - version.create_time > timedelta(days=args.retention_days): + targets[package.name].add(version.name) + unique_expired_versions.add(os.path.basename(version.name)) + + end = time.time() + elapsed = int(end - start) + logging.info( + f"Done. Looked for {elapsed} seconds (that's about ~{elapsed // 60} minutes)" + ) + + if not targets: + logging.info("No expired package versions found, nothing to do!") exit(0) - now = datetime.now() - logging.info("Getting expired packages...") - expired_nightly_packages = [ - package - for package in nightly_package_data - if now - package["Moz-Build-Date"] > timedelta(days=args.retention_days) - ] + + logging.info(f"Found {len(targets)} packages matching {args.package}") logging.info( - f"Found {format(len(expired_nightly_packages), ',')} expired nightly packages. Keeping {format(len(nightly_package_data) - len(expired_nightly_packages), ',')} nightly packages created < {args.retention_days} days ago" + f"There's a total of {sum(len(target) for target in targets.values())} expired versions to clean-up!" ) - targets = defaultdict(set) - for package in expired_nightly_packages: - targets[ - f"projects/{os.environ['GOOGLE_CLOUD_PROJECT']}/locations/{args.region}/repositories/{args.repository}/packages/{package['Package']}" - ].add( - f"projects/{os.environ['GOOGLE_CLOUD_PROJECT']}/locations/{args.region}/repositories/{args.repository}/packages/{package['Package']}/versions/{package['Version']}" + logging.info( + f"Out of those versions, there are {len(unique_expired_versions)} unique versions across all packages." + ) + logging.info(f"Found unique expired versions:\nunique_expired_versions = {json.dumps(list(unique_expired_versions), indent=4)}") + + if args.skip_delete: + logging.info( + 'The skip-delete flag is enabled. Skipping the "delete versions" step!' ) - if not expired_nightly_packages: - logging.info("Nothing to do!") exit(0) - logging.info("Pinging repository...") - repository = await retry_async( - get_repository, - args=[args], - attempts=3, - ) - logging.info(f"Found repository: {repository.name}") + + if args.dry_run: + logging.info("The dry-run mode is enabled. Doing a no-op run!") + await batch_delete_versions(targets, args) @@ -299,21 +181,9 @@ def main(): "clean-up", help="Clean up package versions." ) clean_up_parser.add_argument( - "--product", - type=str, - help="Product in the packages (i.e. firefox)", - required=True, - ) - clean_up_parser.add_argument( - "--channel", + "--package", type=str, - help="Channel of the packages (e.g. nightly, release, beta)", - required=True, - ) - clean_up_parser.add_argument( - "--format", - type=str, - help="The package format (i.e. deb)", + help='The name of the packages to clean-up (ex. "firefox-nightly-*")', required=True, ) clean_up_parser.add_argument( @@ -331,7 +201,8 @@ def main(): clean_up_parser.add_argument( "--retention-days", type=int, - help="Retention period in days for packages in the nightly channel", + required=True, + help="Retention period in days for the selected packages", ) clean_up_parser.add_argument( "--dry-run", @@ -339,27 +210,16 @@ def main(): help="Do a no-op run and print out a summary of the operations that will be executed", default=False, ) + clean_up_parser.add_argument( + "--skip-delete", + action="store_true", + help='Skip the "delete versions" step (for testing)', + default=False, + ) args = parser.parse_args() - - if args.dry_run: - logging.info("The dry-run mode is enabled. Doing a no-op run!") - - logging.info(f"args:\n{pformat(vars(args))}") + logging.info(f"Parsed arguments:\nargs = {json.dumps(vars(args), indent=4)}") if args.command == "clean-up": - if args.product != "firefox": - raise ValueError("firefox is the only supported product") - if args.format != "deb": - raise ValueError("deb is the only supported format") - if args.channel != "nightly": - raise ValueError("nightly is the only supported channel") - if args.channel == "nightly": - if args.retention_days is None: - raise ValueError( - "Retention days must be specified for the nightly channel" - ) - asyncio.run(delete_nightly_versions(args)) - logging.info("Done cleaning up!") - else: - raise ValueError("Only the nightly channel is supported") + asyncio.run(clean_up(args)) + logging.info("Done cleaning up!") From 7cc3bd598db3172cdd65e65978a19d07b0f04f8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 00:38:08 +0000 Subject: [PATCH 2/5] style: pre-commit.ci auto fixes [...] --- README.md | 1 - src/mozilla_linux_pkg_manager/cli.py | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 46a4812..3d4d5d8 100644 --- a/README.md +++ b/README.md @@ -155,4 +155,3 @@ The `mozilla-linux-pkg-manager` package can be packaged into a wheel file for di ### Using the Installed Package After installation, the package can be used from anywhere on your system, provided you are running the Python interpreter where it was installed. - diff --git a/src/mozilla_linux_pkg_manager/cli.py b/src/mozilla_linux_pkg_manager/cli.py index 8877b18..75e6473 100644 --- a/src/mozilla_linux_pkg_manager/cli.py +++ b/src/mozilla_linux_pkg_manager/cli.py @@ -1,8 +1,8 @@ import argparse import asyncio +import json import logging import os -import json import re import time from collections import defaultdict @@ -154,7 +154,9 @@ async def clean_up(args): logging.info( f"Out of those versions, there are {len(unique_expired_versions)} unique versions across all packages." ) - logging.info(f"Found unique expired versions:\nunique_expired_versions = {json.dumps(list(unique_expired_versions), indent=4)}") + logging.info( + f"Found unique expired versions:\nunique_expired_versions = {json.dumps(list(unique_expired_versions), indent=4)}" + ) if args.skip_delete: logging.info( From 952037476d79ab0ac224f20a6362a4086c7658a6 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Wed, 28 Feb 2024 19:15:46 -0600 Subject: [PATCH 3/5] Fix logging and log a tally of the versions that will be left --- src/mozilla_linux_pkg_manager/cli.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/mozilla_linux_pkg_manager/cli.py b/src/mozilla_linux_pkg_manager/cli.py index 75e6473..0b1a81f 100644 --- a/src/mozilla_linux_pkg_manager/cli.py +++ b/src/mozilla_linux_pkg_manager/cli.py @@ -58,7 +58,7 @@ async def batch_delete_versions(targets, args): batches = batched(targets[package], 50) for batch in batches: logging.info( - f"Deleting {format(len(batch), ',')} expired package versions of {package}" + f"Deleting {format(len(batch), ',')} expired package versions of {os.path.basename(package)}" ) request = artifactregistry_v1.BatchDeleteVersionsRequest( parent=package, @@ -124,6 +124,7 @@ async def clean_up(args): targets = defaultdict(set) pattern = re.compile(args.package) unique_expired_versions = set() + all_versions = set() start = time.time() @@ -133,6 +134,7 @@ async def clean_up(args): logging.info(f"Looking for expired package versions of {name}...") versions = await list_versions(package) async for version in versions: + all_versions.add(version.name) if now - version.create_time > timedelta(days=args.retention_days): targets[package.name].add(version.name) unique_expired_versions.add(os.path.basename(version.name)) @@ -148,14 +150,18 @@ async def clean_up(args): exit(0) logging.info(f"Found {len(targets)} packages matching {args.package}") + total_expired_versions = sum(len(target) for target in targets.values()) logging.info( - f"There's a total of {sum(len(target) for target in targets.values())} expired versions to clean-up!" + f"Found unique expired versions:\nunique_expired_versions = {json.dumps(list(unique_expired_versions), indent=4)}" ) logging.info( - f"Out of those versions, there are {len(unique_expired_versions)} unique versions across all packages." + f"There's a total of {total_expired_versions} expired versions to clean-up!" ) logging.info( - f"Found unique expired versions:\nunique_expired_versions = {json.dumps(list(unique_expired_versions), indent=4)}" + f"Out of those expired versions, there are {len(unique_expired_versions)} unique versions across all packages." + ) + logging.info( + f"There's a total of {len(all_versions)} versions. After clean-up, there will be {len(all_versions) - total_expired_versions} versions left." ) if args.skip_delete: From 66dbe6deb07c40a38bd97846959c12fb18d68ba8 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Fri, 1 Mar 2024 12:07:07 -0600 Subject: [PATCH 4/5] Display delete step timer --- src/mozilla_linux_pkg_manager/cli.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/mozilla_linux_pkg_manager/cli.py b/src/mozilla_linux_pkg_manager/cli.py index 0b1a81f..89dda6f 100644 --- a/src/mozilla_linux_pkg_manager/cli.py +++ b/src/mozilla_linux_pkg_manager/cli.py @@ -54,11 +54,12 @@ def should_retry(exc): async def batch_delete_versions(targets, args): client = artifactregistry_v1.ArtifactRegistryAsyncClient() + start = time.time() for package in targets: batches = batched(targets[package], 50) for batch in batches: logging.info( - f"Deleting {format(len(batch), ',')} expired package versions of {os.path.basename(package)}" + f"Deleting {format(len(batch), ',')} expired package versions of {os.path.basename(package)}..." ) request = artifactregistry_v1.BatchDeleteVersionsRequest( parent=package, @@ -69,6 +70,11 @@ async def batch_delete_versions(targets, args): request=request, retry=ASYNC_RETRY ) await operation.result() + end = time.time() + elapsed = int(end - start) + logging.info( + f"Done. Ran delete version requests for {elapsed} seconds (that's about ~{elapsed // 60} minutes.)" + ) async def get_repository(args): @@ -118,7 +124,9 @@ async def list_versions(package): async def clean_up(args): logging.info("Pinging repository...") repository = await get_repository(args) - logging.info(f"Found repository: {repository.name}") + logging.info( + f"Found repository:\nrepository = {json.dumps(artifactregistry_v1.Repository.to_dict(repository), indent=4)}" + ) packages = await list_packages(repository) now = datetime.now(UTC) targets = defaultdict(set) @@ -142,7 +150,7 @@ async def clean_up(args): end = time.time() elapsed = int(end - start) logging.info( - f"Done. Looked for {elapsed} seconds (that's about ~{elapsed // 60} minutes)" + f"Done. Looked for {elapsed} seconds (that's about ~{elapsed // 60} minutes.)" ) if not targets: From 508bd15ec151292053022a0216641f35c960f800 Mon Sep 17 00:00:00 2001 From: Gabriel Bustamante Date: Fri, 1 Mar 2024 13:17:49 -0600 Subject: [PATCH 5/5] Improve logging in delete step --- src/mozilla_linux_pkg_manager/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mozilla_linux_pkg_manager/cli.py b/src/mozilla_linux_pkg_manager/cli.py index 89dda6f..c077e28 100644 --- a/src/mozilla_linux_pkg_manager/cli.py +++ b/src/mozilla_linux_pkg_manager/cli.py @@ -59,7 +59,7 @@ async def batch_delete_versions(targets, args): batches = batched(targets[package], 50) for batch in batches: logging.info( - f"Deleting {format(len(batch), ',')} expired package versions of {os.path.basename(package)}..." + f"{'Would delete' if args.dry_run else 'Deleting'} {format(len(batch), ',')} expired package versions of {os.path.basename(package)}..." ) request = artifactregistry_v1.BatchDeleteVersionsRequest( parent=package,