Add postgresql string

Dockerfile

@@ -68,6 +68,7 @@ RUN groupadd -g "${PGID:-0}" -o fix \
  python3-pip \
  redis-tools \
  mysql-client \
+ postgresql-client \
  && dpkg -i /usr/local/tmp/arangodb3-client_*.deb \
  && ln -s /usr/bin/busybox /usr/local/bin/vi \
  && ln -s /usr/bin/busybox /usr/local/bin/wget \

README.md

@@ -1,21 +1,32 @@
-# `fixbackup` - FIX Database Backup System
+# `fixbackup` - Fix Database Backup and Restore System
+
+A wrapper tool packaged as a container, that runs as a job, creates and restores backups of various databases, and uploads them to an S3 bucket.
+
+## Docker
+
+```
+docker run --it --rm -v /path/to/backups:/backups someengineering/fixbackup --type daily ...
+```
 
 ## Usage
 
 ```bash
-usage: fixbackup [-h] [--backup-directory BACKUP_DIRECTORY] [--verbose | --trace | --quiet] [--s3-bucket S3_BUCKET] --type {daily,weekly,monthly,yearly}
- [--set-lifecycle-policy] [--redis-host REDIS_HOST] [--redis-port REDIS_PORT] [--redis-username REDIS_USERNAME]
- [--redis-password REDIS_PASSWORD] [--redis-database-number REDIS_DATABASE_NUMBER] [--redis-cli-args REDIS_CLI_ARGS]
- [--mysql-host MYSQL_HOST] [--mysql-port MYSQL_PORT] [--mysql-user MYSQL_USER] [--mysql-password MYSQL_PASSWORD]
- [--mysqldump-args MYSQLDUMP_ARGS] [--arangodb-host ARANGODB_HOST] [--arangodb-port ARANGODB_PORT] [--arangodb-username ARANGODB_USERNAME]
- [--arangodb-password ARANGODB_PASSWORD] [--arangodb-database ARANGODB_DATABASE] [--arangodump-args ARANGODUMP_ARGS]
+usage: fixbackup [-h] [--backup-directory BACKUP_DIRECTORY] [-n ENVIRONMENT] [--sleep] [--restore] [--verbose | --trace | --quiet] [--s3-bucket S3_BUCKET] --type {daily,weekly,monthly,yearly} [--set-lifecycle-policy] [--redis-host REDIS_HOST]
+ [--redis-port REDIS_PORT] [--redis-username REDIS_USERNAME] [--redis-password REDIS_PASSWORD] [--redis-database-number REDIS_DATABASE_NUMBER] [--redis-cli-args REDIS_CLI_ARGS] [--redis-tls] [--redis-tls-insecure] [--mysql-host MYSQL_HOST]
+ [--mysql-port MYSQL_PORT] [--mysql-user MYSQL_USER] [--mysql-password MYSQL_PASSWORD] [--mysql-database MYSQL_DATABASE] [--mysqldump-args MYSQLDUMP_ARGS] [--pg-host PG_HOST] [--pg-port PG_PORT] [--pg-user PG_USER] [--pg-password PG_PASSWORD]
+ [--pg-database PG_DATABASE] [--pg-dump-args PG_DUMP_ARGS] [--arangodb-host ARANGODB_HOST] [--arangodb-port ARANGODB_PORT] [--arangodb-username ARANGODB_USERNAME] [--arangodb-password ARANGODB_PASSWORD] [--arangodb-database ARANGODB_DATABASE]
+ [--arangodump-args ARANGODUMP_ARGS] [--arangodb-tls]
 
-FIX Database Backup System
+Fix Database Backup and Restore System
 
 options:
  -h, --help show this help message and exit
  --backup-directory BACKUP_DIRECTORY
  Directory where backups are created
+ -n ENVIRONMENT, --name ENVIRONMENT
+ Name of the environment
+ --sleep Don't do anything, just sleep forever
+ --restore Restore databases from directory
  --verbose, -v Verbose logging
  --trace Trage logging
  --quiet Only log errors
@@ -37,6 +48,8 @@ options:
  Redis database number
  --redis-cli-args REDIS_CLI_ARGS
  Extra arguments to pass to redis-cli
+ --redis-tls Redis uses TLS
+ --redis-tls-insecure Redis uses TLS without verifying the certificate
  --mysql-host MYSQL_HOST
  MySQL host
  --mysql-port MYSQL_PORT
@@ -45,8 +58,19 @@ options:
  MySQL user
  --mysql-password MYSQL_PASSWORD
  MySQL password
+ --mysql-database MYSQL_DATABASE
+ MySQL database
  --mysqldump-args MYSQLDUMP_ARGS
  Extra arguments to pass to mysqldump
+ --pg-host PG_HOST PostgreSQL host
+ --pg-port PG_PORT PostgreSQL port
+ --pg-user PG_USER PostgreSQL user
+ --pg-password PG_PASSWORD
+ PostgreSQL password
+ --pg-database PG_DATABASE
+ PostgreSQL database
+ --pg-dump-args PG_DUMP_ARGS
+ Extra arguments to pass to pg_dump
  --arangodb-host ARANGODB_HOST
  ArangoDB host
  --arangodb-port ARANGODB_PORT
@@ -59,4 +83,5 @@ options:
  ArangoDB database to dump
  --arangodump-args ARANGODUMP_ARGS
  Extra arguments to pass to arangodump
+ --arangodb-tls ArangoDB uses TLS
 ```

fixbackup/__init__.py

@@ -11,4 +11,4 @@
 __author__ = "Some Engineering Inc."
 __license__ = "Apache 2.0"
 __copyright__ = "Copyright © 2023 Some Engineering Inc."
-__version__ = "0.0.7"
+__version__ = "0.0.10"

fixbackup/__main__.py

@@ -1,5 +1,6 @@
 import sys
 import os
+import time
 from pathlib import Path
 from typing import List
 from .logger import add_args as logging_add_args, log
@@ -12,11 +13,24 @@
 def main() -> None:
  args = parse_args([logging_add_args, s3_add_args, *backup_add_args])
  exit_code = 0
- log.info("Starting FIX Databases Backup System")
+ log.info("Starting Fix Databases Backup System")
 
  if not verify_binaries():
  sys.exit(1)
 
+ if args.sleep:
+ # This option is used to keep the container running for debugging purposes.
+ # It allows you to connect to it inside of e.g. a K8s environment
+ # and manually test the backup process. Alternatively, you could
+ # override the entrypoint of the container and sleep indefinitely.
+ log.info("Sleeping forever")
+ try:
+ while True:
+ time.sleep(300)
+ finally:
+ log.info("Shutdown complete")
+ sys.exit(0)
+
  backup_directory = Path(args.backup_directory)
  rmdir_backup_directory = True
  if backup_directory.exists() and not backup_directory.is_dir():

fixbackup/args.py

@@ -4,7 +4,7 @@
 
 
 def parse_args(add_args: List[Callable[[ArgumentParser], None]]) -> Namespace:
- arg_parser = ArgumentParser(prog="fixbackup", description="FIX Database Backup System")
+ arg_parser = ArgumentParser(prog="fixbackup", description="Fix Database Backup and Restore System")
  arg_parser.add_argument(
  "--backup-directory",
  help="Directory where backups are created",
@@ -19,6 +19,20 @@ def parse_args(add_args: List[Callable[[ArgumentParser], None]]) -> Namespace:
  help="Name of the environment",
  default=os.getenv("FIX_ENVIRONMENT", "dev"),
  )
+ arg_parser.add_argument(
+ "--sleep",
+ help="Don't do anything, just sleep forever",
+ dest="sleep",
+ action="store_true",
+ default=False,
+ )
+ arg_parser.add_argument(
+ "--restore",
+ help="Restore databases from directory",
+ dest="restore",
+ action="store_true",
+ default=False,
+ )
 
  for add_arg in add_args:
  add_arg(arg_parser)

fixbackup/backup/__init__.py

@@ -4,10 +4,11 @@
 from typing import List, Tuple
 from .redis import backup as redis_backup, add_args as redis_add_args
 from .mysql import backup as mysql_backup, add_args as mysql_add_args
+from .postgresql import backup as pg_backup, add_args as postgresql_add_args
 from .arangodb import backup as arangodb_backup, add_args as arangodb_add_args
 from ..utils import valid_hostname, valid_ip, valid_dbname
 
-add_args = [redis_add_args, mysql_add_args, arangodb_add_args]
+add_args = [redis_add_args, mysql_add_args, postgresql_add_args, arangodb_add_args]
 
 
 def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]:
@@ -39,14 +40,29 @@ def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]:
  else:
  all_success = False
 
+ if args.pg_host and (valid_hostname(args.pg_host) or valid_ip(args.pg_host)):
+ if args.pg_database:
+ db = str(args.pg_database)
+ if not valid_dbname(db):
+ raise ValueError(f"Invalid database name: {db}")
+ else:
+ db = "all"
+ pg_backup_file = backup_directory / f"{environment}-{date_prefix}-postgresql-{args.pg_host}-{db}.sql.gz"
+ if pg_backup(args, pg_backup_file):
+ result.append(pg_backup_file)
+ else:
+ all_success = False
+
  if args.arangodb_host and (valid_hostname(args.arangodb_host)):
  if args.arangodb_database:
  db = str(args.arangodb_database)
  if not valid_dbname(db):
  raise ValueError(f"Invalid database name: {db}")
  else:
  db = "all"
- arangodb_backup_file = backup_directory / f"{environment}-{date_prefix}-arangodb-{args.arangodb_host}-{db}.tar.gz"
+ arangodb_backup_file = (
+ backup_directory / f"{environment}-{date_prefix}-arangodb-{args.arangodb_host}-{db}.tar.gz"
+ )
  if arangodb_backup(args, arangodb_backup_file):
  result.append(arangodb_backup_file)
  else:

fixbackup/backup/postgresql.py

@@ -0,0 +1,110 @@
+import os
+import subprocess
+from pathlib import Path
+from argparse import ArgumentParser, Namespace
+from ..utils import BackupFile
+from ..logger import log
+
+
+def add_args(arg_parser: ArgumentParser) -> None:
+ arg_parser.add_argument(
+ "--pg-host",
+ help="PostgreSQL host",
+ dest="pg_host",
+ type=str,
+ default=os.getenv("PG_HOST"),
+ )
+
+ arg_parser.add_argument(
+ "--pg-port",
+ help="PostgreSQL port",
+ dest="pg_port",
+ type=int,
+ default=os.getenv("PG_PORT", 5432),
+ )
+
+ arg_parser.add_argument(
+ "--pg-user",
+ help="PostgreSQL user",
+ dest="pg_user",
+ type=str,
+ default=os.getenv("PG_USER", "postgres"),
+ )
+
+ arg_parser.add_argument(
+ "--pg-password",
+ help="PostgreSQL password",
+ dest="pg_password",
+ type=str,
+ default=os.getenv("PG_PASSWORD"),
+ )
+
+ arg_parser.add_argument(
+ "--pg-database",
+ help="PostgreSQL database",
+ dest="pg_database",
+ type=str,
+ default=os.getenv("PG_DATABASE"),
+ )
+
+ arg_parser.add_argument(
+ "--pg-dump-args",
+ help="Extra arguments to pass to pg_dump",
+ dest="pg_dump_args",
+ action="append",
+ default=[],
+ )
+
+
+def backup(args: Namespace, backup_file_path: Path, timeout: int = 900, compress: bool = True) -> bool:
+ log.info("Starting PostgreSQL backup...")
+
+ if not args.pg_host:
+ return False
+
+ env = os.environ.copy()
+ command = [
+ "pg_dump",
+ "-w",
+ "-c",
+ "--if-exists",
+ "--inserts",
+ "-h",
+ str(args.pg_host),
+ "-p",
+ str(args.pg_port),
+ "-U",
+ str(args.pg_user),
+ *args.pg_dump_args,
+ ]
+ if args.pg_database:
+ command.append("-d")
+ command.append(args.pg_database)
+ else:
+ command[0] = "pg_dumpall"
+
+ if args.pg_password:
+ env["PGPASSWORD"] = args.pg_password
+
+ log.debug(f"Running command: {' '.join(command)}")
+
+ try:
+ with BackupFile(backup_file_path, compress) as backup_fd:
+ process = subprocess.Popen(command, stdout=backup_fd, stderr=subprocess.PIPE, env=env)
+ _, stderr = process.communicate(timeout=timeout)
+
+ if process.returncode == 0:
+ log.info(f"PostgreSQL backup completed successfully. Saved to {backup_file_path}")
+ if stderr:
+ log.debug(stderr.decode().strip())
+ return True
+ else:
+ log.error(f"PostgreSQL backup failed with return code: {process.returncode}")
+ if stderr:
+ log.error(stderr.decode().strip())
+ except subprocess.TimeoutExpired:
+ log.error(f"PostgreSQL backup failed with timeout after {timeout} seconds")
+ process.kill()
+ process.communicate()
+
+ return False

fixbackup/logger.py

@@ -126,20 +126,15 @@ def log_to_root(message: str, *args: Any, **kwargs: Any) -> None:
 
 
 class FixLogger(Logger):
- def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None:
- ...
+ def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
- def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None:
- ...
+ def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
- def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None:
- ...
+ def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
- def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None:
- ...
+ def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
- def trace(self, msg: str, *args: Any, **kwargs: Any) -> None:
- ...
+ def trace(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
 
 def get_fix_logger(name: Optional[str] = None) -> FixLogger:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixbackup"
-version = "0.0.7"
+version = "0.0.10"
 authors = [{name="Some Engineering Inc."}]
 description = "FIX Database Backup System"
 license = {file="LICENSE"}