From fa887e1b5bdfffe4b03a752602afc433dad8ebe3 Mon Sep 17 00:00:00 2001 From: Jannik Date: Thu, 28 May 2020 14:26:56 +0200 Subject: [PATCH 1/3] exclude .venv dir from flake8 run --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 272ab66..a39f639 100644 --- a/tox.ini +++ b/tox.ini @@ -53,4 +53,4 @@ norecursedirs = tests/* .venv/* .tox/* build/ docs/ ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503 show-source = True max-line-length = 120 -exclude = .tox,env,tests,build,conf.py +exclude = .tox,env,tests,build,conf.py,.venv From fbe77a10c0283211a920e5f7e0917406f54803b7 Mon Sep 17 00:00:00 2001 From: Jannik Date: Thu, 28 May 2020 17:34:22 +0200 Subject: [PATCH 2/3] Use docker exec api for database backups --- src/restic_compose_backup/cli.py | 4 +- src/restic_compose_backup/commands.py | 48 ++++++++----- src/restic_compose_backup/containers_db.py | 78 +++++++++++----------- src/restic_compose_backup/restic.py | 43 +++++++++--- src/restic_compose_backup/utils.py | 3 +- 5 files changed, 109 insertions(+), 67 deletions(-) diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index fb8cbce..c6e3793 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -105,10 +105,10 @@ def status(config, containers): logger.info( ' - %s (is_ready=%s) -> %s', instance.container_type, - ping == 0, + ping, instance.backup_destination_path(), ) - if ping != 0: + if not ping: logger.error("Database '%s' in service %s cannot be reached", instance.container_type, container.service_name) diff --git a/src/restic_compose_backup/commands.py b/src/restic_compose_backup/commands.py index cf209cb..b1c5558 100644 --- a/src/restic_compose_backup/commands.py +++ b/src/restic_compose_backup/commands.py @@ -1,7 +1,9 @@ import logging -from typing import List, Tuple +from typing import List, Tuple, Union from subprocess import Popen, PIPE +from restic_compose_backup import utils + logger = logging.getLogger(__name__) @@ -9,37 +11,33 @@ def test(): return run(['ls', '/volumes']) -def ping_mysql(host, port, username) -> int: +def ping_mysql(container_id, host, port, username, password) -> int: """Check if the mysql is up and can be reached""" - return run([ + return docker_exec(container_id, [ 'mysqladmin', 'ping', - '--host', - host, - '--port', - port, '--user', username, - ]) + ], environment={ + 'MYSQL_PWD': password + }) -def ping_mariadb(host, port, username) -> int: +def ping_mariadb(container_id, host, port, username, password) -> int: """Check if the mariadb is up and can be reached""" - return run([ + return docker_exec(container_id, [ 'mysqladmin', 'ping', - '--host', - host, - '--port', - port, '--user', username, - ]) + ], environment={ + 'MYSQL_PWD': password + }) -def ping_postgres(host, port, username, password) -> int: +def ping_postgres(container_id, host, port, username, password) -> int: """Check if postgres can be reached""" - return run([ + return docker_exec(container_id, [ "pg_isready", f"--host={host}", f"--port={port}", @@ -47,6 +45,22 @@ def ping_postgres(host, port, username, password) -> int: ]) +def docker_exec(container_id: str, cmd: List[str], environment: Union[dict, list] = []) -> int: + """Execute a command within the given container""" + client = utils.docker_client() + logger.debug('docker exec inside %s: %s', container_id, ' '.join(cmd)) + exit_code, (stdout, stderr) = client.containers.get(container_id).exec_run(cmd, demux=True, environment=environment) + + if stdout: + log_std('stdout', stdout.decode(), + logging.DEBUG if exit_code == 0 else logging.ERROR) + + if stderr: + log_std('stderr', stderr.decode(), logging.ERROR) + + return exit_code + + def run(cmd: List[str]) -> int: """Run a command with parameters""" logger.debug('cmd: %s', ' '.join(cmd)) diff --git a/src/restic_compose_backup/containers_db.py b/src/restic_compose_backup/containers_db.py index ea9c66e..114a67f 100644 --- a/src/restic_compose_backup/containers_db.py +++ b/src/restic_compose_backup/containers_db.py @@ -25,20 +25,19 @@ class MariadbContainer(Container): """Check the availability of the service""" creds = self.get_credentials() - with utils.environment('MYSQL_PWD', creds['password']): - return commands.ping_mariadb( - creds['host'], - creds['port'], - creds['username'], - ) + return commands.ping_mariadb( + self.id, + creds['host'], + creds['port'], + creds['username'], + creds['password'] + ) == 0 def dump_command(self) -> list: """list: create a dump command restic and use to send data through stdin""" creds = self.get_credentials() return [ "mysqldump", - f"--host={creds['host']}", - f"--port={creds['port']}", f"--user={creds['username']}", "--all-databases", ] @@ -47,12 +46,15 @@ class MariadbContainer(Container): config = Config() creds = self.get_credentials() - with utils.environment('MYSQL_PWD', creds['password']): - return restic.backup_from_stdin( - config.repository, - self.backup_destination_path(), - self.dump_command(), - ) + return restic.backup_from_stdin( + config.repository, + self.backup_destination_path(), + self.id, + self.dump_command(), + environment={ + 'MYSQL_PWD': creds['password'] + } + ) def backup_destination_path(self) -> str: destination = Path("/databases") @@ -84,20 +86,19 @@ class MysqlContainer(Container): """Check the availability of the service""" creds = self.get_credentials() - with utils.environment('MYSQL_PWD', creds['password']): - return commands.ping_mysql( - creds['host'], - creds['port'], - creds['username'], - ) + return commands.ping_mysql( + self.id, + creds['host'], + creds['port'], + creds['username'], + creds['password'] + ) == 0 def dump_command(self) -> list: """list: create a dump command restic and use to send data through stdin""" creds = self.get_credentials() return [ "mysqldump", - f"--host={creds['host']}", - f"--port={creds['port']}", f"--user={creds['username']}", "--all-databases", ] @@ -106,12 +107,15 @@ class MysqlContainer(Container): config = Config() creds = self.get_credentials() - with utils.environment('MYSQL_PWD', creds['password']): - return restic.backup_from_stdin( - config.repository, - self.backup_destination_path(), - self.dump_command(), - ) + return restic.backup_from_stdin( + config.repository, + self.backup_destination_path(), + self.id, + self.dump_command(), + environment={ + "MYSQL_PWD": creds['password'] + } + ) def backup_destination_path(self) -> str: destination = Path("/databases") @@ -144,11 +148,12 @@ class PostgresContainer(Container): """Check the availability of the service""" creds = self.get_credentials() return commands.ping_postgres( + self.id, creds['host'], creds['port'], creds['username'], creds['password'], - ) + ) == 0 def dump_command(self) -> list: """list: create a dump command restic and use to send data through stdin""" @@ -156,22 +161,19 @@ class PostgresContainer(Container): creds = self.get_credentials() return [ "pg_dump", - f"--host={creds['host']}", - f"--port={creds['port']}", f"--username={creds['username']}", creds['database'], ] def backup(self): config = Config() - creds = self.get_credentials() - with utils.environment('PGPASSWORD', creds['password']): - return restic.backup_from_stdin( - config.repository, - self.backup_destination_path(), - self.dump_command(), - ) + return restic.backup_from_stdin( + config.repository, + self.backup_destination_path(), + self.id, + self.dump_command(), + ) def backup_destination_path(self) -> str: destination = Path("/databases") diff --git a/src/restic_compose_backup/restic.py b/src/restic_compose_backup/restic.py index 5f79f90..5b59dfc 100644 --- a/src/restic_compose_backup/restic.py +++ b/src/restic_compose_backup/restic.py @@ -2,9 +2,10 @@ Restic commands """ import logging -from typing import List, Tuple +from typing import List, Tuple, Union from subprocess import Popen, PIPE from restic_compose_backup import commands +from restic_compose_backup import utils logger = logging.getLogger(__name__) @@ -27,9 +28,10 @@ def backup_files(repository: str, source='/volumes'): ])) -def backup_from_stdin(repository: str, filename: str, source_command: List[str]): +def backup_from_stdin(repository: str, filename: str, container_id: str, + source_command: List[str], environment: Union[dict, list] = None): """ - Backs up from stdin running the source_command passed in. + Backs up from stdin running the source_command passed in within the given container. It will appear in restic with the filename (including path) passed in. """ dest_command = restic(repository, [ @@ -39,20 +41,43 @@ def backup_from_stdin(repository: str, filename: str, source_command: List[str]) filename, ]) - # pipe source command into dest command - source_process = Popen(source_command, stdout=PIPE, bufsize=65536) - dest_process = Popen(dest_command, stdin=source_process.stdout, stdout=PIPE, stderr=PIPE, bufsize=65536) + client = utils.docker_client() + + logger.debug('docker exec inside %s: %s', container_id, ' '.join(source_command)) + + # Create and start source command inside the given container + handle = client.api.exec_create(container_id, source_command, environment=environment) + exec_id = handle["Id"] + stream = client.api.exec_start(exec_id, stream=True, demux=True) + source_stderr = "" + + # Create the restic process to receive the output of the source command + dest_process = Popen(dest_command, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=65536) + + # Send the ouptut of the source command over to restic in the chunks received + for stdout_chunk, stderr_chunk in stream: + if stdout_chunk: + dest_process.stdin.write(stdout_chunk) + + if stderr_chunk: + source_stderr += stderr_chunk.decode() + + # Wait for restic to finish stdout, stderr = dest_process.communicate() # Ensure both processes exited with code 0 - source_exit, dest_exit = source_process.poll(), dest_process.poll() - exit_code = 0 if (source_exit == 0 and dest_exit == 0) else 1 + source_exit = client.api.exec_inspect(exec_id).get("ExitCode") + dest_exit = dest_process.poll() + exit_code = source_exit or dest_exit if stdout: commands.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR) + if source_stderr: + commands.log_std(f'stderr ({source_command[0]})', source_stderr, logging.ERROR) + if stderr: - commands.log_std('stderr', stderr, logging.ERROR) + commands.log_std('stderr (restic)', stderr, logging.ERROR) return exit_code diff --git a/src/restic_compose_backup/utils.py b/src/restic_compose_backup/utils.py index 0f96fbe..1884927 100644 --- a/src/restic_compose_backup/utils.py +++ b/src/restic_compose_backup/utils.py @@ -3,6 +3,7 @@ import logging from typing import List, TYPE_CHECKING from contextlib import contextmanager import docker +from docker import DockerClient if TYPE_CHECKING: from restic_compose_backup.containers import Container @@ -12,7 +13,7 @@ logger = logging.getLogger(__name__) TRUE_VALUES = ['1', 'true', 'True', True, 1] -def docker_client(): +def docker_client() -> DockerClient: """ Create a docker client from the following environment variables:: From 936089ee995d9688f3668dea77b756cb9878f5d7 Mon Sep 17 00:00:00 2001 From: Jannik Date: Thu, 28 May 2020 17:50:18 +0200 Subject: [PATCH 3/3] Update travis python version to 3.8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 567f8db..f976756 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ sudo: false matrix: include: - python: 3.7 + python: 3.8 dist: bionic sudo: true