diff --git a/.travis.yml b/.travis.yml
index 567f8db..f976756 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,7 +4,7 @@ sudo: false
matrix:
include:
- python: 3.7
+ python: 3.8
dist: bionic
sudo: true
diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py
index fb8cbce..c6e3793 100644
--- a/src/restic_compose_backup/cli.py
+++ b/src/restic_compose_backup/cli.py
@@ -105,10 +105,10 @@ def status(config, containers):
logger.info(
' - %s (is_ready=%s) -> %s',
instance.container_type,
- ping == 0,
+ ping,
instance.backup_destination_path(),
)
- if ping != 0:
+ if not ping:
logger.error("Database '%s' in service %s cannot be reached",
instance.container_type, container.service_name)
diff --git a/src/restic_compose_backup/commands.py b/src/restic_compose_backup/commands.py
index cf209cb..b1c5558 100644
--- a/src/restic_compose_backup/commands.py
+++ b/src/restic_compose_backup/commands.py
@@ -1,7 +1,9 @@
import logging
-from typing import List, Tuple
+from typing import List, Tuple, Union
from subprocess import Popen, PIPE
+from restic_compose_backup import utils
+
logger = logging.getLogger(__name__)
@@ -9,37 +11,33 @@ def test():
return run(['ls', '/volumes'])
-def ping_mysql(host, port, username) -> int:
+def ping_mysql(container_id, host, port, username, password) -> int:
"""Check if the mysql is up and can be reached"""
- return run([
+ return docker_exec(container_id, [
'mysqladmin',
'ping',
- '--host',
- host,
- '--port',
- port,
'--user',
username,
- ])
+ ], environment={
+ 'MYSQL_PWD': password
+ })
-def ping_mariadb(host, port, username) -> int:
+def ping_mariadb(container_id, host, port, username, password) -> int:
"""Check if the mariadb is up and can be reached"""
- return run([
+ return docker_exec(container_id, [
'mysqladmin',
'ping',
- '--host',
- host,
- '--port',
- port,
'--user',
username,
- ])
+ ], environment={
+ 'MYSQL_PWD': password
+ })
-def ping_postgres(host, port, username, password) -> int:
+def ping_postgres(container_id, host, port, username, password) -> int:
"""Check if postgres can be reached"""
- return run([
+ return docker_exec(container_id, [
"pg_isready",
f"--host={host}",
f"--port={port}",
@@ -47,6 +45,22 @@ def ping_postgres(host, port, username, password) -> int:
])
+def docker_exec(container_id: str, cmd: List[str], environment: Union[dict, list] = []) -> int:
+ """Execute a command within the given container"""
+ client = utils.docker_client()
+ logger.debug('docker exec inside %s: %s', container_id, ' '.join(cmd))
+ exit_code, (stdout, stderr) = client.containers.get(container_id).exec_run(cmd, demux=True, environment=environment)
+
+ if stdout:
+ log_std('stdout', stdout.decode(),
+ logging.DEBUG if exit_code == 0 else logging.ERROR)
+
+ if stderr:
+ log_std('stderr', stderr.decode(), logging.ERROR)
+
+ return exit_code
+
+
def run(cmd: List[str]) -> int:
"""Run a command with parameters"""
logger.debug('cmd: %s', ' '.join(cmd))
diff --git a/src/restic_compose_backup/containers_db.py b/src/restic_compose_backup/containers_db.py
index ea9c66e..114a67f 100644
--- a/src/restic_compose_backup/containers_db.py
+++ b/src/restic_compose_backup/containers_db.py
@@ -25,20 +25,19 @@ class MariadbContainer(Container):
"""Check the availability of the service"""
creds = self.get_credentials()
- with utils.environment('MYSQL_PWD', creds['password']):
- return commands.ping_mariadb(
- creds['host'],
- creds['port'],
- creds['username'],
- )
+ return commands.ping_mariadb(
+ self.id,
+ creds['host'],
+ creds['port'],
+ creds['username'],
+ creds['password']
+ ) == 0
def dump_command(self) -> list:
"""list: create a dump command restic and use to send data through stdin"""
creds = self.get_credentials()
return [
"mysqldump",
- f"--host={creds['host']}",
- f"--port={creds['port']}",
f"--user={creds['username']}",
"--all-databases",
]
@@ -47,12 +46,15 @@ class MariadbContainer(Container):
config = Config()
creds = self.get_credentials()
- with utils.environment('MYSQL_PWD', creds['password']):
- return restic.backup_from_stdin(
- config.repository,
- self.backup_destination_path(),
- self.dump_command(),
- )
+ return restic.backup_from_stdin(
+ config.repository,
+ self.backup_destination_path(),
+ self.id,
+ self.dump_command(),
+ environment={
+ 'MYSQL_PWD': creds['password']
+ }
+ )
def backup_destination_path(self) -> str:
destination = Path("/databases")
@@ -84,20 +86,19 @@ class MysqlContainer(Container):
"""Check the availability of the service"""
creds = self.get_credentials()
- with utils.environment('MYSQL_PWD', creds['password']):
- return commands.ping_mysql(
- creds['host'],
- creds['port'],
- creds['username'],
- )
+ return commands.ping_mysql(
+ self.id,
+ creds['host'],
+ creds['port'],
+ creds['username'],
+ creds['password']
+ ) == 0
def dump_command(self) -> list:
"""list: create a dump command restic and use to send data through stdin"""
creds = self.get_credentials()
return [
"mysqldump",
- f"--host={creds['host']}",
- f"--port={creds['port']}",
f"--user={creds['username']}",
"--all-databases",
]
@@ -106,12 +107,15 @@ class MysqlContainer(Container):
config = Config()
creds = self.get_credentials()
- with utils.environment('MYSQL_PWD', creds['password']):
- return restic.backup_from_stdin(
- config.repository,
- self.backup_destination_path(),
- self.dump_command(),
- )
+ return restic.backup_from_stdin(
+ config.repository,
+ self.backup_destination_path(),
+ self.id,
+ self.dump_command(),
+ environment={
+ "MYSQL_PWD": creds['password']
+ }
+ )
def backup_destination_path(self) -> str:
destination = Path("/databases")
@@ -144,11 +148,12 @@ class PostgresContainer(Container):
"""Check the availability of the service"""
creds = self.get_credentials()
return commands.ping_postgres(
+ self.id,
creds['host'],
creds['port'],
creds['username'],
creds['password'],
- )
+ ) == 0
def dump_command(self) -> list:
"""list: create a dump command restic and use to send data through stdin"""
@@ -156,22 +161,19 @@ class PostgresContainer(Container):
creds = self.get_credentials()
return [
"pg_dump",
- f"--host={creds['host']}",
- f"--port={creds['port']}",
f"--username={creds['username']}",
creds['database'],
]
def backup(self):
config = Config()
- creds = self.get_credentials()
- with utils.environment('PGPASSWORD', creds['password']):
- return restic.backup_from_stdin(
- config.repository,
- self.backup_destination_path(),
- self.dump_command(),
- )
+ return restic.backup_from_stdin(
+ config.repository,
+ self.backup_destination_path(),
+ self.id,
+ self.dump_command(),
+ )
def backup_destination_path(self) -> str:
destination = Path("/databases")
diff --git a/src/restic_compose_backup/restic.py b/src/restic_compose_backup/restic.py
index 5f79f90..5b59dfc 100644
--- a/src/restic_compose_backup/restic.py
+++ b/src/restic_compose_backup/restic.py
@@ -2,9 +2,10 @@
Restic commands
"""
import logging
-from typing import List, Tuple
+from typing import List, Tuple, Union
from subprocess import Popen, PIPE
from restic_compose_backup import commands
+from restic_compose_backup import utils
logger = logging.getLogger(__name__)
@@ -27,9 +28,10 @@ def backup_files(repository: str, source='/volumes'):
]))
-def backup_from_stdin(repository: str, filename: str, source_command: List[str]):
+def backup_from_stdin(repository: str, filename: str, container_id: str,
+ source_command: List[str], environment: Union[dict, list] = None):
"""
- Backs up from stdin running the source_command passed in.
+ Backs up from stdin running the source_command passed in within the given container.
It will appear in restic with the filename (including path) passed in.
"""
dest_command = restic(repository, [
@@ -39,20 +41,43 @@ def backup_from_stdin(repository: str, filename: str, source_command: List[str])
filename,
])
- # pipe source command into dest command
- source_process = Popen(source_command, stdout=PIPE, bufsize=65536)
- dest_process = Popen(dest_command, stdin=source_process.stdout, stdout=PIPE, stderr=PIPE, bufsize=65536)
+ client = utils.docker_client()
+
+ logger.debug('docker exec inside %s: %s', container_id, ' '.join(source_command))
+
+ # Create and start source command inside the given container
+ handle = client.api.exec_create(container_id, source_command, environment=environment)
+ exec_id = handle["Id"]
+ stream = client.api.exec_start(exec_id, stream=True, demux=True)
+ source_stderr = ""
+
+ # Create the restic process to receive the output of the source command
+ dest_process = Popen(dest_command, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=65536)
+
+ # Send the ouptut of the source command over to restic in the chunks received
+ for stdout_chunk, stderr_chunk in stream:
+ if stdout_chunk:
+ dest_process.stdin.write(stdout_chunk)
+
+ if stderr_chunk:
+ source_stderr += stderr_chunk.decode()
+
+ # Wait for restic to finish
stdout, stderr = dest_process.communicate()
# Ensure both processes exited with code 0
- source_exit, dest_exit = source_process.poll(), dest_process.poll()
- exit_code = 0 if (source_exit == 0 and dest_exit == 0) else 1
+ source_exit = client.api.exec_inspect(exec_id).get("ExitCode")
+ dest_exit = dest_process.poll()
+ exit_code = source_exit or dest_exit
if stdout:
commands.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR)
+ if source_stderr:
+ commands.log_std(f'stderr ({source_command[0]})', source_stderr, logging.ERROR)
+
if stderr:
- commands.log_std('stderr', stderr, logging.ERROR)
+ commands.log_std('stderr (restic)', stderr, logging.ERROR)
return exit_code
diff --git a/src/restic_compose_backup/utils.py b/src/restic_compose_backup/utils.py
index 0f96fbe..1884927 100644
--- a/src/restic_compose_backup/utils.py
+++ b/src/restic_compose_backup/utils.py
@@ -3,6 +3,7 @@ import logging
from typing import List, TYPE_CHECKING
from contextlib import contextmanager
import docker
+from docker import DockerClient
if TYPE_CHECKING:
from restic_compose_backup.containers import Container
@@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)
TRUE_VALUES = ['1', 'true', 'True', True, 1]
-def docker_client():
+def docker_client() -> DockerClient:
"""
Create a docker client from the following environment variables::
diff --git a/tox.ini b/tox.ini
index 272ab66..a39f639 100644
--- a/tox.ini
+++ b/tox.ini
@@ -53,4 +53,4 @@ norecursedirs = tests/* .venv/* .tox/* build/ docs/
ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503
show-source = True
max-line-length = 120
-exclude = .tox,env,tests,build,conf.py
+exclude = .tox,env,tests,build,conf.py,.venv