jimmy
/
restic-compose-backup
mirror of https://github.com/ZettaIO/restic-compose-backup.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge 936089ee99 into 405bd4af15

This commit is contained in:
Jannik 2023-11-15 18:54:38 +01:00 committed by GitHub
parent 405bd4af15 936089ee99
commit 3be7737232
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 111 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -4,7 +4,7 @@ sudo: false
matrix:
include:
python: 3.7
python: 3.8
dist: bionic
sudo: true

4
src/restic_compose_backup/cli.py
View File

@ -105,10 +105,10 @@ def status(config, containers):
logger.info(
' - %s (is_ready=%s) -> %s',
instance.container_type,
ping == 0,
ping,
instance.backup_destination_path(),
)
if ping != 0:
if not ping:
logger.error("Database '%s' in service %s cannot be reached",
instance.container_type, container.service_name)

48
src/restic_compose_backup/commands.py
View File

@ -1,7 +1,9 @@
import logging
from typing import List, Tuple
from typing import List, Tuple, Union
from subprocess import Popen, PIPE
from restic_compose_backup import utils
logger = logging.getLogger(__name__)
@ -9,37 +11,33 @@ def test():
return run(['ls', '/volumes'])
def ping_mysql(host, port, username) -> int:
def ping_mysql(container_id, host, port, username, password) -> int:
"""Check if the mysql is up and can be reached"""
return run([
return docker_exec(container_id, [
'mysqladmin',
'ping',
'--host',
host,
'--port',
port,
'--user',
username,
])
], environment={
'MYSQL_PWD': password
})
def ping_mariadb(host, port, username) -> int:
def ping_mariadb(container_id, host, port, username, password) -> int:
"""Check if the mariadb is up and can be reached"""
return run([
return docker_exec(container_id, [
'mysqladmin',
'ping',
'--host',
host,
'--port',
port,
'--user',
username,
])
], environment={
'MYSQL_PWD': password
})
def ping_postgres(host, port, username, password) -> int:
def ping_postgres(container_id, host, port, username, password) -> int:
"""Check if postgres can be reached"""
return run([
return docker_exec(container_id, [
"pg_isready",
f"--host={host}",
f"--port={port}",
@ -47,6 +45,22 @@ def ping_postgres(host, port, username, password) -> int:
])
def docker_exec(container_id: str, cmd: List[str], environment: Union[dict, list] = []) -> int:
"""Execute a command within the given container"""
client = utils.docker_client()
logger.debug('docker exec inside %s: %s', container_id, ' '.join(cmd))
exit_code, (stdout, stderr) = client.containers.get(container_id).exec_run(cmd, demux=True, environment=environment)
if stdout:
log_std('stdout', stdout.decode(),
logging.DEBUG if exit_code == 0 else logging.ERROR)
if stderr:
log_std('stderr', stderr.decode(), logging.ERROR)
return exit_code
def run(cmd: List[str]) -> int:
"""Run a command with parameters"""
logger.debug('cmd: %s', ' '.join(cmd))

78
src/restic_compose_backup/containers_db.py
View File

@ -25,20 +25,19 @@ class MariadbContainer(Container):
"""Check the availability of the service"""
creds = self.get_credentials()
with utils.environment('MYSQL_PWD', creds['password']):
return commands.ping_mariadb(
creds['host'],
creds['port'],
creds['username'],
)
return commands.ping_mariadb(
self.id,
creds['host'],
creds['port'],
creds['username'],
creds['password']
) == 0
def dump_command(self) -> list:
"""list: create a dump command restic and use to send data through stdin"""
creds = self.get_credentials()
return [
"mysqldump",
f"--host={creds['host']}",
f"--port={creds['port']}",
f"--user={creds['username']}",
"--all-databases",
"--no-tablespaces",
@ -48,12 +47,15 @@ class MariadbContainer(Container):
config = Config()
creds = self.get_credentials()
with utils.environment('MYSQL_PWD', creds['password']):
return restic.backup_from_stdin(
config.repository,
self.backup_destination_path(),
self.dump_command(),
)
return restic.backup_from_stdin(
config.repository,
self.backup_destination_path(),
self.id,
self.dump_command(),
environment={
'MYSQL_PWD': creds['password']
}
)
def backup_destination_path(self) -> str:
destination = Path("/databases")
@ -85,20 +87,19 @@ class MysqlContainer(Container):
"""Check the availability of the service"""
creds = self.get_credentials()
with utils.environment('MYSQL_PWD', creds['password']):
return commands.ping_mysql(
creds['host'],
creds['port'],
creds['username'],
)
return commands.ping_mysql(
self.id,
creds['host'],
creds['port'],
creds['username'],
creds['password']
) == 0
def dump_command(self) -> list:
"""list: create a dump command restic and use to send data through stdin"""
creds = self.get_credentials()
return [
"mysqldump",
f"--host={creds['host']}",
f"--port={creds['port']}",
f"--user={creds['username']}",
"--all-databases",
"--no-tablespaces",
@ -108,12 +109,15 @@ class MysqlContainer(Container):
config = Config()
creds = self.get_credentials()
with utils.environment('MYSQL_PWD', creds['password']):
return restic.backup_from_stdin(
config.repository,
self.backup_destination_path(),
self.dump_command(),
)
return restic.backup_from_stdin(
config.repository,
self.backup_destination_path(),
self.id,
self.dump_command(),
environment={
"MYSQL_PWD": creds['password']
}
)
def backup_destination_path(self) -> str:
destination = Path("/databases")
@ -146,11 +150,12 @@ class PostgresContainer(Container):
"""Check the availability of the service"""
creds = self.get_credentials()
return commands.ping_postgres(
self.id,
creds['host'],
creds['port'],
creds['username'],
creds['password'],
)
) == 0
def dump_command(self) -> list:
"""list: create a dump command restic and use to send data through stdin"""
@ -158,22 +163,19 @@ class PostgresContainer(Container):
creds = self.get_credentials()
return [
"pg_dump",
f"--host={creds['host']}",
f"--port={creds['port']}",
f"--username={creds['username']}",
creds['database'],
]
def backup(self):
config = Config()
creds = self.get_credentials()
with utils.environment('PGPASSWORD', creds['password']):
return restic.backup_from_stdin(
config.repository,
self.backup_destination_path(),
self.dump_command(),
)
return restic.backup_from_stdin(
config.repository,
self.backup_destination_path(),
self.id,
self.dump_command(),
)
def backup_destination_path(self) -> str:
destination = Path("/databases")

43
src/restic_compose_backup/restic.py
View File

@ -2,9 +2,10 @@
Restic commands
"""
import logging
from typing import List, Tuple
from typing import List, Tuple, Union
from subprocess import Popen, PIPE
from restic_compose_backup import commands
from restic_compose_backup import utils
logger = logging.getLogger(__name__)
@ -27,9 +28,10 @@ def backup_files(repository: str, source='/volumes'):
]))
def backup_from_stdin(repository: str, filename: str, source_command: List[str]):
def backup_from_stdin(repository: str, filename: str, container_id: str,
source_command: List[str], environment: Union[dict, list] = None):
"""
Backs up from stdin running the source_command passed in.
Backs up from stdin running the source_command passed in within the given container.
It will appear in restic with the filename (including path) passed in.
"""
dest_command = restic(repository, [
@ -39,20 +41,43 @@ def backup_from_stdin(repository: str, filename: str, source_command: List[str])
filename,
])
# pipe source command into dest command
source_process = Popen(source_command, stdout=PIPE, bufsize=65536)
dest_process = Popen(dest_command, stdin=source_process.stdout, stdout=PIPE, stderr=PIPE, bufsize=65536)
client = utils.docker_client()
logger.debug('docker exec inside %s: %s', container_id, ' '.join(source_command))
# Create and start source command inside the given container
handle = client.api.exec_create(container_id, source_command, environment=environment)
exec_id = handle["Id"]
stream = client.api.exec_start(exec_id, stream=True, demux=True)
source_stderr = ""
# Create the restic process to receive the output of the source command
dest_process = Popen(dest_command, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=65536)
# Send the ouptut of the source command over to restic in the chunks received
for stdout_chunk, stderr_chunk in stream:
if stdout_chunk:
dest_process.stdin.write(stdout_chunk)
if stderr_chunk:
source_stderr += stderr_chunk.decode()
# Wait for restic to finish
stdout, stderr = dest_process.communicate()
# Ensure both processes exited with code 0
source_exit, dest_exit = source_process.poll(), dest_process.poll()
exit_code = 0 if (source_exit == 0 and dest_exit == 0) else 1
source_exit = client.api.exec_inspect(exec_id).get("ExitCode")
dest_exit = dest_process.poll()
exit_code = source_exit or dest_exit
if stdout:
commands.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR)
if source_stderr:
commands.log_std(f'stderr ({source_command[0]})', source_stderr, logging.ERROR)
if stderr:
commands.log_std('stderr', stderr, logging.ERROR)
commands.log_std('stderr (restic)', stderr, logging.ERROR)
return exit_code

3
src/restic_compose_backup/utils.py
View File

@ -3,6 +3,7 @@ import logging
from typing import List, TYPE_CHECKING
from contextlib import contextmanager
import docker
from docker import DockerClient
if TYPE_CHECKING:
from restic_compose_backup.containers import Container
@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)
TRUE_VALUES = ['1', 'true', 'True', True, 1]
def docker_client():
def docker_client() -> DockerClient:
"""
Create a docker client from the following environment variables::

2
tox.ini
View File

@ -53,4 +53,4 @@ norecursedirs = tests/* .venv/* .tox/* build/ docs/
ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503
show-source = True
max-line-length = 120
exclude = .tox,env,tests,build,conf.py
exclude = .tox,env,tests,build,conf.py,.venv