Compare commits
7 Commits
d20d5756aa
...
3be7737232
Author | SHA1 | Date |
---|---|---|
Jannik | 3be7737232 | |
einarf | 405bd4af15 | |
dreadper | 28dda6b09d | |
dreadper | b400138b73 | |
Jannik | 936089ee99 | |
Jannik | fbe77a10c0 | |
Jannik | fa887e1b5b |
|
@ -4,7 +4,7 @@ sudo: false
|
||||||
|
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
python: 3.7
|
python: 3.8
|
||||||
dist: bionic
|
dist: bionic
|
||||||
sudo: true
|
sudo: true
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ services:
|
||||||
build: ./src
|
build: ./src
|
||||||
env_file:
|
env_file:
|
||||||
- restic_compose_backup.env
|
- restic_compose_backup.env
|
||||||
- alerts.env
|
# - alerts.env
|
||||||
labels:
|
labels:
|
||||||
restic-compose-backup.volumes: true
|
restic-compose-backup.volumes: true
|
||||||
restic-compose-backup.volumes.include: 'src'
|
restic-compose-backup.volumes.include: 'src'
|
||||||
|
@ -32,7 +32,7 @@ services:
|
||||||
- SOME_VALUE=test
|
- SOME_VALUE=test
|
||||||
- ANOTHER_VALUE=1
|
- ANOTHER_VALUE=1
|
||||||
|
|
||||||
mysql:
|
mysql5:
|
||||||
image: mysql:5
|
image: mysql:5
|
||||||
labels:
|
labels:
|
||||||
restic-compose-backup.mysql: true
|
restic-compose-backup.mysql: true
|
||||||
|
@ -42,7 +42,19 @@ services:
|
||||||
- MYSQL_USER=myuser
|
- MYSQL_USER=myuser
|
||||||
- MYSQL_PASSWORD=mypassword
|
- MYSQL_PASSWORD=mypassword
|
||||||
volumes:
|
volumes:
|
||||||
- mysqldata:/var/lib/mysql
|
- mysqldata5:/var/lib/mysql
|
||||||
|
|
||||||
|
mysql8:
|
||||||
|
image: mysql:8
|
||||||
|
labels:
|
||||||
|
restic-compose-backup.mysql: true
|
||||||
|
environment:
|
||||||
|
- MYSQL_ROOT_PASSWORD=my-secret-pw
|
||||||
|
- MYSQL_DATABASE=mydb
|
||||||
|
- MYSQL_USER=myuser
|
||||||
|
- MYSQL_PASSWORD=mypassword
|
||||||
|
volumes:
|
||||||
|
- mysqldata8:/var/lib/mysql
|
||||||
|
|
||||||
mariadb:
|
mariadb:
|
||||||
image: mariadb:10
|
image: mariadb:10
|
||||||
|
@ -68,7 +80,8 @@ services:
|
||||||
- pgdata:/var/lib/postgresql/data
|
- pgdata:/var/lib/postgresql/data
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
mysqldata:
|
mysqldata5:
|
||||||
|
mysqldata8:
|
||||||
mariadbdata:
|
mariadbdata:
|
||||||
pgdata:
|
pgdata:
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
FROM restic/restic:0.9.6
|
FROM restic/restic:0.9.6
|
||||||
|
|
||||||
RUN apk update && apk add python3 dcron mariadb-client postgresql-client
|
RUN apk update && apk add python3 \
|
||||||
|
dcron \
|
||||||
|
mariadb-client \
|
||||||
|
postgresql-client \
|
||||||
|
mariadb-connector-c-dev
|
||||||
|
|
||||||
ADD . /restic-compose-backup
|
ADD . /restic-compose-backup
|
||||||
WORKDIR /restic-compose-backup
|
WORKDIR /restic-compose-backup
|
||||||
|
|
|
@ -105,10 +105,10 @@ def status(config, containers):
|
||||||
logger.info(
|
logger.info(
|
||||||
' - %s (is_ready=%s) -> %s',
|
' - %s (is_ready=%s) -> %s',
|
||||||
instance.container_type,
|
instance.container_type,
|
||||||
ping == 0,
|
ping,
|
||||||
instance.backup_destination_path(),
|
instance.backup_destination_path(),
|
||||||
)
|
)
|
||||||
if ping != 0:
|
if not ping:
|
||||||
logger.error("Database '%s' in service %s cannot be reached",
|
logger.error("Database '%s' in service %s cannot be reached",
|
||||||
instance.container_type, container.service_name)
|
instance.container_type, container.service_name)
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Tuple
|
from typing import List, Tuple, Union
|
||||||
from subprocess import Popen, PIPE
|
from subprocess import Popen, PIPE
|
||||||
|
|
||||||
|
from restic_compose_backup import utils
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@ -9,37 +11,33 @@ def test():
|
||||||
return run(['ls', '/volumes'])
|
return run(['ls', '/volumes'])
|
||||||
|
|
||||||
|
|
||||||
def ping_mysql(host, port, username) -> int:
|
def ping_mysql(container_id, host, port, username, password) -> int:
|
||||||
"""Check if the mysql is up and can be reached"""
|
"""Check if the mysql is up and can be reached"""
|
||||||
return run([
|
return docker_exec(container_id, [
|
||||||
'mysqladmin',
|
'mysqladmin',
|
||||||
'ping',
|
'ping',
|
||||||
'--host',
|
|
||||||
host,
|
|
||||||
'--port',
|
|
||||||
port,
|
|
||||||
'--user',
|
'--user',
|
||||||
username,
|
username,
|
||||||
])
|
], environment={
|
||||||
|
'MYSQL_PWD': password
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
def ping_mariadb(host, port, username) -> int:
|
def ping_mariadb(container_id, host, port, username, password) -> int:
|
||||||
"""Check if the mariadb is up and can be reached"""
|
"""Check if the mariadb is up and can be reached"""
|
||||||
return run([
|
return docker_exec(container_id, [
|
||||||
'mysqladmin',
|
'mysqladmin',
|
||||||
'ping',
|
'ping',
|
||||||
'--host',
|
|
||||||
host,
|
|
||||||
'--port',
|
|
||||||
port,
|
|
||||||
'--user',
|
'--user',
|
||||||
username,
|
username,
|
||||||
])
|
], environment={
|
||||||
|
'MYSQL_PWD': password
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
def ping_postgres(host, port, username, password) -> int:
|
def ping_postgres(container_id, host, port, username, password) -> int:
|
||||||
"""Check if postgres can be reached"""
|
"""Check if postgres can be reached"""
|
||||||
return run([
|
return docker_exec(container_id, [
|
||||||
"pg_isready",
|
"pg_isready",
|
||||||
f"--host={host}",
|
f"--host={host}",
|
||||||
f"--port={port}",
|
f"--port={port}",
|
||||||
|
@ -47,6 +45,22 @@ def ping_postgres(host, port, username, password) -> int:
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def docker_exec(container_id: str, cmd: List[str], environment: Union[dict, list] = []) -> int:
|
||||||
|
"""Execute a command within the given container"""
|
||||||
|
client = utils.docker_client()
|
||||||
|
logger.debug('docker exec inside %s: %s', container_id, ' '.join(cmd))
|
||||||
|
exit_code, (stdout, stderr) = client.containers.get(container_id).exec_run(cmd, demux=True, environment=environment)
|
||||||
|
|
||||||
|
if stdout:
|
||||||
|
log_std('stdout', stdout.decode(),
|
||||||
|
logging.DEBUG if exit_code == 0 else logging.ERROR)
|
||||||
|
|
||||||
|
if stderr:
|
||||||
|
log_std('stderr', stderr.decode(), logging.ERROR)
|
||||||
|
|
||||||
|
return exit_code
|
||||||
|
|
||||||
|
|
||||||
def run(cmd: List[str]) -> int:
|
def run(cmd: List[str]) -> int:
|
||||||
"""Run a command with parameters"""
|
"""Run a command with parameters"""
|
||||||
logger.debug('cmd: %s', ' '.join(cmd))
|
logger.debug('cmd: %s', ' '.join(cmd))
|
||||||
|
|
|
@ -25,34 +25,37 @@ class MariadbContainer(Container):
|
||||||
"""Check the availability of the service"""
|
"""Check the availability of the service"""
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
|
|
||||||
with utils.environment('MYSQL_PWD', creds['password']):
|
return commands.ping_mariadb(
|
||||||
return commands.ping_mariadb(
|
self.id,
|
||||||
creds['host'],
|
creds['host'],
|
||||||
creds['port'],
|
creds['port'],
|
||||||
creds['username'],
|
creds['username'],
|
||||||
)
|
creds['password']
|
||||||
|
) == 0
|
||||||
|
|
||||||
def dump_command(self) -> list:
|
def dump_command(self) -> list:
|
||||||
"""list: create a dump command restic and use to send data through stdin"""
|
"""list: create a dump command restic and use to send data through stdin"""
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
return [
|
return [
|
||||||
"mysqldump",
|
"mysqldump",
|
||||||
f"--host={creds['host']}",
|
|
||||||
f"--port={creds['port']}",
|
|
||||||
f"--user={creds['username']}",
|
f"--user={creds['username']}",
|
||||||
"--all-databases",
|
"--all-databases",
|
||||||
|
"--no-tablespaces",
|
||||||
]
|
]
|
||||||
|
|
||||||
def backup(self):
|
def backup(self):
|
||||||
config = Config()
|
config = Config()
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
|
|
||||||
with utils.environment('MYSQL_PWD', creds['password']):
|
return restic.backup_from_stdin(
|
||||||
return restic.backup_from_stdin(
|
config.repository,
|
||||||
config.repository,
|
self.backup_destination_path(),
|
||||||
self.backup_destination_path(),
|
self.id,
|
||||||
self.dump_command(),
|
self.dump_command(),
|
||||||
)
|
environment={
|
||||||
|
'MYSQL_PWD': creds['password']
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def backup_destination_path(self) -> str:
|
def backup_destination_path(self) -> str:
|
||||||
destination = Path("/databases")
|
destination = Path("/databases")
|
||||||
|
@ -84,34 +87,37 @@ class MysqlContainer(Container):
|
||||||
"""Check the availability of the service"""
|
"""Check the availability of the service"""
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
|
|
||||||
with utils.environment('MYSQL_PWD', creds['password']):
|
return commands.ping_mysql(
|
||||||
return commands.ping_mysql(
|
self.id,
|
||||||
creds['host'],
|
creds['host'],
|
||||||
creds['port'],
|
creds['port'],
|
||||||
creds['username'],
|
creds['username'],
|
||||||
)
|
creds['password']
|
||||||
|
) == 0
|
||||||
|
|
||||||
def dump_command(self) -> list:
|
def dump_command(self) -> list:
|
||||||
"""list: create a dump command restic and use to send data through stdin"""
|
"""list: create a dump command restic and use to send data through stdin"""
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
return [
|
return [
|
||||||
"mysqldump",
|
"mysqldump",
|
||||||
f"--host={creds['host']}",
|
|
||||||
f"--port={creds['port']}",
|
|
||||||
f"--user={creds['username']}",
|
f"--user={creds['username']}",
|
||||||
"--all-databases",
|
"--all-databases",
|
||||||
|
"--no-tablespaces",
|
||||||
]
|
]
|
||||||
|
|
||||||
def backup(self):
|
def backup(self):
|
||||||
config = Config()
|
config = Config()
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
|
|
||||||
with utils.environment('MYSQL_PWD', creds['password']):
|
return restic.backup_from_stdin(
|
||||||
return restic.backup_from_stdin(
|
config.repository,
|
||||||
config.repository,
|
self.backup_destination_path(),
|
||||||
self.backup_destination_path(),
|
self.id,
|
||||||
self.dump_command(),
|
self.dump_command(),
|
||||||
)
|
environment={
|
||||||
|
"MYSQL_PWD": creds['password']
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def backup_destination_path(self) -> str:
|
def backup_destination_path(self) -> str:
|
||||||
destination = Path("/databases")
|
destination = Path("/databases")
|
||||||
|
@ -144,11 +150,12 @@ class PostgresContainer(Container):
|
||||||
"""Check the availability of the service"""
|
"""Check the availability of the service"""
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
return commands.ping_postgres(
|
return commands.ping_postgres(
|
||||||
|
self.id,
|
||||||
creds['host'],
|
creds['host'],
|
||||||
creds['port'],
|
creds['port'],
|
||||||
creds['username'],
|
creds['username'],
|
||||||
creds['password'],
|
creds['password'],
|
||||||
)
|
) == 0
|
||||||
|
|
||||||
def dump_command(self) -> list:
|
def dump_command(self) -> list:
|
||||||
"""list: create a dump command restic and use to send data through stdin"""
|
"""list: create a dump command restic and use to send data through stdin"""
|
||||||
|
@ -156,22 +163,19 @@ class PostgresContainer(Container):
|
||||||
creds = self.get_credentials()
|
creds = self.get_credentials()
|
||||||
return [
|
return [
|
||||||
"pg_dump",
|
"pg_dump",
|
||||||
f"--host={creds['host']}",
|
|
||||||
f"--port={creds['port']}",
|
|
||||||
f"--username={creds['username']}",
|
f"--username={creds['username']}",
|
||||||
creds['database'],
|
creds['database'],
|
||||||
]
|
]
|
||||||
|
|
||||||
def backup(self):
|
def backup(self):
|
||||||
config = Config()
|
config = Config()
|
||||||
creds = self.get_credentials()
|
|
||||||
|
|
||||||
with utils.environment('PGPASSWORD', creds['password']):
|
return restic.backup_from_stdin(
|
||||||
return restic.backup_from_stdin(
|
config.repository,
|
||||||
config.repository,
|
self.backup_destination_path(),
|
||||||
self.backup_destination_path(),
|
self.id,
|
||||||
self.dump_command(),
|
self.dump_command(),
|
||||||
)
|
)
|
||||||
|
|
||||||
def backup_destination_path(self) -> str:
|
def backup_destination_path(self) -> str:
|
||||||
destination = Path("/databases")
|
destination = Path("/databases")
|
||||||
|
|
|
@ -2,9 +2,10 @@
|
||||||
Restic commands
|
Restic commands
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Tuple
|
from typing import List, Tuple, Union
|
||||||
from subprocess import Popen, PIPE
|
from subprocess import Popen, PIPE
|
||||||
from restic_compose_backup import commands
|
from restic_compose_backup import commands
|
||||||
|
from restic_compose_backup import utils
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -27,9 +28,10 @@ def backup_files(repository: str, source='/volumes'):
|
||||||
]))
|
]))
|
||||||
|
|
||||||
|
|
||||||
def backup_from_stdin(repository: str, filename: str, source_command: List[str]):
|
def backup_from_stdin(repository: str, filename: str, container_id: str,
|
||||||
|
source_command: List[str], environment: Union[dict, list] = None):
|
||||||
"""
|
"""
|
||||||
Backs up from stdin running the source_command passed in.
|
Backs up from stdin running the source_command passed in within the given container.
|
||||||
It will appear in restic with the filename (including path) passed in.
|
It will appear in restic with the filename (including path) passed in.
|
||||||
"""
|
"""
|
||||||
dest_command = restic(repository, [
|
dest_command = restic(repository, [
|
||||||
|
@ -39,20 +41,43 @@ def backup_from_stdin(repository: str, filename: str, source_command: List[str])
|
||||||
filename,
|
filename,
|
||||||
])
|
])
|
||||||
|
|
||||||
# pipe source command into dest command
|
client = utils.docker_client()
|
||||||
source_process = Popen(source_command, stdout=PIPE, bufsize=65536)
|
|
||||||
dest_process = Popen(dest_command, stdin=source_process.stdout, stdout=PIPE, stderr=PIPE, bufsize=65536)
|
logger.debug('docker exec inside %s: %s', container_id, ' '.join(source_command))
|
||||||
|
|
||||||
|
# Create and start source command inside the given container
|
||||||
|
handle = client.api.exec_create(container_id, source_command, environment=environment)
|
||||||
|
exec_id = handle["Id"]
|
||||||
|
stream = client.api.exec_start(exec_id, stream=True, demux=True)
|
||||||
|
source_stderr = ""
|
||||||
|
|
||||||
|
# Create the restic process to receive the output of the source command
|
||||||
|
dest_process = Popen(dest_command, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=65536)
|
||||||
|
|
||||||
|
# Send the ouptut of the source command over to restic in the chunks received
|
||||||
|
for stdout_chunk, stderr_chunk in stream:
|
||||||
|
if stdout_chunk:
|
||||||
|
dest_process.stdin.write(stdout_chunk)
|
||||||
|
|
||||||
|
if stderr_chunk:
|
||||||
|
source_stderr += stderr_chunk.decode()
|
||||||
|
|
||||||
|
# Wait for restic to finish
|
||||||
stdout, stderr = dest_process.communicate()
|
stdout, stderr = dest_process.communicate()
|
||||||
|
|
||||||
# Ensure both processes exited with code 0
|
# Ensure both processes exited with code 0
|
||||||
source_exit, dest_exit = source_process.poll(), dest_process.poll()
|
source_exit = client.api.exec_inspect(exec_id).get("ExitCode")
|
||||||
exit_code = 0 if (source_exit == 0 and dest_exit == 0) else 1
|
dest_exit = dest_process.poll()
|
||||||
|
exit_code = source_exit or dest_exit
|
||||||
|
|
||||||
if stdout:
|
if stdout:
|
||||||
commands.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR)
|
commands.log_std('stdout', stdout, logging.DEBUG if exit_code == 0 else logging.ERROR)
|
||||||
|
|
||||||
|
if source_stderr:
|
||||||
|
commands.log_std(f'stderr ({source_command[0]})', source_stderr, logging.ERROR)
|
||||||
|
|
||||||
if stderr:
|
if stderr:
|
||||||
commands.log_std('stderr', stderr, logging.ERROR)
|
commands.log_std('stderr (restic)', stderr, logging.ERROR)
|
||||||
|
|
||||||
return exit_code
|
return exit_code
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ import logging
|
||||||
from typing import List, TYPE_CHECKING
|
from typing import List, TYPE_CHECKING
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
import docker
|
import docker
|
||||||
|
from docker import DockerClient
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from restic_compose_backup.containers import Container
|
from restic_compose_backup.containers import Container
|
||||||
|
@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)
|
||||||
TRUE_VALUES = ['1', 'true', 'True', True, 1]
|
TRUE_VALUES = ['1', 'true', 'True', True, 1]
|
||||||
|
|
||||||
|
|
||||||
def docker_client():
|
def docker_client() -> DockerClient:
|
||||||
"""
|
"""
|
||||||
Create a docker client from the following environment variables::
|
Create a docker client from the following environment variables::
|
||||||
|
|
||||||
|
|
|
@ -3,12 +3,15 @@ from setuptools import setup, find_namespace_packages
|
||||||
setup(
|
setup(
|
||||||
name="restic-compose-backup",
|
name="restic-compose-backup",
|
||||||
url="https://github.com/ZettaIO/restic-compose-backup",
|
url="https://github.com/ZettaIO/restic-compose-backup",
|
||||||
version="0.6.0",
|
version="0.7.0",
|
||||||
author="Einar Forselv",
|
author="Einar Forselv",
|
||||||
author_email="eforselv@gmail.com",
|
author_email="eforselv@gmail.com",
|
||||||
packages=find_namespace_packages(include=['restic_compose_backup']),
|
packages=find_namespace_packages(include=[
|
||||||
|
'restic_compose_backup',
|
||||||
|
'restic_compose_backup.*',
|
||||||
|
]),
|
||||||
install_requires=[
|
install_requires=[
|
||||||
'docker==4.1.*',
|
'docker~=6.1.3',
|
||||||
],
|
],
|
||||||
entry_points={'console_scripts': [
|
entry_points={'console_scripts': [
|
||||||
'restic-compose-backup = restic_compose_backup.cli:main',
|
'restic-compose-backup = restic_compose_backup.cli:main',
|
||||||
|
|
2
tox.ini
2
tox.ini
|
@ -53,4 +53,4 @@ norecursedirs = tests/* .venv/* .tox/* build/ docs/
|
||||||
ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503
|
ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503
|
||||||
show-source = True
|
show-source = True
|
||||||
max-line-length = 120
|
max-line-length = 120
|
||||||
exclude = .tox,env,tests,build,conf.py
|
exclude = .tox,env,tests,build,conf.py,.venv
|
||||||
|
|
Loading…
Reference in New Issue