Compare commits

...

26 Commits

Author SHA1 Message Date
einarf e6ca4aa9ca Add rtd config file 2024-01-05 04:13:20 +01:00
einarf 093dab93ca Make rcb dump env vars to properly escape them 2024-01-05 03:50:13 +01:00
einarf 405bd4af15 Various tweaks
* Use --no-tablespaces in mysqldump
* Dump to 0.7.0
* pin docker version
* Include missing packages in setup.py
2023-11-10 22:24:29 +01:00
dreadper 28dda6b09d fix TypeError: request() got an unexpected keyword argument 'chunked' by upgrading pip package docker (from 4.1.* to 5.1.*) 2023-10-28 14:54:20 +02:00
dreadper b400138b73 fix [Plugin caching_sha2_password could not be loaded](https://github.com/arey/mysql-client/issues/5) 2023-10-28 14:20:40 +02:00
Einar Forselv b52655a23b
Merge pull request #37 from wehrstedt/email-allow-empty-password
Allow empty password for smpt
2021-06-18 09:54:06 +02:00
Maximilian Wehrstedt 323e299b7e Allow empty password for smpt 2021-06-17 15:47:39 +02:00
einarf 5c33ccf0b1
Bump version 2020-05-28 01:27:06 +02:00
einarf 764aac6744
Docs for EXCLUDE_BIND_MOUNTS and INCLUDE_PROJECT_NAME 2020-05-28 01:18:21 +02:00
einarf bbe57dfd69
Broken test intialization 2020-05-28 00:18:04 +02:00
einarf 4517880846
pep8 2020-05-28 00:13:27 +02:00
einarf 1fefd63c72
Use py38 in tests 2020-05-28 00:13:04 +02:00
Einar Forselv 4e1af219e2
Merge pull request #27 from jannikw/exclude-bind-mounts
Allow excluding all bind mounts from backups
2020-05-28 00:03:35 +02:00
Einar Forselv 93f080d5b3
Merge pull request #26 from jannikw/project-names
Allow grouping of backup directories by project name
2020-05-27 23:59:27 +02:00
Jannik 0fc620bb1f Allow excluding all bind mounts from backups 2020-05-26 15:30:54 +02:00
Jannik f7958d7db9 Add docstring for get_volume_backup_destination 2020-05-26 14:54:36 +02:00
Jannik 18ddb173ac Allow inclusion of project name in backup path 2020-05-26 14:30:59 +02:00
Jannik f59a046bbc Output backup destination path with output command 2020-05-26 10:42:29 +02:00
Einar Forselv 8b934dc12f Update README.md 2020-05-15 23:37:44 +02:00
Einar Forselv 8e9105fed5 Install wheel 2020-05-15 23:37:39 +02:00
Einar Forselv d7492e51f6 Test command for listing swarm nodes 2020-03-08 18:36:29 +01:00
Einar Forselv 07a19f7f42 Get swarm nodes + robust fallback 2020-03-08 18:36:07 +01:00
Einar Forselv 13d8e07a33 Add swarm node json example 2020-03-08 18:35:37 +01:00
Einar Forselv 25b39b9908 Missing label in swarm test file 2020-03-08 17:38:01 +01:00
Einar Forselv d0fdf2d1d3 README: dev note about joining and leaving swarm 2020-03-08 17:37:40 +01:00
Einar Forselv 3aa0704045 Add service with volume in local dev stack sample 2020-03-08 14:51:15 +01:00
22 changed files with 330 additions and 38 deletions

24
.readthedocs.yaml Normal file
View File

@ -0,0 +1,24 @@
# Read the Docs configuration file for Sphinx projects
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
# Required
version: 2
# Set the OS, Python version and other tools you might need
build:
os: ubuntu-22.04
tools:
python: "3.10"
# Build documentation in the "docs/" directory with Sphinx
sphinx:
configuration: docs/conf.py
# Optionally build your docs in additional formats such as PDF and ePub
# formats:
# - pdf
# - epub
python:
install:
- requirements: docs/requirements.txt

View File

@ -176,15 +176,20 @@ docker stack deploy -c swarm-stack.yml test
```
In dev we should ideally start the backup container manually
```bash
docker-compose run --rm backup sh
# pip install the package in the container in editable mode to auto sync changes from host source
pip3 install -e .
```
Remember to enable swarm mode with `docker swarm init/join` and disable swarm
mode with `docker swarm leave --force` when needed in development (single node setup).
## Contributing
Contributions are welcome regardless of experience level. Don't hesitate submitting issues, opening partial or completed pull requests.
Contributions are welcome regardless of experience level.
Don't hesitate submitting issues, opening partial or completed pull requests.
[restic]: https://restic.net/
[documentation]: https://restic-compose-backup.readthedocs.io

View File

@ -4,7 +4,7 @@ services:
build: ./src
env_file:
- restic_compose_backup.env
- alerts.env
# - alerts.env
labels:
restic-compose-backup.volumes: true
restic-compose-backup.volumes.include: 'src'
@ -32,7 +32,7 @@ services:
- SOME_VALUE=test
- ANOTHER_VALUE=1
mysql:
mysql5:
image: mysql:5
labels:
restic-compose-backup.mysql: true
@ -42,7 +42,19 @@ services:
- MYSQL_USER=myuser
- MYSQL_PASSWORD=mypassword
volumes:
- mysqldata:/var/lib/mysql
- mysqldata5:/var/lib/mysql
mysql8:
image: mysql:8
labels:
restic-compose-backup.mysql: true
environment:
- MYSQL_ROOT_PASSWORD=my-secret-pw
- MYSQL_DATABASE=mydb
- MYSQL_USER=myuser
- MYSQL_PASSWORD=mypassword
volumes:
- mysqldata8:/var/lib/mysql
mariadb:
image: mariadb:10
@ -68,7 +80,8 @@ services:
- pgdata:/var/lib/postgresql/data
volumes:
mysqldata:
mysqldata5:
mysqldata8:
mariadbdata:
pgdata:

View File

@ -22,7 +22,7 @@ copyright = '2019, Zetta.IO Technology AS'
author = 'Zetta.IO Technology AS'
# The full version, including alpha/beta/rc tags
release = '0.5.0'
release = '0.6.0'
# -- General configuration ---------------------------------------------------

View File

@ -188,6 +188,28 @@ connecting to the Docker host. Combined with ``DOCKER_TLS_VERIFY``
this can be used to talk to docker through TLS in cases
were we cannot map in the docker socket.
INCLUDE_PROJECT_NAME
~~~~~~~~~~~~~~~~~~~~
Define this environment variable if your backup destination
paths needs project name as a prefix. This is useful
when running multiple projects.
EXCLUDE_BIND_MOUNTS
~~~~~~~~~~~~~~~~~~~
Docker has to volumes types. Binds and volumes.
Volumes are docker volumes (``docker`volume list``).
Binds are paths mapped into the container from
the host for example in the ``volumes`` section
of a service.
If defined all host binds will be ignored globally.
This is useful when you only care about actual
docker volumes. Often host binds are only used
for mapping in configuration. This saves the user
from manually excluding these bind volumes.
SWARM_MODE
~~~~~~~~~~

View File

@ -0,0 +1,93 @@
{
"ID": "k5427pk4t7ss4d7ylacumeavz",
"Version": {
"Index": 30
},
"CreatedAt": "2020-03-08T17:25:59.451947759Z",
"UpdatedAt": "2020-03-08T17:26:38.552002711Z",
"Spec": {
"Labels": {},
"Role": "manager",
"Availability": "active"
},
"Description": {
"Hostname": "docker-desktop",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 4000000000,
"MemoryBytes": 2085535744
},
"Engine": {
"EngineVersion": "19.03.5",
"Plugins": [{
"Type": "Log",
"Name": "awslogs"
}, {
"Type": "Log",
"Name": "fluentd"
}, {
"Type": "Log",
"Name": "gcplogs"
}, {
"Type": "Log",
"Name": "gelf"
}, {
"Type": "Log",
"Name": "journald"
}, {
"Type": "Log",
"Name": "json-file"
}, {
"Type": "Log",
"Name": "local"
}, {
"Type": "Log",
"Name": "logentries"
}, {
"Type": "Log",
"Name": "splunk"
}, {
"Type": "Log",
"Name": "syslog"
}, {
"Type": "Network",
"Name": "bridge"
}, {
"Type": "Network",
"Name": "host"
}, {
"Type": "Network",
"Name": "ipvlan"
}, {
"Type": "Network",
"Name": "macvlan"
}, {
"Type": "Network",
"Name": "null"
}, {
"Type": "Network",
"Name": "overlay"
}, {
"Type": "Volume",
"Name": "local"
}]
},
"TLSInfo": {
"TrustRoot": "-----BEGIN CERTIFICATE-----\nMIIBazCCARCgAwIBAgIUfx7TP8c4SHCrwPPxjSFJQcfTP5QwCgYIKoZIzj0EAwIw\nEzERMA8GA1UEAxMIc3dhcm0tY2EwHhcNMjAwMzA4MTcyMTAwWhcNNDAwMzAzMTcy\nMTAwWjATMREwDwYDVQQDEwhzd2FybS1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEH\nA0IABGOa/9Rdd6qNc24wvuL/I9t5Vt3MJzlwC+WN0R6HrA4Ik1h2dmSRZTQqnCI7\nWh16y+PLaFwIfN0JkN4FrpnUBsyjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB\nAf8EBTADAQH/MB0GA1UdDgQWBBTAPDjHRwjQhNGUxqE3COHCOQrOkjAKBggqhkjO\nPQQDAgNJADBGAiEAxd/lPEKy3gt3nfZ8DX7kDaaNH8jSPgCBx3ejUs3SoaUCIQD3\nZ8dVxNvG4+Gvn28mDjWhTNLCn0BYW6JFE8eTI0xv4A==\n-----END CERTIFICATE-----\n",
"CertIssuerSubject": "MBMxETAPBgNVBAMTCHN3YXJtLWNh",
"CertIssuerPublicKey": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEY5r/1F13qo1zbjC+4v8j23lW3cwnOXAL5Y3RHoesDgiTWHZ2ZJFlNCqcIjtaHXrL48toXAh83QmQ3gWumdQGzA=="
}
},
"Status": {
"State": "ready",
"Addr": "192.168.65.3"
},
"ManagerStatus": {
"Leader": true,
"Reachability": "reachable",
"Addr": "192.168.65.3:2377"
}
}

View File

@ -13,8 +13,8 @@ When releasing a bugfix version we need to update the
main image as well.
```bash
docker build src --tag zettaio/restic-compose-backup:0.5
docker build src --tag zettaio/restic-compose-backup:0.5.0
docker build src --tag zettaio/restic-compose-backup:0.6
docker build src --tag zettaio/restic-compose-backup:0.6.0
docker push zettaio/restic-compose-backup:0.5
docker push zettaio/restic-compose-backup:0.5.0

View File

@ -5,6 +5,8 @@
# DOCKER_CERT_PATH=''
SWARM_MODE=true
INCLUDE_PROJECT_NAME=false
EXCLUDE_BIND_MOUNTS=false
RESTIC_REPOSITORY=/restic_data
RESTIC_PASSWORD=password

View File

@ -1,10 +1,14 @@
FROM restic/restic:0.9.6
RUN apk update && apk add python3 dcron mariadb-client postgresql-client
RUN apk update && apk add python3 \
dcron \
mariadb-client \
postgresql-client \
mariadb-connector-c-dev
ADD . /restic-compose-backup
WORKDIR /restic-compose-backup
RUN pip3 install -U pip setuptools && pip3 install -e .
RUN pip3 install -U pip setuptools wheel && pip3 install -e .
ENV XDG_CACHE_HOME=/cache
ENTRYPOINT []

View File

@ -1,7 +1,7 @@
#!/bin/sh
# Dump all env vars so we can source them in cron jobs
printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
rcb dump-env > /env.sh
# Write crontab
rcb crontab > crontab

View File

@ -1 +1 @@
__version__ = '0.5.0'
__version__ = '0.7.1'

View File

@ -15,7 +15,7 @@ class SMTPAlert(BaseAlert):
self.host = host
self.port = port
self.user = user
self.password = password
self.password = password or ""
self.to = to
@classmethod
@ -34,7 +34,7 @@ class SMTPAlert(BaseAlert):
@property
def properly_configured(self) -> bool:
return self.host and self.port and self.user and self.password and len(self.to) > 0
return self.host and self.port and self.user and len(self.to) > 0
def send(self, subject: str = None, body: str = None, alert_type: str = 'INFO'):
# send_mail("Hello world!")

View File

@ -51,12 +51,26 @@ def main():
elif args.action == "crontab":
crontab(config)
elif args.action == "dump-env":
dump_env()
# Random test stuff here
elif args.action == "test":
nodes = utils.get_swarm_nodes()
print("Swarm nodes:")
for node in nodes:
addr = node.attrs['Status']['Addr']
state = node.attrs['Status']['State']
print(' - {} {} {}'.format(node.id, addr, state))
def status(config, containers):
"""Outputs the backup config for the compose setup"""
logger.info("Status for compose project '%s'", containers.project_name)
logger.info("Repository: '%s'", config.repository)
logger.info("Backup currently running?: %s", containers.backup_process_running)
logger.info("Include project name in backup path?: %s", utils.is_true(config.include_project_name))
logger.debug("Exclude bind mounts from backups?: %s", utils.is_true(config.exclude_bind_mounts))
logger.info("Checking docker availability")
utils.list_containers()
@ -82,12 +96,21 @@ def status(config, containers):
if container.volume_backup_enabled:
for mount in container.filter_mounts():
logger.info(' - volume: %s', mount.source)
logger.info(
' - volume: %s -> %s',
mount.source,
container.get_volume_backup_destination(mount, '/volumes'),
)
if container.database_backup_enabled:
instance = container.instance
ping = instance.ping()
logger.info(' - %s (is_ready=%s)', instance.container_type, ping == 0)
logger.info(
' - %s (is_ready=%s) -> %s',
instance.container_type,
ping == 0,
instance.backup_destination_path(),
)
if ping != 0:
logger.error("Database '%s' in service %s cannot be reached",
instance.container_type, container.service_name)
@ -270,6 +293,14 @@ def crontab(config):
print(cron.generate_crontab(config))
def dump_env():
"""Dump all environment variables to a script that can be sourced from cron"""
print("#!/bin/bash")
print("# This file was generated by restic-compose-backup")
for key, value in os.environ.items():
print("export {}='{}'".format(key, value))
def parse_args():
parser = argparse.ArgumentParser(prog='restic_compose_backup')
parser.add_argument(
@ -283,6 +314,8 @@ def parse_args():
'cleanup',
'version',
'crontab',
'dump-env',
'test',
],
)
parser.add_argument(

View File

@ -13,6 +13,8 @@ class Config:
self.cron_schedule = os.environ.get('CRON_SCHEDULE') or self.default_crontab_schedule
self.cron_command = os.environ.get('CRON_COMMAND') or self.default_backup_command
self.swarm_mode = os.environ.get('SWARM_MODE') or False
self.include_project_name = os.environ.get('INCLUDE_PROJECT_NAME') or False
self.exclude_bind_mounts = os.environ.get('EXCLUDE_BIND_MOUNTS') or False
# Log
self.log_level = os.environ.get('LOG_LEVEL')

View File

@ -72,7 +72,8 @@ class Container:
@property
def service_name(self) -> str:
"""Name of the container/service"""
return self.get_label('com.docker.compose.service', default='') or self.get_label('com.docker.swarm.service.name', default='')
return self.get_label('com.docker.compose.service', default='') or \
self.get_label('com.docker.swarm.service.name', default='')
@property
def backup_process_label(self) -> str:
@ -193,11 +194,15 @@ class Container:
"""Get all mounts for this container matching include/exclude filters"""
filtered = []
# If exclude_bind_mounts is true, only volume mounts are kept in the list of mounts
exclude_bind_mounts = utils.is_true(config.exclude_bind_mounts)
mounts = list(filter(lambda m: not exclude_bind_mounts or m.type == "volume", self._mounts))
if not self.volume_backup_enabled:
return filtered
if self._include:
for mount in self._mounts:
for mount in mounts:
for pattern in self._include:
if pattern in mount.source:
break
@ -207,14 +212,14 @@ class Container:
filtered.append(mount)
elif self._exclude:
for mount in self._mounts:
for mount in mounts:
for pattern in self._exclude:
if pattern in mount.source:
break
else:
filtered.append(mount)
else:
return self._mounts
return mounts
return filtered
@ -224,12 +229,26 @@ class Container:
volumes = {}
for mount in mounts:
volumes[mount.source] = {
'bind': str(Path(source_prefix) / self.service_name / Path(utils.strip_root(mount.destination))),
'bind': self.get_volume_backup_destination(mount, source_prefix),
'mode': mode,
}
return volumes
def get_volume_backup_destination(self, mount, source_prefix) -> str:
"""Get the destination path for backups of the given mount"""
destination = Path(source_prefix)
if utils.is_true(config.include_project_name):
project_name = self.project_name
if project_name != '':
destination /= project_name
destination /= self.service_name
destination /= Path(utils.strip_root(mount.destination))
return str(destination)
def get_credentials(self) -> dict:
"""dict: get credentials for the service"""
raise NotImplementedError("Base container class don't implement this")
@ -242,6 +261,10 @@ class Container:
"""Back up this service"""
raise NotImplementedError("Base container class don't implement this")
def backup_destination_path(self) -> str:
"""Return the path backups will be saved at"""
raise NotImplementedError("Base container class don't implement this")
def dump_command(self) -> list:
"""list: create a dump command restic and use to send data through stdin"""
raise NotImplementedError("Base container class don't implement this")

View File

@ -1,5 +1,7 @@
from pathlib import Path
from restic_compose_backup.containers import Container
from restic_compose_backup.config import Config
from restic_compose_backup.config import config, Config
from restic_compose_backup import (
commands,
restic,
@ -39,6 +41,7 @@ class MariadbContainer(Container):
f"--port={creds['port']}",
f"--user={creds['username']}",
"--all-databases",
"--no-tablespaces",
]
def backup(self):
@ -48,10 +51,23 @@ class MariadbContainer(Container):
with utils.environment('MYSQL_PWD', creds['password']):
return restic.backup_from_stdin(
config.repository,
f'/databases/{self.service_name}/all_databases.sql',
self.backup_destination_path(),
self.dump_command(),
)
def backup_destination_path(self) -> str:
destination = Path("/databases")
if utils.is_true(config.include_project_name):
project_name = self.project_name
if project_name != "":
destination /= project_name
destination /= self.service_name
destination /= "all_databases.sql"
return destination
class MysqlContainer(Container):
container_type = 'mysql'
@ -85,6 +101,7 @@ class MysqlContainer(Container):
f"--port={creds['port']}",
f"--user={creds['username']}",
"--all-databases",
"--no-tablespaces",
]
def backup(self):
@ -94,10 +111,23 @@ class MysqlContainer(Container):
with utils.environment('MYSQL_PWD', creds['password']):
return restic.backup_from_stdin(
config.repository,
f'/databases/{self.service_name}/all_databases.sql',
self.backup_destination_path(),
self.dump_command(),
)
def backup_destination_path(self) -> str:
destination = Path("/databases")
if utils.is_true(config.include_project_name):
project_name = self.project_name
if project_name != "":
destination /= project_name
destination /= self.service_name
destination /= "all_databases.sql"
return destination
class PostgresContainer(Container):
container_type = 'postgres'
@ -141,6 +171,19 @@ class PostgresContainer(Container):
with utils.environment('PGPASSWORD', creds['password']):
return restic.backup_from_stdin(
config.repository,
f"/databases/{self.service_name}/{creds['database']}.sql",
self.backup_destination_path(),
self.dump_command(),
)
def backup_destination_path(self) -> str:
destination = Path("/databases")
if utils.is_true(config.include_project_name):
project_name = self.project_name
if project_name != "":
destination /= project_name
destination /= self.service_name
destination /= f"{self.get_credentials()['database']}.sql"
return destination

View File

@ -22,7 +22,7 @@ def setup(level: str = 'warning'):
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(level)
# ch.setFormatter(logging.Formatter(f'%(asctime)s - {HOSTNAME} - %(name)s - %(levelname)s - %(message)s'))
# ch.setFormatter(logging.Formatter(f'%(asctime)s - {HOSTNAME} - %(levelname)s - %(message)s'))
ch.setFormatter(logging.Formatter(f'%(asctime)s - %(levelname)s: %(message)s'))
# ch.setFormatter(logging.Formatter('%(asctime)s - {HOSTNAME} - %(name)s - %(levelname)s - %(message)s'))
# ch.setFormatter(logging.Formatter('%(asctime)s - {HOSTNAME} - %(levelname)s - %(message)s'))
ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s: %(message)s'))
logger.addHandler(ch)

View File

@ -1,9 +1,12 @@
import os
import logging
from typing import List
from typing import List, TYPE_CHECKING
from contextlib import contextmanager
import docker
if TYPE_CHECKING:
from restic_compose_backup.containers import Container
logger = logging.getLogger(__name__)
TRUE_VALUES = ['1', 'true', 'True', True, 1]
@ -37,6 +40,18 @@ def list_containers() -> List[dict]:
return [c.attrs for c in all_containers]
def get_swarm_nodes():
client = docker_client()
# NOTE: If not a swarm node docker.errors.APIError is raised
# 503 Server Error: Service Unavailable
# ("This node is not a swarm manager. Use "docker swarm init" or
# "docker swarm join" to connect this node to swarm and try again.")
try:
return client.nodes.list()
except docker.errors.APIError:
return []
def remove_containers(containers: List['Container']):
client = docker_client()
logger.info('Attempting to delete stale backup process containers')

View File

@ -3,12 +3,15 @@ from setuptools import setup, find_namespace_packages
setup(
name="restic-compose-backup",
url="https://github.com/ZettaIO/restic-compose-backup",
version="0.5.0",
version="0.7.1",
author="Einar Forselv",
author_email="eforselv@gmail.com",
packages=find_namespace_packages(include=['restic_compose_backup']),
packages=find_namespace_packages(include=[
'restic_compose_backup',
'restic_compose_backup.*',
]),
install_requires=[
'docker==4.1.*',
'docker~=6.1.3',
],
entry_points={'console_scripts': [
'restic-compose-backup = restic_compose_backup.cli:main',

View File

@ -3,6 +3,9 @@ import os
import unittest
from unittest import mock
os.environ['RESTIC_REPOSITORY'] = "test"
os.environ['RESTIC_PASSWORD'] = "password"
from restic_compose_backup import utils
from restic_compose_backup.containers import RunningContainers
import fixtures
@ -15,8 +18,8 @@ class ResticBackupTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
"""Set up basic environment variables"""
os.environ['RESTIC_REPOSITORY'] = "test"
os.environ['RESTIC_PASSWORD'] = "password"
# os.environ['RESTIC_REPOSITORY'] = "test"
# os.environ['RESTIC_PASSWORD'] = "password"
def createContainers(self):
backup_hash = fixtures.generate_sha256()

View File

@ -14,9 +14,16 @@ services:
- global
volumes:
- mariadbdata:/var/lib/mysql
files:
image: nginx:1.17-alpine
labels:
restic-compose-backup.volumes: "true"
volumes:
- files:/srv/files
volumes:
mariadbdata:
files:
networks:
global:

View File

@ -5,13 +5,13 @@
skipsdist = True
setupdir={toxinidir}/src
envlist =
py37
py38
pep8
[testenv]
usedevelop = True
basepython =
py37: python3.7
py38: python3.8
deps =
-r{toxinidir}/src//tests/requirements.txt
@ -23,7 +23,7 @@ commands =
[testenv:pep8]
usedevelop = false
deps = flake8
basepython = python3.7
basepython = python3.8
commands = flake8
[pytest]
@ -53,4 +53,4 @@ norecursedirs = tests/* .venv/* .tox/* build/ docs/
ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503
show-source = True
max-line-length = 120
exclude = .tox,.venv*,tests,build,conf.py
exclude = .tox,env,tests,build,conf.py