From f1738147d637327c4464ae2e0a48b291302d744e Mon Sep 17 00:00:00 2001 From: Einar Forselv Date: Fri, 15 Nov 2019 14:23:56 +0100 Subject: [PATCH] Cleanup --- .dockerignore | 2 ++ .gitignore | 3 ++ Dockerfile | 3 +- crontab | 2 +- entrypoint.sh | 2 +- restic_volume_backup/cli.py | 51 +++++++++++++++++++----------- restic_volume_backup/config.py | 1 + restic_volume_backup/containers.py | 17 +++++----- restic_volume_backup/restic.py | 8 +++++ restic_volume_backup/utils.py | 2 +- test.sh | 2 -- 11 files changed, 60 insertions(+), 33 deletions(-) delete mode 100755 test.sh diff --git a/.dockerignore b/.dockerignore index 1dfa4ad..f8831a7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,5 @@ Dockerfile tests/ docker-compose.yaml *.env +*.egg-info +__pycache__ diff --git a/.gitignore b/.gitignore index 5f3184b..524c008 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ __pycache__ env .venv venv + +# misc +/private/ diff --git a/Dockerfile b/Dockerfile index ca4f188..d2da096 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ RUN apk update && apk add python3 dcron mariadb-client postgresql-client ADD . /restic-volume-backup WORKDIR /restic-volume-backup -RUN pip3 install -U pip setuptools && python3 setup.py develop && pip3 install . +RUN pip3 install -U pip setuptools +RUN python3 setup.py develop && pip3 install . ENTRYPOINT [] CMD ["./entrypoint.sh"] diff --git a/crontab b/crontab index eaf730f..e45d143 100644 --- a/crontab +++ b/crontab @@ -1 +1 @@ -30 * * * * source /env.sh && python3 restic-backup/backup.py backup >> /backup.log 2>&1 +1 * * * * source /env.sh && rvb backup > /proc/1/fd/1 diff --git a/entrypoint.sh b/entrypoint.sh index 511b586..8448a86 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh # Dump all env vars so we can source them in cron jobs -printenv | sed 's/^\(.*\)$/export \1/g' > /root/env.sh +printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh # start cron in the foreground crontab crontab diff --git a/restic_volume_backup/cli.py b/restic_volume_backup/cli.py index 1ccde04..e54cbb4 100644 --- a/restic_volume_backup/cli.py +++ b/restic_volume_backup/cli.py @@ -1,13 +1,28 @@ import argparse import pprint +import logging from restic_volume_backup.config import Config from restic_volume_backup.containers import RunningContainers from restic_volume_backup import backup_runner from restic_volume_backup import restic +logger = logging.getLogger(__name__) + + +def setup_logger(level=logging.INFO): + logger.setLevel(level) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + ch.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + logger.addHandler(ch) + + +setup_logger() + def main(): + """CLI entrypoint""" args = parse_args() config = Config() containers = RunningContainers() @@ -18,50 +33,45 @@ def main(): elif args.action == 'backup': backup(config, containers) - # Separate command to avoid spawning infinite containers :) elif args.action == 'start-backup-process': start_backup_process(config, containers) def status(config, containers): """Outputs the backup config for the compose setup""" - print() - print("Backup config for compose project '{}'".format(containers.this_container.project_name)) - print("Current service:", containers.this_container.name) - print("Backup process :", containers.backup_process_container.name - if containers.backup_process_container else 'Not Running') - print("Backup running :", containers.backup_process_running) - print() + logger.info("Backup config for compose project '%s'", containers.this_container.project_name) + logger.info("Current service: %s", containers.this_container.name) + logger.info("Backup process: %s", containers.backup_process_container.name + if containers.backup_process_container else 'Not Running') + logger.info("Backup running: %s", containers.backup_process_running) backup_containers = containers.containers_for_backup() for container in backup_containers: if container.backup_enabled: - print('service: {}'.format(container.service_name)) + logger.info('service: %s', container.service_name) for mount in container.filter_mounts(): - print(' - {}'.format(mount.source)) + logger.info(' - %s', mount.source) if len(backup_containers) == 0: - print("No containers in the project has 'restic-volume-backup.enabled' label") - - print() + logger.info("No containers in the project has 'restic-volume-backup.enabled' label") def backup(config, containers): - """Start backup""" + """Request a backup to start""" # Make sure we don't spawn multiple backup processes if containers.backup_process_running: raise ValueError("Backup process already running") - print("Initializing repository") + logger.info("Initializing repository") # TODO: Errors when repo already exists restic.init_repo(config.repository) - print("Starting backup container..") + logger.info("Starting backup container..") # Map all volumes from the backup container into the backup process container - volumes = containers.this_container.volumes() + volumes = containers.this_container.volumes # Map volumes from other containers we are backing up mounts = containers.generate_backup_mounts('/backup') @@ -90,14 +100,17 @@ def start_backup_process(config, containers): ) return - print("start-backup-process") + logger.info("start-backup-process") status(config, containers) + + # Waste a few seconds faking a backup + print("Fake backup running") import time for i in range(5): time.sleep(1) print(i) - exit(1) + exit(0) def parse_args(): diff --git a/restic_volume_backup/config.py b/restic_volume_backup/config.py index b86efa6..04f2238 100644 --- a/restic_volume_backup/config.py +++ b/restic_volume_backup/config.py @@ -2,6 +2,7 @@ import os class Config: + """Bag for config values""" def __init__(self, check=True): self.repository = os.environ['RESTIC_REPOSITORY'] self.password = os.environ['RESTIC_PASSWORD'] diff --git a/restic_volume_backup/containers.py b/restic_volume_backup/containers.py index 802055e..bcf49f1 100644 --- a/restic_volume_backup/containers.py +++ b/restic_volume_backup/containers.py @@ -9,7 +9,7 @@ VOLUME_TYPE_VOLUME = "volume" class Container: """Represents a docker container""" - def __init__(self, data): + def __init__(self, data: dict): self._data = data self.id = data['Id'] self.name = data['Name'].replace('/', '') @@ -31,17 +31,17 @@ class Container: self._exclude = self._parse_pattern(self.get_label('restic-volume-backup.exclude')) @property - def image(self): + def image(self) -> str: """Image name""" return self.get_config('Image') @property - def environment(self): + def environment(self) -> dict: """All configured env vars for the container""" return self.get_config('Env', default=[]) @property - def volumes(self): + def volumes(self) -> dict: """ Return volumes for the container in the following format: {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'},} @@ -65,15 +65,15 @@ class Container: ]) @property - def volume_backup_enabled(self): + def volume_backup_enabled(self) -> bool: return utils.is_true(self.get_label('restic-volume-backup.volumes')) @property - def mysql_backup_enabled(self): + def mysql_backup_enabled(self) -> bool: return utils.is_true(self.get_label('restic-volume-backup.mysql')) @property - def postgresql_backup_enabled(self): + def postgresql_backup_enabled(self) -> bool: return utils.is_true(self.get_label('restic-volume-backup.postgresql')) @property @@ -135,6 +135,7 @@ class Container: return filtered def volumes_for_backup(self, source_prefix='/backup', mode='ro'): + """Get volumes configured for backup""" mounts = self.filter_mounts() volumes = {} for mount in mounts: @@ -255,7 +256,7 @@ class RunningContainers: """Obtain all containers with backup enabled""" return [container for container in self.containers if container.backup_enabled] - def generate_backup_mounts(self, dest_prefix): + def generate_backup_mounts(self, dest_prefix) -> dict: mounts = {} for container in self.containers_for_backup(): mounts.update(container.volumes_for_backup(source_prefix=dest_prefix, mode='ro')) diff --git a/restic_volume_backup/restic.py b/restic_volume_backup/restic.py index ba724f7..273e6b8 100644 --- a/restic_volume_backup/restic.py +++ b/restic_volume_backup/restic.py @@ -1,7 +1,14 @@ +import logging from subprocess import Popen, PIPE +logger = logging.getLogger(__name__) + def init_repo(repository): + """ + Attempt to initialize the repository. + Doing this after the repository is initialized + """ run_command([ "restic", "-r", @@ -40,6 +47,7 @@ def check(repository): def run_command(cmd): + logger.info('cmd: %s', ' '.join(cmd)) child = Popen(cmd, stdout=PIPE, stderr=PIPE) stdoutdata, stderrdata = child.communicate() diff --git a/restic_volume_backup/utils.py b/restic_volume_backup/utils.py index 8d8d6db..cee752d 100644 --- a/restic_volume_backup/utils.py +++ b/restic_volume_backup/utils.py @@ -19,7 +19,7 @@ def list_containers(): return [c.attrs for c in all_containers] -def is_true(self, value): +def is_true(value): """ Evaluates the truthfullness of a bool value in container labels """ diff --git a/test.sh b/test.sh deleted file mode 100755 index 58de8f9..0000000 --- a/test.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -for i in 1 2 3 4 5; do echo 'moo'; sleep 2; done