From a3cdb38239e3c4c7823cb90639409c20d3a52114 Mon Sep 17 00:00:00 2001 From: Einar Forselv Date: Sat, 13 Apr 2019 19:04:54 +0200 Subject: [PATCH] Initial --- .dockerignore | 1 + .gitignore | 2 + .vscode/settings.json | 3 + Dockerfile | 12 ++++ README.md | 15 +++++ requirements.txt | 1 + restic-backup/backup.py | 54 +++++++++++++++ restic-backup/containers.py | 128 ++++++++++++++++++++++++++++++++++++ restic-backup/crontab | 1 + restic-backup/entrypoint.sh | 8 +++ restic-backup/restic.py | 55 ++++++++++++++++ 11 files changed, 280 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 restic-backup/backup.py create mode 100644 restic-backup/containers.py create mode 100644 restic-backup/crontab create mode 100755 restic-backup/entrypoint.sh create mode 100644 restic-backup/restic.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..21d0b89 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.venv/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..033df5f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.venv +__pycache__ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a3e76b4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "${workspaceFolder}/.venv/bin/python" +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0500db0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM restic/restic + +ADD requirements.txt / +RUN apk update && apk add python3 dcron +RUN pip3 install -r requirements.txt + +ADD restic-backup /restic-backup + +WORKDIR /restic-backup + +ENTRYPOINT [] +CMD ["/restic-backup/entrypoint.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..d47c293 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ + +# restic-volume-backup + +*WORK IN PROGRESS* + +Backup using using https://restic.net/ for a docker-compose setup. + +Backs up all docker volumes. This includes both host mapped volumes and actual docker volumes. + +* Cron triggers backup +* Volumes for all running containers are backed up + +## Configuration + +TODO diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fe88db8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +docker-py==1.10.6 diff --git a/restic-backup/backup.py b/restic-backup/backup.py new file mode 100644 index 0000000..1f5e6f5 --- /dev/null +++ b/restic-backup/backup.py @@ -0,0 +1,54 @@ +import os +import sys +from containers import RunningContainers +import restic + +cmds = ['volumes', 'backup', 'snapshots', 'check'] + + +class Config: + container_name = os.environ['CONTAINER_NAME'] + password = os.environ['RESTIC_PASSWORD'] + + @classmethod + def check(cls): + if not cls.container_name: + raise ValueError("CONTAINER env var not set") + + if not cls.password: + raise ValueError("PASSWORD env var not set") + + +def main(): + if len(sys.argv) < 2: + raise ValueError("Missing argument: {}".format(cmds)) + + mode = sys.argv[1] + if mode not in cmds: + raise ValueError("Valid arguments: {}".format(cmds)) + + Config.check() + + containers = RunningContainers() + + if mode == 'volumes': + volumes = containers.volume_mounts() + for vol in volumes: + print(vol) + print(vol.mount_string()) + + binds = containers.bind_mounts() + for vol in binds: + print(vol.mount_string()) + + if mode == 'backup': + restic.init_repo(Config.container_name) + + for vol in containers.backup_volumes(): + restic.backup_volume(Config.container_name, vol) + + if mode == 'snapshots': + restic.snapshots(Config.container_name) + +if __name__ == '__main__': + main() diff --git a/restic-backup/containers.py b/restic-backup/containers.py new file mode 100644 index 0000000..050dc4e --- /dev/null +++ b/restic-backup/containers.py @@ -0,0 +1,128 @@ +import os +import docker +import json + +DOCKER_BASE_URL = os.environ.get('DOCKER_BASE_URL') or "unix://var/run/docker.sock" +VOLUME_TYPE_BIND = "bind" +VOLUME_TYPE_VOLUME = "volume" + +class Container: + + def __init__(self, data): + self.id = data.get('Id') + self.names = data.get('Names') + self.mounts = [Mount(mnt, container=self) for mnt in data.get('Mounts')] + + def to_dict(self): + return { + "Id": self.id, + "Mounts": [mnt.data for mnt in self.mounts] + } + + +class Mount: + """Mount wrapper""" + def __init__(self, data, container=None): + self.data = data + self._container = container + + @property + def container(self): + return self._container + + @property + def type(self): + return self.data.get('Type') + + @property + def name(self): + return self.data.get('Name') + + @property + def source(self): + return self.data.get('Source') + + @property + def destination(self): + return self.data.get('Destination') + + @property + def driver(self): + return self.data.get('Driver') + + @property + def mode(self): + return self.data.get('Mode') + + @property + def rw(self): + return self.data.get('RW') + + def mount_string(self): + if self.type == VOLUME_TYPE_VOLUME: + return "- {}:{}:ro".format(self.name.split('_')[-1], self.destination) + elif self.type == VOLUME_TYPE_BIND: + return "- {}:{}:ro".format(self.source, self.destination) + else: + raise ValueError("Uknown volume type: {}".format(self.type)) + + def __repr__(self): + return str(self) + + def __str__(self): + return str(self.data) + + def __hash__(self): + """Uniquness for a volume""" + if self.type == VOLUME_TYPE_VOLUME: + return hash(self.name) + elif self.type == VOLUME_TYPE_BIND: + return hash(self.source) + else: + raise ValueError("Uknown volume type: {}".format(self.type)) + + +class RunningContainers: + + def __init__(self): + client = docker.Client(base_url=DOCKER_BASE_URL) + all_containers = client.containers() + client.close() + + self.containers = [] + self.backup_container = None + + for entry in all_containers: + if entry['Id'].startswith(os.environ['HOSTNAME']): + self.backup_container = Container(entry) + else: + if entry['State'] == "running": + self.containers.append(Container(entry)) + + if not self.backup_container: + raise ValueError("Cannot find metadata for backup container") + + def backup_volumes(self): + return self.backup_container.mounts + + def gen_volumes(self, volume_type): + """Generator yielding volumes of a specific type""" + for cont in self.containers: + for mnt in cont.mounts: + if mnt.type == volume_type: + yield mnt + + def volume_mounts(self): + """Docker volumes""" + return set(mnt for mnt in self.gen_volumes(VOLUME_TYPE_VOLUME)) + + def bind_mounts(self): + """Host mapped volumes""" + return set(mnt for mnt in self.gen_volumes(VOLUME_TYPE_BIND)) + + def print_all(self): + print("Backup container:") + print(json.dumps(self.backup_container.to_dict(), indent=2)) + + print("All containers:") + print(json.dumps([cnt.to_dict() for cnt in self.containers], indent=2)) diff --git a/restic-backup/crontab b/restic-backup/crontab new file mode 100644 index 0000000..eaf730f --- /dev/null +++ b/restic-backup/crontab @@ -0,0 +1 @@ +30 * * * * source /env.sh && python3 restic-backup/backup.py backup >> /backup.log 2>&1 diff --git a/restic-backup/entrypoint.sh b/restic-backup/entrypoint.sh new file mode 100755 index 0000000..511b586 --- /dev/null +++ b/restic-backup/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Dump all env vars so we can source them in cron jobs +printenv | sed 's/^\(.*\)$/export \1/g' > /root/env.sh + +# start cron in the foreground +crontab crontab +crond -f diff --git a/restic-backup/restic.py b/restic-backup/restic.py new file mode 100644 index 0000000..bf73b0f --- /dev/null +++ b/restic-backup/restic.py @@ -0,0 +1,55 @@ +import os +from subprocess import Popen, PIPE, check_call + + +def repo_path(container_name): + return "swift:{}:/".format(container_name) + + +def init_repo(container_name): + run_command([ + "restic", + "-r", + repo_path(container_name), + "init", + ]) + + +def backup_volume(container_name, volume): + run_command([ + "restic", + "-r", + repo_path(container_name), + "--verbose", + "backup", + volume.destination, + ]) + + +def snapshots(container_name): + run_command([ + "restic", + "-r", + repo_path(container_name), + "snapshots", + ]) + + +def check(container_name): + run_command([ + "restic", + "-r", + repo_path(container_name), + "check", + ]) + + +def run_command(cmd): + child = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdoutdata, stderrdata = child.communicate() + + if stdoutdata: + print(stdoutdata.decode()) + + if stderrdata: + print(stderrdata.decode())