From a3cdb38239e3c4c7823cb90639409c20d3a52114 Mon Sep 17 00:00:00 2001
From: Einar Forselv <eforselv@gmail.com>
Date: Sat, 13 Apr 2019 19:04:54 +0200
Subject: [PATCH] Initial
---
.dockerignore | 1 +
.gitignore | 2 +
.vscode/settings.json | 3 +
Dockerfile | 12 ++++
README.md | 15 +++++
requirements.txt | 1 +
restic-backup/backup.py | 54 +++++++++++++++
restic-backup/containers.py | 128 ++++++++++++++++++++++++++++++++++++
restic-backup/crontab | 1 +
restic-backup/entrypoint.sh | 8 +++
restic-backup/restic.py | 55 ++++++++++++++++
11 files changed, 280 insertions(+)
create mode 100644 .dockerignore
create mode 100644 .gitignore
create mode 100644 .vscode/settings.json
create mode 100644 Dockerfile
create mode 100644 README.md
create mode 100644 requirements.txt
create mode 100644 restic-backup/backup.py
create mode 100644 restic-backup/containers.py
create mode 100644 restic-backup/crontab
create mode 100755 restic-backup/entrypoint.sh
create mode 100644 restic-backup/restic.py
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..21d0b89
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1 @@
+.venv/
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..033df5f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.venv
+__pycache__
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..a3e76b4
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+ "python.pythonPath": "${workspaceFolder}/.venv/bin/python"
+}
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..0500db0
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,12 @@
+FROM restic/restic
+
+ADD requirements.txt /
+RUN apk update && apk add python3 dcron
+RUN pip3 install -r requirements.txt
+
+ADD restic-backup /restic-backup
+
+WORKDIR /restic-backup
+
+ENTRYPOINT []
+CMD ["/restic-backup/entrypoint.sh"]
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..d47c293
--- /dev/null
+++ b/README.md
@@ -0,0 +1,15 @@
+
+# restic-volume-backup
+
+*WORK IN PROGRESS*
+
+Backup using using https://restic.net/ for a docker-compose setup.
+
+Backs up all docker volumes. This includes both host mapped volumes and actual docker volumes.
+
+* Cron triggers backup
+* Volumes for all running containers are backed up
+
+## Configuration
+
+TODO
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..fe88db8
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+docker-py==1.10.6
diff --git a/restic-backup/backup.py b/restic-backup/backup.py
new file mode 100644
index 0000000..1f5e6f5
--- /dev/null
+++ b/restic-backup/backup.py
@@ -0,0 +1,54 @@
+import os
+import sys
+from containers import RunningContainers
+import restic
+
+cmds = ['volumes', 'backup', 'snapshots', 'check']
+
+
+class Config:
+ container_name = os.environ['CONTAINER_NAME']
+ password = os.environ['RESTIC_PASSWORD']
+
+ @classmethod
+ def check(cls):
+ if not cls.container_name:
+ raise ValueError("CONTAINER env var not set")
+
+ if not cls.password:
+ raise ValueError("PASSWORD env var not set")
+
+
+def main():
+ if len(sys.argv) < 2:
+ raise ValueError("Missing argument: {}".format(cmds))
+
+ mode = sys.argv[1]
+ if mode not in cmds:
+ raise ValueError("Valid arguments: {}".format(cmds))
+
+ Config.check()
+
+ containers = RunningContainers()
+
+ if mode == 'volumes':
+ volumes = containers.volume_mounts()
+ for vol in volumes:
+ print(vol)
+ print(vol.mount_string())
+
+ binds = containers.bind_mounts()
+ for vol in binds:
+ print(vol.mount_string())
+
+ if mode == 'backup':
+ restic.init_repo(Config.container_name)
+
+ for vol in containers.backup_volumes():
+ restic.backup_volume(Config.container_name, vol)
+
+ if mode == 'snapshots':
+ restic.snapshots(Config.container_name)
+
+if __name__ == '__main__':
+ main()
diff --git a/restic-backup/containers.py b/restic-backup/containers.py
new file mode 100644
index 0000000..050dc4e
--- /dev/null
+++ b/restic-backup/containers.py
@@ -0,0 +1,128 @@
+import os
+import docker
+import json
+
+DOCKER_BASE_URL = os.environ.get('DOCKER_BASE_URL') or "unix://var/run/docker.sock"
+VOLUME_TYPE_BIND = "bind"
+VOLUME_TYPE_VOLUME = "volume"
+
+class Container:
+
+ def __init__(self, data):
+ self.id = data.get('Id')
+ self.names = data.get('Names')
+ self.mounts = [Mount(mnt, container=self) for mnt in data.get('Mounts')]
+
+ def to_dict(self):
+ return {
+ "Id": self.id,
+ "Mounts": [mnt.data for mnt in self.mounts]
+ }
+
+
+class Mount:
+ """Mount wrapper"""
+ def __init__(self, data, container=None):
+ self.data = data
+ self._container = container
+
+ @property
+ def container(self):
+ return self._container
+
+ @property
+ def type(self):
+ return self.data.get('Type')
+
+ @property
+ def name(self):
+ return self.data.get('Name')
+
+ @property
+ def source(self):
+ return self.data.get('Source')
+
+ @property
+ def destination(self):
+ return self.data.get('Destination')
+
+ @property
+ def driver(self):
+ return self.data.get('Driver')
+
+ @property
+ def mode(self):
+ return self.data.get('Mode')
+
+ @property
+ def rw(self):
+ return self.data.get('RW')
+
+ def mount_string(self):
+ if self.type == VOLUME_TYPE_VOLUME:
+ return "- {}:{}:ro".format(self.name.split('_')[-1], self.destination)
+ elif self.type == VOLUME_TYPE_BIND:
+ return "- {}:{}:ro".format(self.source, self.destination)
+ else:
+ raise ValueError("Uknown volume type: {}".format(self.type))
+
+ def __repr__(self):
+ return str(self)
+
+ def __str__(self):
+ return str(self.data)
+
+ def __hash__(self):
+ """Uniquness for a volume"""
+ if self.type == VOLUME_TYPE_VOLUME:
+ return hash(self.name)
+ elif self.type == VOLUME_TYPE_BIND:
+ return hash(self.source)
+ else:
+ raise ValueError("Uknown volume type: {}".format(self.type))
+
+
+class RunningContainers:
+
+ def __init__(self):
+ client = docker.Client(base_url=DOCKER_BASE_URL)
+ all_containers = client.containers()
+ client.close()
+
+ self.containers = []
+ self.backup_container = None
+
+ for entry in all_containers:
+ if entry['Id'].startswith(os.environ['HOSTNAME']):
+ self.backup_container = Container(entry)
+ else:
+ if entry['State'] == "running":
+ self.containers.append(Container(entry))
+
+ if not self.backup_container:
+ raise ValueError("Cannot find metadata for backup container")
+
+ def backup_volumes(self):
+ return self.backup_container.mounts
+
+ def gen_volumes(self, volume_type):
+ """Generator yielding volumes of a specific type"""
+ for cont in self.containers:
+ for mnt in cont.mounts:
+ if mnt.type == volume_type:
+ yield mnt
+
+ def volume_mounts(self):
+ """Docker volumes"""
+ return set(mnt for mnt in self.gen_volumes(VOLUME_TYPE_VOLUME))
+
+ def bind_mounts(self):
+ """Host mapped volumes"""
+ return set(mnt for mnt in self.gen_volumes(VOLUME_TYPE_BIND))
+
+ def print_all(self):
+ print("Backup container:")
+ print(json.dumps(self.backup_container.to_dict(), indent=2))
+
+ print("All containers:")
+ print(json.dumps([cnt.to_dict() for cnt in self.containers], indent=2))
diff --git a/restic-backup/crontab b/restic-backup/crontab
new file mode 100644
index 0000000..eaf730f
--- /dev/null
+++ b/restic-backup/crontab
@@ -0,0 +1 @@
+30 * * * * source /env.sh && python3 restic-backup/backup.py backup >> /backup.log 2>&1
diff --git a/restic-backup/entrypoint.sh b/restic-backup/entrypoint.sh
new file mode 100755
index 0000000..511b586
--- /dev/null
+++ b/restic-backup/entrypoint.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Dump all env vars so we can source them in cron jobs
+printenv | sed 's/^\(.*\)$/export \1/g' > /root/env.sh
+
+# start cron in the foreground
+crontab crontab
+crond -f
diff --git a/restic-backup/restic.py b/restic-backup/restic.py
new file mode 100644
index 0000000..bf73b0f
--- /dev/null
+++ b/restic-backup/restic.py
@@ -0,0 +1,55 @@
+import os
+from subprocess import Popen, PIPE, check_call
+
+
+def repo_path(container_name):
+ return "swift:{}:/".format(container_name)
+
+
+def init_repo(container_name):
+ run_command([
+ "restic",
+ "-r",
+ repo_path(container_name),
+ "init",
+ ])
+
+
+def backup_volume(container_name, volume):
+ run_command([
+ "restic",
+ "-r",
+ repo_path(container_name),
+ "--verbose",
+ "backup",
+ volume.destination,
+ ])
+
+
+def snapshots(container_name):
+ run_command([
+ "restic",
+ "-r",
+ repo_path(container_name),
+ "snapshots",
+ ])
+
+
+def check(container_name):
+ run_command([
+ "restic",
+ "-r",
+ repo_path(container_name),
+ "check",
+ ])
+
+
+def run_command(cmd):
+ child = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ stdoutdata, stderrdata = child.communicate()
+
+ if stdoutdata:
+ print(stdoutdata.decode())
+
+ if stderrdata:
+ print(stderrdata.decode())