Major repo reorganizaion

restic_volume_backup/backup.py

@@ -0,0 +1,61 @@
+import os
+import sys
+from containers import RunningContainers
+import restic
+
+cmds = ['status', 'backup', 'snapshots', 'check']
+
+
+class Config:
+    repository = os.environ['RESTIC_REPOSITORY']
+    password = os.environ['RESTIC_PASSWORD']
+
+    @classmethod
+    def check(cls):
+        if not cls.repository:
+            raise ValueError("CONTAINER env var not set")
+
+        if not cls.password:
+            raise ValueError("PASSWORD env var not set")
+
+
+def main():
+    if len(sys.argv) < 2:
+        raise ValueError("Missing argument: {}".format(cmds))
+
+    mode = sys.argv[1]
+    if mode not in cmds:
+        raise ValueError("Valid arguments: {}".format(cmds))
+
+    Config.check()
+
+    containers = RunningContainers()
+
+    if mode == 'status':
+        containers.print_services()
+        # volumes = containers.volume_mounts()
+        # for vol in volumes:
+        #     print(vol)
+        #     print(vol.mount_string())
+
+        # binds = containers.bind_mounts()
+        # for vol in binds:
+        #     print(binds)
+        #     print(vol.mount_string())
+
+    if mode == 'backup':
+        print("Starting backup ..")
+        # TODO: Errors when repo already exists
+        # restic.init_repo(Config.repository)
+
+        # for vol in containers.backup_volumes():
+        #     restic.backup_volume(Config.repository, vol)
+
+
+
+    if mode == 'snapshots':
+        restic.snapshots(Config.repository)
+
+
+if __name__ == '__main__':
+    main()

restic_volume_backup/containers.py

@@ -0,0 +1,189 @@
+import os
+import docker
+import json
+import pprint
+
+DOCKER_BASE_URL = os.environ.get('DOCKER_BASE_URL') or "unix://tmp/docker.sock"
+VOLUME_TYPE_BIND = "bind"
+VOLUME_TYPE_VOLUME = "volume"
+
+
+class Container:
+
+    def __init__(self, data):
+        self.id = data.get('Id')
+        self.state = data.get('State')
+        self.labels = data.get('Labels', {})
+        self.names = data.get('Names', [])
+        self.mounts = [Mount(mnt, container=self) for mnt in data.get('Mounts')]
+
+        self.include = self.labels.get('restic-volume-backup.enabled', '').split(',')
+        self.exlude = self.labels.get('restic-volume-backup.exclude', '').split(',')
+
+    @property
+    def backup_enabled(self):
+        return self.labels.get('restic-volume-backup.enabled') == 'True'
+
+    @property
+    def is_running(self):
+        return self.state == 'running'
+
+    @property
+    def service_name(self):
+        return self.labels['com.docker.compose.service']
+
+    @property
+    def project_name(self):
+        return self.labels['com.docker.compose.project']
+
+    @property
+    def is_oneoff(self):
+        return self.labels['com.docker.compose.oneoff'] == 'True'
+
+    def filter_mounts(self):
+        """Get all mounts for this container matching filters"""
+        for mount in self.mounts:
+            if self.include:
+                for pattern in self.include:
+                    if pattern in mount.source:
+                        yield mount
+                        continue
+            elif self.exlude:
+                for pattern in self.exlude:
+                    if pattern in mount.source:
+                        continue
+                    yield mount
+            else:
+                yield mount
+
+    def to_dict(self):
+        return {
+            'Id': self.id,
+            'Names': self.names,
+            'State': self.state,
+            'Labels': self.labels,
+            'Mounts': [mnt.data for mnt in self.mounts],
+            'include': self.include,
+            'exclude': self.exlude,
+        }
+
+
+class Mount:
+    """Mount wrapper"""
+    def __init__(self, data, container=None):
+        self._data = data
+        self._container = container
+
+    @property
+    def container(self):
+        return self._container
+
+    @property
+    def type(self):
+        return self._data.get('Type')
+
+    @property
+    def name(self):
+        return self._data.get('Name')
+
+    @property
+    def source(self):
+        return self._data.get('Source')
+
+    @property
+    def destination(self):
+        return self._data.get('Destination')
+
+    def mount_string(self):
+        if self.type == VOLUME_TYPE_VOLUME:
+            return "- {}:{}:ro".format(self.name.split('_')[-1], self.destination)
+        elif self.type == VOLUME_TYPE_BIND:
+            return "- {}:{}:ro".format(self.source, self.destination)
+        else:
+            raise ValueError("Uknown volume type: {}".format(self.type))
+
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self):
+        return str(self._data)
+
+    def __hash__(self):
+        """Uniquness for a volume"""
+        if self.type == VOLUME_TYPE_VOLUME:
+            return hash(self.name)
+        elif self.type == VOLUME_TYPE_BIND:
+            return hash(self.source)
+        else:
+            raise ValueError("Uknown volume type: {}".format(self.type))
+
+
+class RunningContainers:
+
+    def __init__(self):
+        client = docker.Client(base_url=DOCKER_BASE_URL)
+        all_containers = client.containers()
+        client.close()
+
+        self.containers = []
+        self.this_container = None
+
+        # Find the container we are running in.
+        # If we don't have this information we cannot continue
+        for container_data in all_containers:
+            if container_data.get('Id').startswith(os.environ['HOSTNAME']):
+                self.this_container = Container(container_data)
+
+        if not self.this_container:
+            raise ValueError("Cannot find metadata for backup container")
+
+        # Gather all containers in the current compose setup
+        for container_data in all_containers:
+            # pprint.pprint(container_data, indent=2)
+            container = Container(container_data)
+            if container.project_name == self.this_container.project_name:
+                if container.id != self.this_container.id:
+                    self.containers.append(container)
+
+        # for container in self.all_containers:
+        #     # Weed out containers not beloging to this project.
+        #     if container.project_name != self.backup_container.project_name:
+        #         continue
+
+        #     # Keep only containers with backup enabled
+        #     if not container.backup_enabled:
+        #         continue
+
+        #     # and not oneoffs (started manually with run or similar)
+        #     if container.is_oneoff:
+        #         continue
+
+        #     self.containers.append(container)
+
+    # def gen_volumes(self, volume_type):
+    #     """Generator yielding volumes of a specific type"""
+    #     for cont in self.containers:
+    #         for mnt in cont.mounts:
+    #             if mnt.type == volume_type:
+    #                 yield mnt
+
+    # def volume_mounts(self):
+    #     """Docker volumes"""
+    #     return set(mnt for mnt in self.gen_volumes(VOLUME_TYPE_VOLUME))
+
+    # def bind_mounts(self):
+    #     """Host mapped volumes"""
+    #     return set(mnt for mnt in self.gen_volumes(VOLUME_TYPE_BIND))
+
+    def print_services(self):
+        print()
+        print("Backup config for compose project '{}'".format(self.this_container.project_name))
+        print()
+
+        for container in self.containers:
+            print('service: {}'.format(container.service_name))
+            mounts = container.filter_mounts()
+            for mount in mounts:
+                print(' - {}'.format(mount.source))
+
+        print()

restic_volume_backup/restic.py

@@ -0,0 +1,51 @@
+import os
+from subprocess import Popen, PIPE, check_call
+
+
+def init_repo(repository):
+    run_command([
+        "restic",
+        "-r",
+        repository,
+        "init",
+    ])
+
+
+def backup_volume(repository, volume):
+    run_command([
+        "restic",
+        "-r",
+        repository,
+        "--verbose",
+        "backup",
+        volume.destination,
+    ])
+
+
+def snapshots(repository):
+    run_command([
+        "restic",
+        "-r",
+        repository,
+        "snapshots",
+    ])
+
+
+def check(repository):
+    run_command([
+        "restic",
+        "-r",
+        repository,
+        "check",
+    ])
+
+
+def run_command(cmd):
+    child = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    stdoutdata, stderrdata = child.communicate()
+
+    if stdoutdata:
+        print(stdoutdata.decode())
+
+    if stderrdata:
+        print(stderrdata.decode())

restic_volume_backup/runner.py

@@ -0,0 +1,25 @@
+import os
+import docker
+
+
+client = docker.Client(base_url=DOCKER_BASE_URL)
+
+container = client.containers.run(
+    'image',
+    'command',
+    labels={"restic-volume-backup.process": True},
+    auto_remove=True,
+    remove=True,
+    detach=True,
+    environment={
+        'test1': 'value1',
+        'test2': 'value2',
+    },
+    volumes={
+        '/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'},
+        '/var/www': {'bind': '/mnt/vol1', 'mode': 'ro'},
+    },
+    working_dir=os.getcwd(),
+)
+
+# Pull logs and exist status of container