status command should display repository

.dockerignore

@@ -1,9 +0,0 @@
-.venv/
-.vscode/
-.gitignore
-Dockerfile
-tests/
-docker-compose.yaml
-*.env
-*.egg-info
-__pycache__

.gitignore

@@ -19,3 +19,11 @@ venv
 /private/
 restic_data/
 restic_cache/
+alerts.env
+
+# docs
+build/
+docs/_build
+
+# tests
+.tox

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Zetta.IO
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,22 +1,42 @@
 
 # restic-compose-backup
 
-*WORK IN PROGRESS*
+![docs](https://readthedocs.org/projects/restic-compose-backup/badge/?version=latest)
 
 Backup using https://restic.net/ for a docker-compose setup.
 
+* [restic-compose-backup Documentation](https://restic-compose-backup.readthedocs.io)
+* [restic-compose-backup on Github](https://github.com/ZettaIO/restic-compose-backup)
+* [restic-compose-backup on Docker Hub](https://hub.docker.com/r/zettaio/restic-compose-backup)
+
+Features:
+
+* Back up docker volumes or host binds
+* Back up mariadb postgres
+* Back up mariadb databases
+* Back up mysql databases
+* Notifications over mail/smtp
+* Notifications to Discord through webhooks
+
+Please report issus on [github](https://github.com/ZettaIO/restic-compose-backup/issues).
+
 Automatically detects and backs up volumes, mysql, mariadb and postgres databases in a docker-compose setup.
-This includes both host mapped volumes and actual docker volumes.
 
 * Each service in the compose setup is configured with a label
   to enable backup of volumes or databases
 * When backup starts a new instance of the container is created
   mapping in all the needed volumes. It will copy networks etc
   to ensure databases can be reached
-* Volumes are mounted to `/backup/<service_name>/<path>`
+* Volumes are mounted to `/volumes/<service_name>/<path>`
-  in the backup process container. `/backup` is pushed into restic
+  in the backup process container. `/volumes` is pushed into restic
-* Databases are backed up from stdin / dumps
+* Databases are backed up from stdin / dumps into restic using path `/databases/<service_name>/dump.sql`
-* Cron triggers backup
+* Cron triggers backup at 2AM every day
+
+## Install
+
+```bash
+docker pull zettaio/restic-compose-backup
+```
 
 ## Configuration
 
@@ -29,6 +49,29 @@ RESTIC_PASSWORD
 
 Backend specific env vars : https://restic.readthedocs.io/en/stable/040_backup.html#environment-variables
 
+Additional env vars:
+
+```bash
+# Prune rules
+RESTIC_KEEP_DAILY=7
+RESTIC_KEEP_WEEKLY=4
+RESTIC_KEEP_MONTHLY=12
+RESTIC_KEEP_YEARLY=3
+
+# Logging level (debug,info,warning,error)
+LOG_LEVEL=info
+
+# SMTP alerts
+EMAIL_HOST=my.mail.host
+EMAIL_PORT=465
+EMAIL_HOST_USER=johndoe
+EMAIL_HOST_PASSWORD=s3cr3tpassw0rd
+EMAIL_SEND_TO=johndoe@gmail.com
+
+# Discord webhook
+DISCORD_WEBHOOK=https://discordapp.com/api/webhooks/...
+```
+
 ### Volumes
 
 ```yaml
@@ -36,10 +79,14 @@ version: '3'
 services:
   # The backup service
   backup:
-    build: restic-compose-backup
+    image: zettaio/restic-compose-backup
     environment:
       - RESTIC_REPOSITORY=<whatever restic supports>
       - RESTIC_PASSWORD=hopefullyasecturepw
+      - RESTIC_KEEP_DAILY=7
+      - RESTIC_KEEP_WEEKLY=4
+      - RESTIC_KEEP_MONTHLY=12
+      - RESTIC_KEEP_YEARLY=3
     env_file:
       - some_other_vars.env
     volumes:
@@ -106,7 +153,7 @@ volumes:
 
 Will dump databases directly into restic through stdin.
 They will appear in restic as a separate snapshot with
-path `/backup/<service_name>/dump.sql` or similar.
+path `/databases/<service_name>/dump.sql` or similar.
 
 ```yaml
   mariadb:
@@ -129,11 +176,21 @@ path `/backup/<service_name>/dump.sql` or similar.
       restic-compose-backup.postgres: true
 ```
 
-
 ## Running Tests
 
+```bash
+pip install -e src/
+pip install -r src/tests/requirements.txt
+tox
 ```
-python setup.py develop
+
-pip install -r tests/requirements.txt
+## Building Docs
-pytest tests
+
+```bash
+pip install -r docs/requirements.txt
+python src/setup.py build_sphinx
 ```
+
+## Contributing
+
+Contributions are welcome regardless of experience level. Don't hesitate submitting issues, opening partial or completed pull requests.

crontab

@@ -1 +0,0 @@
-1 * * * * source /env.sh && rcb backup > /proc/1/fd/1

docker-compose.yaml

@@ -1,17 +1,18 @@
 version: '3'
 services:
   backup:
-    build: .
+    build: ./src
     env_file:
       - restic_compose_backup.env
+      - alerts.env
     volumes:
       # Map in docker socket
       - /var/run/docker.sock:/tmp/docker.sock:ro
       # Map backup database locally
       - ./restic_data:/restic_data
-      - ./restic_cache:/restic_cache
+      - ./restic_cache:/cache
-      # Map in project source
+      # Map in project source in dev
-      - .:/restic-compose-backup
+      - ./src:/restic-compose-backup
   web:
     image: nginx
     labels:

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,57 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'restic-compose-backup'
+copyright = '2019, Zetta.IO Technology AS'
+author = 'Zetta.IO Technology AS'
+
+# The full version, including alpha/beta/rc tags
+release = '0.3.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+master_doc = 'index'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']

docs/index.rst

@@ -0,0 +1,21 @@
+.. restic-compose-backup documentation master file, created by
+   sphinx-quickstart on Thu Dec  5 01:34:58 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to restic-compose-backup's documentation!
+=================================================
+
+Simple backup with restic for small to medium docker-compose setups.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/requirements.txt

@@ -0,0 +1,2 @@
+sphinx
+sphinx-rtd-theme

extras/release.md

@@ -0,0 +1,20 @@
+# Making a release
+
+- Update version in `setup.py`
+- Update version in `docs/conf.py`
+- Build and tag image
+- push: `docker push zettaio/restic-compose-backup:<version>`
+- Ensure RTD has new docs published
+
+## Example
+
+When releasing a bugfix version we need to update the
+main image as well.
+
+```bash
+docker build . --tag zettaio/restic-compose-backup:0.3
+docker build . --tag zettaio/restic-compose-backup:0.3.1
+
+docker push zettaio/restic-compose-backup:0.3
+docker push zettaio/restic-compose-backup:0.3.1
+```

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-testpaths = tests
+testpaths = src/tests
 python_files=test*.py
 addopts = -v --verbose

restic_compose_backup.env

@@ -4,6 +4,21 @@ DOCKER_BASE_URL=unix://tmp/docker.sock
 RESTIC_REPOSITORY=/restic_data
 RESTIC_PASSWORD=password
 
+RESTIC_KEEP_DAILY=7
+RESTIC_KEEP_WEEKLY=4
+RESTIC_KEEP_MONTHLY=12
+RESTIC_KEEP_YEARLY=3
+
+LOG_LEVEL=info
+
+# EMAIL_HOST=
+# EMAIL_PORT=
+# EMAIL_HOST_USER=
+# EMAIL_HOST_PASSWORD=
+# EMAIL_SEND_TO=
+
+# DISCORD_WEBHOOK=u
+
 # Various env vars for restic : https://restic.readthedocs.io/en/stable/040_backup.html#environment-variables
 # RESTIC_REPOSITORY                   Location of repository (replaces -r)
 # RESTIC_PASSWORD_FILE                Location of password file (replaces --password-file)

restic_compose_backup/cli.py

@@ -1,139 +0,0 @@
-import argparse
-import pprint
-import logging
-
-from restic_compose_backup import (
-    backup_runner,
-    log,
-    restic,
-)
-from restic_compose_backup.config import Config
-from restic_compose_backup.containers import RunningContainers
-
-logger = logging.getLogger(__name__)
-
-
-def main():
-    """CLI entrypoint"""
-    args = parse_args()
-    config = Config()
-    containers = RunningContainers()
-
-    if args.action == 'status':
-        status(config, containers)
-
-    elif args.action == 'backup':
-        backup(config, containers)
-
-    elif args.action == 'start-backup-process':
-        start_backup_process(config, containers)
-
-
-def status(config, containers):
-    """Outputs the backup config for the compose setup"""
-
-    logger.info("Backup config for compose project '%s'", containers.this_container.project_name)
-    logger.info("Current service: %s", containers.this_container.name)
-    # logger.info("Backup process: %s", containers.backup_process_container.name
-    #             if containers.backup_process_container else 'Not Running')
-    logger.info("Backup running: %s", containers.backup_process_running)
-
-    backup_containers = containers.containers_for_backup()
-    for container in backup_containers:
-        logger.info('service: %s', container.service_name)
-
-        if container.volume_backup_enabled:
-            for mount in container.filter_mounts():
-                logger.info(' - volume: %s', mount.source)
-
-        if container.database_backup_enabled:
-            instance = container.instance
-            ping = instance.ping()
-            logger.info(' - %s (is_ready=%s)', instance.container_type, ping == 0)
-
-    if len(backup_containers) == 0:
-        logger.info("No containers in the project has 'restic-compose-backup.enabled' label")
-
-
-def backup(config, containers):
-    """Request a backup to start"""
-    # Make sure we don't spawn multiple backup processes
-    if containers.backup_process_running:
-        raise ValueError("Backup process already running")
-
-    logger.info("Initializing repository")
-
-    # TODO: Errors when repo already exists
-    restic.init_repo(config.repository)
-
-    logger.info("Starting backup container..")
-
-    # Map all volumes from the backup container into the backup process container
-    volumes = containers.this_container.volumes
-
-    # Map volumes from other containers we are backing up
-    mounts = containers.generate_backup_mounts('/backup')
-    volumes.update(mounts)
-
-    result = backup_runner.run(
-        image=containers.this_container.image,
-        command='restic-compose-backup start-backup-process',
-        volumes=volumes,
-        environment=containers.this_container.environment,
-        source_container_id=containers.this_container.id,
-        labels={
-            "restic-compose-backup.backup_process": 'True',
-            "com.docker.compose.project": containers.this_container.project_name,
-        },
-    )
-    logger.info('Backup container exit code: %s', result)
-    # TODO: Alert
-
-
-def start_backup_process(config, containers):
-    """The actual backup process running inside the spawned container"""
-    if (not containers.backup_process_container
-       or containers.this_container == containers.backup_process_container is False):
-        logger.error(
-            "Cannot run backup process in this container. Use backup command instead. "
-            "This will spawn a new container with the necessary mounts."
-        )
-        return
-
-    status(config, containers)
-    logger.info("start-backup-process")
-
-    # Back up volumes
-    try:
-        vol_result = restic.backup_files(config.repository, source='/backup')
-        logger.info('Volume backup exit code: %s', vol_result)
-        # TODO: Alert
-    except Exception as ex:
-        logger.error(ex)
-        # TODO: Alert
-
-    # back up databases
-    for container in containers.containers_for_backup():
-        if container.database_backup_enabled:
-            try:
-                instance = container.instance
-                logger.info('Backing up %s in service %s', instance.container_type, instance.service_name)
-                result = instance.backup()
-                logger.info('Exit code: %s', result)
-                # TODO: Alert
-            except Exception as ex:
-                logger.error(ex)
-                # TODO: Alert
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(prog='restic_compose_backup')
-    parser.add_argument(
-        'action',
-        choices=['status', 'backup', 'start-backup-process'],
-    )
-    return parser.parse_args()
-
-
-if __name__ == '__main__':
-    main()

restic_compose_backup/config.py

@@ -1,19 +0,0 @@
-import os
-
-
-class Config:
-    """Bag for config values"""
-    def __init__(self, check=True):
-        self.repository = os.environ['RESTIC_REPOSITORY']
-        self.password = os.environ['RESTIC_PASSWORD']
-        self.docker_base_url = os.environ.get('DOCKER_BASE_URL') or "unix://tmp/docker.sock"
-
-        if check:
-            self.check()
-
-    def check(self):
-        if not self.repository:
-            raise ValueError("CONTAINER env var not set")
-
-        if not self.password:
-            raise ValueError("PASSWORD env var not set")

restic_compose_backup/log.py

@@ -1,13 +0,0 @@
-import logging
-import os
-import sys
-
-logger = logging.getLogger('restic_compose_backup')
-HOSTNAME = os.environ['HOSTNAME']
-
-level = logging.INFO
-logger.setLevel(level)
-ch = logging.StreamHandler(stream=sys.stdout)
-ch.setLevel(level)
-ch.setFormatter(logging.Formatter(f'%(asctime)s - {HOSTNAME} - %(name)s - %(levelname)s - %(message)s'))
-logger.addHandler(ch)

restic_compose_backup/mail.py

@@ -1,37 +0,0 @@
-"""
-"""
-import smtplib
-from email.mime.text import MIMEText
-
-EMAIL_HOST = "smtp.gmail.com"
-EMAIL_PORT = 465
-EMAIL_HOST_USER = ""
-EMAIL_HOST_PASSWORD = ""
-EMAIL_SEND_TO = ['']
-
-
-def main():
-    send_mail("Hello world!")
-
-
-def send_mail(text):
-    msg = MIMEText(text)
-    msg['Subject'] = "Message from restic-compose-backup"
-    msg['From'] = EMAIL_HOST_USER
-    msg['To'] = ', '.join(EMAIL_SEND_TO)
-
-    try:
-        print("Connecting to {} port {}".format(EMAIL_HOST, EMAIL_PORT))
-        server = smtplib.SMTP_SSL(EMAIL_HOST, EMAIL_PORT)
-        server.ehlo()
-        server.login(EMAIL_HOST_USER, EMAIL_HOST_PASSWORD)
-        server.sendmail(EMAIL_HOST_USER, EMAIL_SEND_TO, msg.as_string())
-        print('Email Sent')
-    except Exception as e:
-        print(e)
-    finally:
-        server.close()
-
-
-if __name__ == '__main__':
-    main()

src/.dockerignore

@@ -0,0 +1,19 @@
+.venv/
+.vscode/
+extras/
+restic_cache/
+restic_data/
+tests/
+.gitignore
+*.env
+*.log
+docker-compose.yaml
+*.ini
+*.egg-info
+__pycache__
+.DS_Store
+.git
+.pytest_cache
+.dockerignore
+build/
+docs/

src/Dockerfile

@@ -4,8 +4,8 @@ RUN apk update && apk add python3 dcron mariadb-client postgresql-client
 
 ADD . /restic-compose-backup
 WORKDIR /restic-compose-backup
-RUN pip3 install -U pip setuptools
+RUN pip3 install -U pip setuptools && pip3 install -e .
-RUN pip3 install -e .
+ENV XDG_CACHE_HOME=/cache
 
 ENTRYPOINT []
 CMD ["./entrypoint.sh"]

src/crontab

@@ -0,0 +1 @@
+0 2 * * * source /env.sh && rcb backup > /proc/1/fd/1

src/restic_compose_backup/alerts/__init__.py

@@ -0,0 +1,43 @@
+import logging
+
+from restic_compose_backup.alerts.smtp import SMTPAlert
+from restic_compose_backup.alerts.discord import DiscordWebhookAlert
+
+logger = logging.getLogger(__name__)
+
+ALERT_INFO = 'INFO',
+ALERT_ERROR = 'ERROR'
+ALERT_TYPES = [ALERT_INFO, ALERT_ERROR]
+BACKENDS = [SMTPAlert, DiscordWebhookAlert]
+
+
+def send(subject: str = None, body: str = None, alert_type: str = 'INFO'):
+    """Send alert to all configured backends"""
+    alert_classes = configured_alert_types()
+    for instance in alert_classes:
+        logger.info('Configured: %s', instance.name)
+        try:
+            instance.send(
+                subject=f'[{alert_type}] {subject}',
+                body=body,
+            )
+        except Exception as ex:
+            logger.error("Exception raised when sending alert [%s]: %s", instance.name, ex)
+            logger.exception(ex)
+
+    if len(alert_classes) == 0:
+        logger.info("No alerts configured")
+
+
+def configured_alert_types():
+    """Returns a list of configured alert class instances"""
+    logger.debug('Getting alert backends')
+    entires = []
+
+    for cls in BACKENDS:
+        instance = cls.create_from_env()
+        logger.debug("Alert backend '%s' configured: %s", cls.name, instance is not None)
+        if instance:
+            entires.append(instance)
+
+    return entires

src/restic_compose_backup/alerts/base.py

@@ -0,0 +1,14 @@
+
+
+class BaseAlert:
+    name = None
+
+    def create_from_env(self):
+        return None
+
+    @property
+    def properly_configured(self) -> bool:
+        return False
+
+    def send(self, subject: str = None, body: str = None, alert_type: str = None):
+        pass

src/restic_compose_backup/alerts/discord.py

@@ -0,0 +1,45 @@
+import os
+import logging
+
+import requests
+from restic_compose_backup.alerts.base import BaseAlert
+
+logger = logging.getLogger(__name__)
+
+
+class DiscordWebhookAlert(BaseAlert):
+    name = 'discord_webhook'
+    success_codes = [200]
+
+    def __init__(self, webhook_url):
+        self.url = webhook_url
+
+    @classmethod
+    def create_from_env(cls):
+        instance = cls(os.environ.get('DISCORD_WEBHOOK'))
+
+        if instance.properly_configured:
+            return instance
+
+        return None
+
+    @property
+    def properly_configured(self) -> bool:
+        return isinstance(self.url, str) and self.url.startswith("https://")
+
+    def send(self, subject: str = None, body: str = None, alert_type: str = None):
+        """Send basic webhook request. Max embed size is 6000"""
+        logger.info("Triggering discord webhook")
+        data = {
+            'embeds': [
+                {
+                    'title': subject,
+                    'description': body[:5000],
+                },
+            ]
+        }
+        response = requests.post(self.url, params={'wait': True}, json=data)
+        if response.status_code not in self.success_codes:
+            logger.error("Discord webhook failed: %s: %s", response.status_code, response.content)
+        else:
+            logger.info('Discord webhook successful')

src/restic_compose_backup/alerts/smtp.py

@@ -0,0 +1,56 @@
+import os
+import smtplib
+import logging
+from email.mime.text import MIMEText
+
+from restic_compose_backup.alerts.base import BaseAlert
+
+logger = logging.getLogger(__name__)
+
+
+class SMTPAlert(BaseAlert):
+    name = 'smtp'
+
+    def __init__(self, host, port, user, password, to):
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.to = to
+
+    @classmethod
+    def create_from_env(cls):
+        instance = cls(
+            os.environ.get('EMAIL_HOST'),
+            os.environ.get('EMAIL_PORT'),
+            os.environ.get('EMAIL_HOST_USER'),
+            os.environ.get('EMAIL_HOST_PASSWORD'),
+            (os.environ.get('EMAIL_SEND_TO') or "").split(','),
+        )
+        if instance.properly_configured:
+            return instance
+
+        return None
+
+    @property
+    def properly_configured(self) -> bool:
+        return self.host and self.port and self.user and self.password and len(self.to) > 0
+
+    def send(self, subject: str = None, body: str = None, alert_type: str = 'INFO'):
+        # send_mail("Hello world!")
+        msg = MIMEText(body)
+        msg['Subject'] = f"[{alert_type}] {subject}"
+        msg['From'] = self.user
+        msg['To'] = ', '.join(self.to)
+
+        try:
+            logger.info("Connecting to %s port %s", self.host, self.port)
+            server = smtplib.SMTP_SSL(self.host, self.port)
+            server.ehlo()
+            server.login(self.user, self.password)
+            server.sendmail(self.user, self.to, msg.as_string())
+            logger.info('Email sent')
+        except Exception as ex:
+            logger.exception(ex)
+        finally:
+            server.close()

src/restic_compose_backup/backup_runner.py

@@ -51,9 +51,9 @@ def run(image: str = None, command: str = None, volumes: dict = None,
             fd.write('\n')
             logger.info(line)
 
-
     container.reload()
     logger.debug("Container ExitCode %s", container.attrs['State']['ExitCode'])
+    container.stop()
     container.remove()
 
     return container.attrs['State']['ExitCode']

src/restic_compose_backup/cli.py

@@ -0,0 +1,220 @@
+import argparse
+import logging
+
+from restic_compose_backup import (
+    alerts,
+    backup_runner,
+    log,
+    restic,
+)
+from restic_compose_backup.config import Config
+from restic_compose_backup.containers import RunningContainers
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """CLI entrypoint"""
+    args = parse_args()
+    config = Config()
+    log.setup(level=args.log_level or config.log_level)
+    containers = RunningContainers()
+
+    # Ensure log level is propagated to parent container if overridden
+    if args.log_level:
+        containers.this_container.set_config_env('LOG_LEVEL', args.log_level)
+
+    if args.action == 'status':
+        status(config, containers)
+
+    elif args.action == 'snapshots':
+        snapshots(config, containers)
+
+    elif args.action == 'backup':
+        backup(config, containers)
+
+    elif args.action == 'start-backup-process':
+        start_backup_process(config, containers)
+
+    elif args.action == 'cleanup':
+        cleanup(config, containers)
+
+    elif args.action == 'alert':
+        alert(config, containers)
+
+
+def status(config, containers):
+    """Outputs the backup config for the compose setup"""
+    logger.info("Status for compose project '%s'", containers.project_name)
+    logger.info("Repository: '%s'", config.repository)
+    logger.info("Backup currently running?: %s", containers.backup_process_running)
+    logger.info("%s Detected Config %s", "-" * 25, "-" * 25)
+
+    logger.info("Initializing repository (may fail if already initalized)")
+    restic.init_repo(config.repository)
+
+    backup_containers = containers.containers_for_backup()
+    for container in backup_containers:
+        logger.info('service: %s', container.service_name)
+
+        if container.volume_backup_enabled:
+            for mount in container.filter_mounts():
+                logger.info(' - volume: %s', mount.source)
+
+        if container.database_backup_enabled:
+            instance = container.instance
+            ping = instance.ping()
+            logger.info(' - %s (is_ready=%s)', instance.container_type, ping == 0)
+            if ping != 0:
+                logger.error("Database '%s' in service %s cannot be reached",
+                             instance.container_type, container.service_name)
+
+    if len(backup_containers) == 0:
+        logger.info("No containers in the project has 'restic-compose-backup.*' label")
+
+    logger.info("-" * 67)
+
+
+def backup(config, containers):
+    """Request a backup to start"""
+    # Make sure we don't spawn multiple backup processes
+    if containers.backup_process_running:
+        raise ValueError("Backup process already running")
+
+    # Map all volumes from the backup container into the backup process container
+    volumes = containers.this_container.volumes
+
+    # Map volumes from other containers we are backing up
+    mounts = containers.generate_backup_mounts('/volumes')
+    volumes.update(mounts)
+
+    try:
+        result = backup_runner.run(
+            image=containers.this_container.image,
+            command='restic-compose-backup start-backup-process',
+            volumes=volumes,
+            environment=containers.this_container.environment,
+            source_container_id=containers.this_container.id,
+            labels={
+                "restic-compose-backup.backup_process": 'True',
+                "com.docker.compose.project": containers.project_name,
+            },
+        )
+    except Exception as ex:
+        logger.exception(ex)
+        alerts.send(
+            subject="Exception during backup",
+            body=str(ex),
+            alert_type='ERROR',
+        )
+        return
+
+    logger.info('Backup container exit code: %s', result)
+
+    # Alert the user if something went wrong
+    if result != 0:
+        alerts.send(
+            subject="Backup process exited with non-zero code",
+            body=open('backup.log').read(),
+            alert_type='ERROR',
+        )
+
+
+def start_backup_process(config, containers):
+    """The actual backup process running inside the spawned container"""
+    if (not containers.backup_process_container
+       or containers.this_container == containers.backup_process_container is False):
+        logger.error(
+            "Cannot run backup process in this container. Use backup command instead. "
+            "This will spawn a new container with the necessary mounts."
+        )
+        return
+
+    status(config, containers)
+    errors = False
+
+    # Back up volumes
+    try:
+        logger.info('Backing up volumes')
+        vol_result = restic.backup_files(config.repository, source='/volumes')
+        logger.debug('Volume backup exit code: %s', vol_result)
+        if vol_result != 0:
+            logger.error('Backup command exited with non-zero code: %s', vol_result)
+            errors = True
+    except Exception as ex:
+        logger.exception(ex)
+        errors = True
+
+    # back up databases
+    for container in containers.containers_for_backup():
+        if container.database_backup_enabled:
+            try:
+                instance = container.instance
+                logger.info('Backing up %s in service %s', instance.container_type, instance.service_name)
+                result = instance.backup()
+                logger.debug('Exit code: %s', result)
+                if result != 0:
+                    logger.error('Backup command exited with non-zero code: %s', result)
+                    errors = True
+            except Exception as ex:
+                logger.exception(ex)
+                errors = True
+
+    if errors:
+        exit(1)
+
+    # Only run cleanup if backup was successful
+    result = cleanup(config, container)
+    logger.debug('cleanup exit code: %s', errors)
+    if result != 0:
+        exit(1)
+
+
+def cleanup(config, containers):
+    """Run forget / prune to minimize storage space"""
+    logger.info('Forget outdated snapshots')
+    forget_result = restic.forget(
+        config.repository,
+        config.keep_daily,
+        config.keep_weekly,
+        config.keep_monthly,
+        config.keep_yearly,
+    )
+    logger.info('Prune stale data freeing storage space')
+    prune_result = restic.prune(config.repository)
+    return forget_result and prune_result
+
+
+def snapshots(config, containers):
+    """Display restic snapshots"""
+    stdout, stderr = restic.snapshots(config.repository, last=True)
+    for line in stdout.decode().split('\n'):
+        print(line)
+
+
+def alert(config, containers):
+    """Test alerts"""
+    logger.info("Testing alerts")
+    alerts.send(
+        subject="{}: Test Alert".format(containers.project_name),
+        body="Test message",
+    )
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(prog='restic_compose_backup')
+    parser.add_argument(
+        'action',
+        choices=['status', 'snapshots', 'backup', 'start-backup-process', 'alert', 'cleanup'],
+    )
+    parser.add_argument(
+        '--log-level',
+        default=None,
+        choices=list(log.LOG_LEVELS.keys()),
+        help="Log level"
+    )
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    main()

src/restic_compose_backup/commands.py

@@ -1,15 +1,15 @@
 import logging
-from typing import List
+from typing import List, Tuple
 from subprocess import Popen, PIPE
 
 logger = logging.getLogger(__name__)
 
 
 def test():
-    return run_command(['ls', '/backup'])
+    return run(['ls', '/volumes'])
 
 
-def ping_mysql(host, port, username, password) -> int:
+def ping_mysql(host, port, username) -> int:
     """Check if the mysql is up and can be reached"""
     return run([
         'mysqladmin',
@@ -20,11 +20,10 @@ def ping_mysql(host, port, username, password) -> int:
         port,
         '--user',
         username,
-        f'--password={password}',
     ])
 
 
-def ping_mariadb(host, port, username, password) -> int:
+def ping_mariadb(host, port, username) -> int:
     """Check if the mariadb is up and can be reached"""
     return run([
         'mysqladmin',
@@ -35,7 +34,6 @@ def ping_mariadb(host, port, username, password) -> int:
         port,
         '--user',
         username,
-        f'--password={password}',
     ])
 
 
@@ -51,18 +49,25 @@ def ping_postgres(host, port, username, password) -> int:
 
 def run(cmd: List[str]) -> int:
     """Run a command with parameters"""
-    logger.info('cmd: %s', ' '.join(cmd))
+    logger.debug('cmd: %s', ' '.join(cmd))
     child = Popen(cmd, stdout=PIPE, stderr=PIPE)
     stdoutdata, stderrdata = child.communicate()
 
     if stdoutdata:
-        logger.info(stdoutdata.decode().strip())
+        logger.debug(stdoutdata.decode().strip())
-        logger.info('-' * 28)
+        logger.debug('-' * 28)
 
     if stderrdata:
-        logger.info('%s STDERR %s', '-' * 10, '-' * 10)
+        logger.error('%s STDERR %s', '-' * 10, '-' * 10)
-        logger.info(stderrdata.decode().strip())
+        logger.error(stderrdata.decode().strip())
-        logger.info('-' * 28)
+        logger.error('-' * 28)
 
-    logger.info("returncode %s", child.returncode)
+    logger.debug("returncode %s", child.returncode)
     return child.returncode
+
+
+def run_capture_std(cmd: List[str]) -> Tuple[str, str]:
+    """Run a command with parameters and return stdout, stderr"""
+    logger.debug('cmd: %s', ' '.join(cmd))
+    child = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    return child.communicate()

src/restic_compose_backup/config.py

@@ -0,0 +1,29 @@
+import os
+
+
+class Config:
+    """Bag for config values"""
+    def __init__(self, check=True):
+        # Mandatory values
+        self.repository = os.environ.get('RESTIC_REPOSITORY')
+        self.password = os.environ.get('RESTIC_REPOSITORY')
+        self.docker_base_url = os.environ.get('DOCKER_BASE_URL') or "unix://tmp/docker.sock"
+
+        # Log
+        self.log_level = os.environ.get('LOG_LEVEL')
+
+        # forget / keep
+        self.keep_daily = os.environ.get('KEEP_DAILY') or "7"
+        self.keep_weekly = os.environ.get('KEEP_WEEKLY') or "4"
+        self.keep_monthly = os.environ.get('KEEP_MONTHLY') or "12"
+        self.keep_yearly = os.environ.get('KEEP_YEARLY') or "3"
+
+        if check:
+            self.check()
+
+    def check(self):
+        if not self.repository:
+            raise ValueError("RESTIC_REPOSITORY env var not set")
+
+        if not self.password:
+            raise ValueError("RESTIC_REPOSITORY env var not set")

src/restic_compose_backup/containers.py

@@ -63,7 +63,7 @@ class Container:
     @property
     def environment(self) -> list:
         """All configured env vars for the container as a list"""
-        return self.get_config('Env', default=[])
+        return self.get_config('Env')
 
     def get_config_env(self, name) -> str:
         """Get a config environment variable by name"""
@@ -71,6 +71,17 @@ class Container:
         data = {i[0:i.find('=')]: i[i.find('=') + 1:] for i in self.environment}
         return data.get(name)
 
+    def set_config_env(self, name, value):
+        """Set an environment variable"""
+        env = self.environment
+        new_value = f'{name}={value}'
+        for i, entry in enumerate(env):
+            if f'{name}=' in entry:
+                env[i] = new_value
+                break
+        else:
+            env.append(new_value)
+
     @property
     def volumes(self) -> dict:
         """
@@ -160,6 +171,10 @@ class Container:
     def filter_mounts(self):
         """Get all mounts for this container matching include/exclude filters"""
         filtered = []
+
+        if not self.volume_backup_enabled:
+            return filtered
+
         if self._include:
             for mount in self._mounts:
                 for pattern in self._include:
@@ -182,7 +197,7 @@ class Container:
 
         return filtered
 
-    def volumes_for_backup(self, source_prefix='/backup', mode='ro'):
+    def volumes_for_backup(self, source_prefix='/volumes', mode='ro'):
         """Get volumes configured for backup"""
         mounts = self.filter_mounts()
         volumes = {}
@@ -319,6 +334,11 @@ class RunningContainers:
                 if container.id != self.this_container.id:
                     self.containers.append(container)
 
+    @property
+    def project_name(self) -> str:
+        """str: Name of the compose project"""
+        return self.this_container.project_name
+
     @property
     def backup_process_running(self) -> bool:
         """Is the backup process container running?"""
@@ -328,7 +348,7 @@ class RunningContainers:
         """Obtain all containers with backup enabled"""
         return [container for container in self.containers if container.backup_enabled]
 
-    def generate_backup_mounts(self, dest_prefix='/backup') -> dict:
+    def generate_backup_mounts(self, dest_prefix='/volumes') -> dict:
         """Generate mounts for backup for the entire compose setup"""
         mounts = {}
         for container in self.containers_for_backup():

src/restic_compose_backup/containers_db.py

@@ -22,11 +22,12 @@ class MariadbContainer(Container):
     def ping(self) -> bool:
         """Check the availability of the service"""
         creds = self.get_credentials()
-        return commands.ping_mysql(
+
+        with utils.environment('MYSQL_PWD', creds['password']):
+            return commands.ping_mariadb(
                 creds['host'],
                 creds['port'],
                 creds['username'],
-            creds['password'],
             )
 
     def dump_command(self) -> list:
@@ -37,15 +38,17 @@ class MariadbContainer(Container):
             f"--host={creds['host']}",
             f"--port={creds['port']}",
             f"--user={creds['username']}",
-            f"--password={creds['password']}",
             "--all-databases",
         ]
 
     def backup(self):
         config = Config()
+        creds = self.get_credentials()
+
+        with utils.environment('MYSQL_PWD', creds['password']):
             return restic.backup_from_stdin(
                 config.repository,
-            f'/backup/{self.service_name}/all_databases.sql',
+                f'/databases/{self.service_name}/all_databases.sql',
                 self.dump_command(),
             )
 
@@ -65,11 +68,12 @@ class MysqlContainer(Container):
     def ping(self) -> bool:
         """Check the availability of the service"""
         creds = self.get_credentials()
+
+        with utils.environment('MYSQL_PWD', creds['password']):
             return commands.ping_mysql(
                 creds['host'],
                 creds['port'],
                 creds['username'],
-            creds['password'],
             )
 
     def dump_command(self) -> list:
@@ -80,15 +84,17 @@ class MysqlContainer(Container):
             f"--host={creds['host']}",
             f"--port={creds['port']}",
             f"--user={creds['username']}",
-            f"--password={creds['password']}",
             "--all-databases",
         ]
 
     def backup(self):
         config = Config()
+        creds = self.get_credentials()
+
+        with utils.environment('MYSQL_PWD', creds['password']):
             return restic.backup_from_stdin(
                 config.repository,
-            f'/backup/{self.service_name}/all_databases.sql',
+                f'/databases/{self.service_name}/all_databases.sql',
                 self.dump_command(),
             )
 
@@ -135,6 +141,6 @@ class PostgresContainer(Container):
         with utils.environment('PGPASSWORD', creds['password']):
             return restic.backup_from_stdin(
                 config.repository,
-                f"/backup/{self.service_name}/{creds['database']}.sql",
+                f"/databases/{self.service_name}/{creds['database']}.sql",
                 self.dump_command(),
             )

src/restic_compose_backup/log.py

@@ -0,0 +1,28 @@
+import logging
+import os
+import sys
+
+logger = logging.getLogger('restic_compose_backup')
+HOSTNAME = os.environ['HOSTNAME']
+
+DEFAULT_LOG_LEVEL = logging.INFO
+LOG_LEVELS = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR,
+}
+
+
+def setup(level: str = 'warning'):
+    """Set up logging"""
+    level = level or ""
+    level = LOG_LEVELS.get(level.lower(), DEFAULT_LOG_LEVEL)
+    logger.setLevel(level)
+
+    ch = logging.StreamHandler(stream=sys.stdout)
+    ch.setLevel(level)
+    # ch.setFormatter(logging.Formatter(f'%(asctime)s - {HOSTNAME} - %(name)s - %(levelname)s - %(message)s'))
+    # ch.setFormatter(logging.Formatter(f'%(asctime)s - {HOSTNAME} - %(levelname)s - %(message)s'))
+    ch.setFormatter(logging.Formatter(f'%(asctime)s - %(levelname)s: %(message)s'))
+    logger.addHandler(ch)

src/restic_compose_backup/restic.py

@@ -2,7 +2,7 @@
 Restic commands
 """
 import logging
-from typing import List
+from typing import List, Tuple
 from subprocess import Popen, PIPE
 from restic_compose_backup import commands
 
@@ -19,7 +19,7 @@ def init_repo(repository: str):
     ]))
 
 
-def backup_files(repository: str, source='/backup'):
+def backup_files(repository: str, source='/volumes'):
     return commands.run(restic(repository, [
         "--verbose",
         "backup",
@@ -40,18 +40,50 @@ def backup_from_stdin(repository: str, filename: str, source_command: List[str])
     ])
 
     # pipe source command into dest command
+    # NOTE: Using the default buffer size: io.DEFAULT_BUFFER_SIZE = 8192
+    #       We might want to tweak that to speed up large dumps.
+    #       Actual tests tests must be done.
     source_process = Popen(source_command, stdout=PIPE)
-    dest_process = Popen(dest_command, stdin=source_process.stdout)
+    dest_process = Popen(dest_command, stdin=source_process.stdout, stdout=PIPE, stderr=PIPE)
-    dest_process.communicate()
+    stdout, stderr = dest_process.communicate()
+
+    if stdout:
+        for line in stdout.decode().split('\n'):
+            logger.debug(line)
+
+    if stderr:
+        for line in stderr.decode().split('\n'):
+            logger.error(line)
 
     # Ensure both processes exited with code 0
     source_exit, dest_exit = source_process.poll(), dest_process.poll()
     return 0 if (source_exit == 0 and dest_exit == 0) else 1
 
 
-def snapshots(repository: str):
+def snapshots(repository: str, last=True) -> Tuple[str, str]:
+    args = ["snapshots"]
+    if last:
+        args.append('--last')
+    return commands.run_capture_std(restic(repository, args))
+
+
+def forget(repository: str, daily: str, weekly: str, monthly: str, yearly: str):
     return commands.run(restic(repository, [
-        "snapshots",
+        'forget',
+        '--keep-daily',
+        daily,
+        '--keep-weekly',
+        weekly,
+        '--keep-monthly',
+        monthly,
+        '--keep-yearly',
+        yearly,
+    ]))
+
+
+def prune(repository: str):
+    return commands.run(restic(repository, [
+        'prune',
     ]))
 
 
@@ -65,8 +97,6 @@ def restic(repository: str, args: List[str]):
     """Generate restic command"""
     return [
         "restic",
-        "--cache-dir",
-        "/restic_cache",
         "-r",
         repository,
     ] + args

src/setup.py

@@ -3,12 +3,12 @@ from setuptools import setup, find_namespace_packages
 setup(
     name="restic-compose-backup",
     url="https://github.com/ZettaIO/restic-compose-backup",
-    version="0.2.0",
+    version="0.3.0",
     author="Einar Forselv",
     author_email="eforselv@gmail.com",
     packages=find_namespace_packages(include=['restic_compose_backup']),
     install_requires=[
-        'docker==3.7.2',
+        'docker==4.1.*',
     ],
     entry_points={'console_scripts': [
         'restic-compose-backup = restic_compose_backup.cli:main',

src/tests/requirements.txt

@@ -1 +1,2 @@
 pytest==4.3.1
+tox

src/tests/tests.py

@@ -59,6 +59,7 @@ class ResticBackupTests(unittest.TestCase):
             {
                 'service': 'web',
                 'labels': {
+                    'restic-compose-backup.volumes': True,
                     'test': 'test',
                 },
                 'mounts': [{
@@ -111,7 +112,7 @@ class ResticBackupTests(unittest.TestCase):
         with mock.patch(list_containers_func, fixtures.containers(containers=containers)):
             cnt = RunningContainers()
             self.assertTrue(len(cnt.containers_for_backup()) == 2)
-            self.assertEqual(cnt.generate_backup_mounts(), {'test': {'bind': '/backup/web/test', 'mode': 'ro'}})
+            self.assertEqual(cnt.generate_backup_mounts(), {'test': {'bind': '/volumes/web/test', 'mode': 'ro'}})
 
     def test_include(self):
         containers = self.createContainers()
@@ -119,7 +120,8 @@ class ResticBackupTests(unittest.TestCase):
             {
                 'service': 'web',
                 'labels': {
-                    'restic-compose-backup.include': 'media',
+                    'restic-compose-backup.volumes': True,
+                    'restic-compose-backup.volumes.include': 'media',
                 },
                 'mounts': [
                     {
@@ -142,6 +144,7 @@ class ResticBackupTests(unittest.TestCase):
         self.assertNotEqual(web_service, None, msg="Web service not found")
 
         mounts = web_service.filter_mounts()
+        print(mounts)
         self.assertEqual(len(mounts), 1)
         self.assertEqual(mounts[0].source, '/srv/files/media')
 
@@ -151,7 +154,8 @@ class ResticBackupTests(unittest.TestCase):
             {
                 'service': 'web',
                 'labels': {
-                    'restic-compose-backup.exclude': 'stuff',
+                    'restic-compose-backup.volumes': True,
+                    'restic-compose-backup.volumes.exclude': 'stuff',
                 },
                 'mounts': [
                     {

tox.ini

@@ -0,0 +1,56 @@
+# Ensure that this file do not contain non-ascii characters
+# as flake8 can fail to parse the file on OS X and Windows
+
+[tox]
+skipsdist = True
+setupdir={toxinidir}/src
+envlist =
+    py37
+    pep8
+
+[testenv]
+usedevelop = True
+basepython =
+    py37: python3.7
+
+deps = 
+    -r{toxinidir}/tests/requirements.txt
+commands =
+    ; coverage run --source=restic_compose_backup -m pytest tests/
+    ; coverage report
+    pytest
+
+[testenv:pep8]
+usedevelop = false
+deps = flake8
+basepython = python3.7
+commands = flake8
+
+[pytest]
+norecursedirs = tests/* .venv/* .tox/* build/ docs/
+
+[flake8]
+# H405: multi line docstring summary not separated with an empty line
+# D100: Missing docstring in public module
+# D101: Missing docstring in public class
+# D102: Missing docstring in public method
+# D103: Missing docstring in public function
+# D104: Missing docstring in public package
+# D105: Missing docstring in magic method
+# D200: One-line docstring should fit on one line with quotes
+# D202: No blank lines allowed after function docstring
+# D203: 1 blank required before class docstring.
+# D204: 1 blank required after class docstring
+# D205: Blank line required between one-line summary and description.
+# D207: Docstring is under-indented
+# D208: Docstring is over-indented
+# D211: No blank lines allowed before class docstring
+# D301: Use r""" if any backslashes in a docstring
+# D400: First line should end with a period.
+# D401: First line should be in imperative mood.
+# *** E302 expected 2 blank lines, found 1
+# *** W503 line break before binary operator
+ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503
+show-source = True
+max-line-length = 120
+exclude = .tox,.venv*,tests,build,conf.py