status command should display repository

.gitignore

@@ -24,3 +24,6 @@ alerts.env
 # docs
 build/
 docs/_build
+
+# tests
+.tox

README.md

@@ -38,8 +38,6 @@ Automatically detects and backs up volumes, mysql, mariadb and postgres database
 docker pull zettaio/restic-compose-backup
 ```
 
-.. or clone this repo and build it.
-
 ## Configuration
 
 Required env variables for restic:
@@ -81,7 +79,7 @@ version: '3'
 services:
   # The backup service
   backup:
-    build: restic-compose-backup
+    image: zettaio/restic-compose-backup
     environment:
       - RESTIC_REPOSITORY=<whatever restic supports>
       - RESTIC_PASSWORD=hopefullyasecturepw
@@ -178,22 +176,21 @@ path `/databases/<service_name>/dump.sql` or similar.
       restic-compose-backup.postgres: true
 ```
 
-
 ## Running Tests
 
 ```bash
-python setup.py develop
-pip install -r tests/requirements.txt
-pytest tests
+pip install -e src/
+pip install -r src/tests/requirements.txt
+tox
 ```
 
 ## Building Docs
 
 ```bash
 pip install -r docs/requirements.txt
-python setup.py build_sphinx
+python src/setup.py build_sphinx
 ```
 
 ## Contributing
 
-Contributions are welcome regardless of experience level. Don't hesitate submitting issues, opening partial or completed pull requests.
+Contributions are welcome regardless of experience level. Don't hesitate submitting issues, opening partial or completed pull requests.

docker-compose.yaml

@@ -1,9 +1,10 @@
 version: '3'
 services:
   backup:
-    build: .
+    build: ./src
     env_file:
       - restic_compose_backup.env
+      - alerts.env
     volumes:
       # Map in docker socket
       - /var/run/docker.sock:/tmp/docker.sock:ro
@@ -11,7 +12,7 @@ services:
       - ./restic_data:/restic_data
       - ./restic_cache:/cache
       # Map in project source in dev
-      - .:/restic-compose-backup
+      - ./src:/restic-compose-backup
   web:
     image: nginx
     labels:

docs/conf.py

@@ -22,7 +22,7 @@ copyright = '2019, Zetta.IO Technology AS'
 author = 'Zetta.IO Technology AS'
 
 # The full version, including alpha/beta/rc tags
-release = '0.2.0'
+release = '0.3.0'
 
 
 # -- General configuration ---------------------------------------------------

extras/release.md

@@ -1,13 +1,20 @@
 # Making a release
 
-- Update version in setup.py
+- Update version in `setup.py`
+- Update version in `docs/conf.py`
 - Build and tag image
 - push: `docker push zettaio/restic-compose-backup:<version>`
 - Ensure RTD has new docs published
 
 ## Example
 
+When releasing a bugfix version we need to update the
+main image as well.
+
 ```bash
-docker build . --tag zettaio/restic-compose-backup:0.2.0
-docker push zettaio/restic-compose-backup:0.2.0
+docker build . --tag zettaio/restic-compose-backup:0.3
+docker build . --tag zettaio/restic-compose-backup:0.3.1
+
+docker push zettaio/restic-compose-backup:0.3
+docker push zettaio/restic-compose-backup:0.3.1
 ```

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-testpaths = tests
+testpaths = src/tests
 python_files=test*.py
 addopts = -v --verbose

restic_compose_backup/mail.py

@@ -1,37 +0,0 @@
-"""
-"""
-import smtplib
-from email.mime.text import MIMEText
-
-EMAIL_HOST = "smtp.gmail.com"
-EMAIL_PORT = 465
-EMAIL_HOST_USER = ""
-EMAIL_HOST_PASSWORD = ""
-EMAIL_SEND_TO = ['']
-
-
-def main():
-    send_mail("Hello world!")
-
-
-def send_mail(text):
-    msg = MIMEText(text)
-    msg['Subject'] = "Message from restic-compose-backup"
-    msg['From'] = EMAIL_HOST_USER
-    msg['To'] = ', '.join(EMAIL_SEND_TO)
-
-    try:
-        print("Connecting to {} port {}".format(EMAIL_HOST, EMAIL_PORT))
-        server = smtplib.SMTP_SSL(EMAIL_HOST, EMAIL_PORT)
-        server.ehlo()
-        server.login(EMAIL_HOST_USER, EMAIL_HOST_PASSWORD)
-        server.sendmail(EMAIL_HOST_USER, EMAIL_SEND_TO, msg.as_string())
-        print('Email Sent')
-    except Exception as e:
-        print(e)
-    finally:
-        server.close()
-
-
-if __name__ == '__main__':
-    main()

src/restic_compose_backup/alerts/__init__.py

@@ -2,7 +2,6 @@ import logging
 
 from restic_compose_backup.alerts.smtp import SMTPAlert
 from restic_compose_backup.alerts.discord import DiscordWebhookAlert
-from restic_compose_backup.config import Config
 
 logger = logging.getLogger(__name__)
 
@@ -24,6 +23,7 @@ def send(subject: str = None, body: str = None, alert_type: str = 'INFO'):
             )
         except Exception as ex:
             logger.error("Exception raised when sending alert [%s]: %s", instance.name, ex)
+            logger.exception(ex)
 
     if len(alert_classes) == 0:
         logger.info("No alerts configured")
@@ -36,7 +36,7 @@ def configured_alert_types():
 
     for cls in BACKENDS:
         instance = cls.create_from_env()
-        logger.debug("Alert backend '%s' configured: %s", cls.name, instance != None)
+        logger.debug("Alert backend '%s' configured: %s", cls.name, instance is not None)
         if instance:
             entires.append(instance)
 

src/restic_compose_backup/alerts/discord.py

@@ -1,6 +1,5 @@
 import os
 import logging
-from urllib.parse import urlparse
 
 import requests
 from restic_compose_backup.alerts.base import BaseAlert
@@ -41,6 +40,6 @@ class DiscordWebhookAlert(BaseAlert):
         }
         response = requests.post(self.url, params={'wait': True}, json=data)
         if response.status_code not in self.success_codes:
-            log.error("Discord webhook failed: %s: %s", response.status_code, response.content)
+            logger.error("Discord webhook failed: %s: %s", response.status_code, response.content)
         else:
             logger.info('Discord webhook successful')

src/restic_compose_backup/alerts/smtp.py

@@ -51,6 +51,6 @@ class SMTPAlert(BaseAlert):
             server.sendmail(self.user, self.to, msg.as_string())
             logger.info('Email sent')
         except Exception as ex:
-            logger.error(ex)
+            logger.exception(ex)
         finally:
             server.close()

src/restic_compose_backup/backup_runner.py

@@ -51,9 +51,9 @@ def run(image: str = None, command: str = None, volumes: dict = None,
             fd.write('\n')
             logger.info(line)
 
-
     container.reload()
     logger.debug("Container ExitCode %s", container.attrs['State']['ExitCode'])
+    container.stop()
     container.remove()
 
     return container.attrs['State']['ExitCode']

src/restic_compose_backup/cli.py

@@ -1,5 +1,4 @@
 import argparse
-import pprint
 import logging
 
 from restic_compose_backup import (
@@ -47,9 +46,13 @@ def main():
 def status(config, containers):
     """Outputs the backup config for the compose setup"""
     logger.info("Status for compose project '%s'", containers.project_name)
+    logger.info("Repository: '%s'", config.repository)
     logger.info("Backup currently running?: %s", containers.backup_process_running)
     logger.info("%s Detected Config %s", "-" * 25, "-" * 25)
 
+    logger.info("Initializing repository (may fail if already initalized)")
+    restic.init_repo(config.repository)
+
     backup_containers = containers.containers_for_backup()
     for container in backup_containers:
         logger.info('service: %s', container.service_name)
@@ -63,10 +66,11 @@ def status(config, containers):
             ping = instance.ping()
             logger.info(' - %s (is_ready=%s)', instance.container_type, ping == 0)
             if ping != 0:
-                logger.error("Database '%s' in service %s cannot be reached", instance.container_type, container.service_name)
+                logger.error("Database '%s' in service %s cannot be reached",
+                             instance.container_type, container.service_name)
 
     if len(backup_containers) == 0:
-        logger.info("No containers in the project has 'restic-compose-backup.enabled' label")
+        logger.info("No containers in the project has 'restic-compose-backup.*' label")
 
     logger.info("-" * 67)
 
@@ -77,11 +81,6 @@ def backup(config, containers):
     if containers.backup_process_running:
         raise ValueError("Backup process already running")
 
-    logger.info("Initializing repository (may fail if already initalized)")
-
-    # TODO: Errors when repo already exists
-    restic.init_repo(config.repository)
-
     # Map all volumes from the backup container into the backup process container
     volumes = containers.this_container.volumes
 
@@ -89,17 +88,27 @@ def backup(config, containers):
     mounts = containers.generate_backup_mounts('/volumes')
     volumes.update(mounts)
 
-    result = backup_runner.run(
-        image=containers.this_container.image,
-        command='restic-compose-backup start-backup-process',
-        volumes=volumes,
-        environment=containers.this_container.environment,
-        source_container_id=containers.this_container.id,
-        labels={
-            "restic-compose-backup.backup_process": 'True',
-            "com.docker.compose.project": containers.project_name,
-        },
-    )
+    try:
+        result = backup_runner.run(
+            image=containers.this_container.image,
+            command='restic-compose-backup start-backup-process',
+            volumes=volumes,
+            environment=containers.this_container.environment,
+            source_container_id=containers.this_container.id,
+            labels={
+                "restic-compose-backup.backup_process": 'True',
+                "com.docker.compose.project": containers.project_name,
+            },
+        )
+    except Exception as ex:
+        logger.exception(ex)
+        alerts.send(
+            subject="Exception during backup",
+            body=str(ex),
+            alert_type='ERROR',
+        )
+        return
+
     logger.info('Backup container exit code: %s', result)
 
     # Alert the user if something went wrong
@@ -133,7 +142,7 @@ def start_backup_process(config, containers):
             logger.error('Backup command exited with non-zero code: %s', vol_result)
             errors = True
     except Exception as ex:
-        logger.error(ex)
+        logger.exception(ex)
         errors = True
 
     # back up databases
@@ -148,7 +157,7 @@ def start_backup_process(config, containers):
                     logger.error('Backup command exited with non-zero code: %s', result)
                     errors = True
             except Exception as ex:
-                logger.error(ex)
+                logger.exception(ex)
                 errors = True
 
     if errors:
@@ -173,7 +182,8 @@ def cleanup(config, containers):
     )
     logger.info('Prune stale data freeing storage space')
     prune_result = restic.prune(config.repository)
-    return forget_result == 0 and prune_result == 0
+    return forget_result and prune_result
+
 
 def snapshots(config, containers):
     """Display restic snapshots"""

src/restic_compose_backup/commands.py

@@ -6,7 +6,7 @@ logger = logging.getLogger(__name__)
 
 
 def test():
-    return run_command(['ls', '/volumes'])
+    return run(['ls', '/volumes'])
 
 
 def ping_mysql(host, port, username) -> int:
@@ -23,7 +23,7 @@ def ping_mysql(host, port, username) -> int:
     ])
 
 
-def ping_mariadb(host, port, username):  #, password) -> int:
+def ping_mariadb(host, port, username) -> int:
     """Check if the mariadb is up and can be reached"""
     return run([
         'mysqladmin',
@@ -43,7 +43,7 @@ def ping_postgres(host, port, username, password) -> int:
         "pg_isready",
         f"--host={host}",
         f"--port={port}",
-        f"--username={username}", 
+        f"--username={username}",
     ])
 
 

src/restic_compose_backup/containers.py

@@ -68,7 +68,7 @@ class Container:
     def get_config_env(self, name) -> str:
         """Get a config environment variable by name"""
         # convert to dict and fetch env var by name
-        data = {i[0:i.find('=')]: i[i.find('=')+1:] for i in self.environment}
+        data = {i[0:i.find('=')]: i[i.find('=') + 1:] for i in self.environment}
         return data.get(name)
 
     def set_config_env(self, name, value):
@@ -169,7 +169,7 @@ class Container:
         return self._labels.get(name, None)
 
     def filter_mounts(self):
-        """Get all mounts for this container matching include/exclude filters"""            
+        """Get all mounts for this container matching include/exclude filters"""
         filtered = []
 
         if not self.volume_backup_enabled:

src/restic_compose_backup/log.py

@@ -13,6 +13,7 @@ LOG_LEVELS = {
     'error': logging.ERROR,
 }
 
+
 def setup(level: str = 'warning'):
     """Set up logging"""
     level = level or ""

src/restic_compose_backup/restic.py

@@ -63,12 +63,12 @@ def backup_from_stdin(repository: str, filename: str, source_command: List[str])
 def snapshots(repository: str, last=True) -> Tuple[str, str]:
     args = ["snapshots"]
     if last:
-        args.append('--last')        
+        args.append('--last')
     return commands.run_capture_std(restic(repository, args))
 
 
 def forget(repository: str, daily: str, weekly: str, monthly: str, yearly: str):
-    return restic(repository, [
+    return commands.run(restic(repository, [
         'forget',
         '--keep-daily',
         daily,
@@ -78,13 +78,13 @@ def forget(repository: str, daily: str, weekly: str, monthly: str, yearly: str):
         monthly,
         '--keep-yearly',
         yearly,
-    ])
+    ]))
 
 
 def prune(repository: str):
-    return restic(repository, [
+    return commands.run(restic(repository, [
         'prune',
-    ])
+    ]))
 
 
 def check(repository: str):

src/setup.py

@@ -8,7 +8,7 @@ setup(
     author_email="eforselv@gmail.com",
     packages=find_namespace_packages(include=['restic_compose_backup']),
     install_requires=[
-        'docker==3.7.2',
+        'docker==4.1.*',
     ],
     entry_points={'console_scripts': [
         'restic-compose-backup = restic_compose_backup.cli:main',

src/tests/requirements.txt

@@ -1 +1,2 @@
 pytest==4.3.1
+tox

tox.ini

@@ -0,0 +1,56 @@
+# Ensure that this file do not contain non-ascii characters
+# as flake8 can fail to parse the file on OS X and Windows
+
+[tox]
+skipsdist = True
+setupdir={toxinidir}/src
+envlist =
+    py37
+    pep8
+
+[testenv]
+usedevelop = True
+basepython =
+    py37: python3.7
+
+deps = 
+    -r{toxinidir}/tests/requirements.txt
+commands =
+    ; coverage run --source=restic_compose_backup -m pytest tests/
+    ; coverage report
+    pytest
+
+[testenv:pep8]
+usedevelop = false
+deps = flake8
+basepython = python3.7
+commands = flake8
+
+[pytest]
+norecursedirs = tests/* .venv/* .tox/* build/ docs/
+
+[flake8]
+# H405: multi line docstring summary not separated with an empty line
+# D100: Missing docstring in public module
+# D101: Missing docstring in public class
+# D102: Missing docstring in public method
+# D103: Missing docstring in public function
+# D104: Missing docstring in public package
+# D105: Missing docstring in magic method
+# D200: One-line docstring should fit on one line with quotes
+# D202: No blank lines allowed after function docstring
+# D203: 1 blank required before class docstring.
+# D204: 1 blank required after class docstring
+# D205: Blank line required between one-line summary and description.
+# D207: Docstring is under-indented
+# D208: Docstring is over-indented
+# D211: No blank lines allowed before class docstring
+# D301: Use r""" if any backslashes in a docstring
+# D400: First line should end with a period.
+# D401: First line should be in imperative mood.
+# *** E302 expected 2 blank lines, found 1
+# *** W503 line break before binary operator
+ignore = H405,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D211,D301,D400,D401,W503
+show-source = True
+max-line-length = 120
+exclude = .tox,.venv*,tests,build,conf.py