pep8

README.md

@@ -167,4 +167,9 @@ python src/setup.py build_sphinx
 Contributions are welcome regardless of experience level. Don't hesitate submitting issues, opening partial or completed pull requests.
 
 [restic]: https://restic.net/
-[documentation]: https://restic-compose-backup.readthedocs.io
+[documentation]: https://restic-compose-backup.readthedocs.io
+
+---
+This project is sponsored by [zetta.io](https://www.zetta.io)
+
+[![Zetta.IO](https://raw.githubusercontent.com/ZettaIO/restic-compose-backup/master/.github/logo.png)](https://www.zetta.io)

docker-compose.yaml

@@ -5,6 +5,9 @@ services:
     env_file:
       - restic_compose_backup.env
       - alerts.env
+    labels:
+      restic-compose-backup.volumes: true
+      restic-compose-backup.volumes.include: 'src'
     volumes:
       # Map in docker socket
       - /var/run/docker.sock:/tmp/docker.sock:ro
@@ -20,7 +23,7 @@ services:
       restic-compose-backup.volumes: true
       restic-compose-backup.volumes.include: "/tests"
     volumes:
-      - ./tests:/srv/tests
+      - ./src/tests:/srv/tests
       - ./.vscode:/srv/code
     environment:
       - SOME_VALUE=test

docs/conf.py

@@ -22,7 +22,7 @@ copyright = '2019, Zetta.IO Technology AS'
 author = 'Zetta.IO Technology AS'
 
 # The full version, including alpha/beta/rc tags
-release = '0.4.0'
+release = '0.4.2'
 
 # -- General configuration ---------------------------------------------------
 

docs/guide/advanced.rst

@@ -0,0 +1,30 @@
+Advanced
+--------
+
+Currently work in progress. These are only notes :D
+
+Temp Notes
+~~~~~~~~~~
+
+* Quick setup guide from start to end
+* we group snapshots by path when forgetting
+* explain rcb commands
+* examples of using restic directly
+* Explain what happens during backup process
+* Explain the backup process container
+* cache directory
+* Not displaying passwords in logs
+
+Inner workings
+~~~~~~~~~~~~~~
+
+* Each service in the compose setup is configured with a label
+  to enable backup of volumes or databases
+* When backup starts a new instance of the container is created
+  mapping in all the needed volumes. It will copy networks etc
+  to ensure databases can be reached
+* Volumes are mounted to `/volumes/<service_name>/<path>`
+  in the backup process container. `/volumes` is pushed into restic
+* Databases are backed up from stdin / dumps into restic using path
+  `/databases/<service_name>/dump.sql`
+* Cron triggers backup at 2AM every day

docs/guide/configuration.rst

@@ -0,0 +1,420 @@
+Configuration
+=============
+
+Environment Variables
+---------------------
+
+RESTIC_REPOSITORY
+~~~~~~~~~~~~~~~~~
+
+Sets the restic repository path.
+
+This is a standard environment variable
+the ``restic`` command will read making it simple for
+us to enter the container and use the restic command directly.
+
+More about this value and supported backends:
+https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html
+
+RESTIC_PASSWORD
+~~~~~~~~~~~~~~~
+
+Sets the password is used to encrypt/decrypt data.
+Losing this password will make recovery impossible.
+
+This is a standard environment variable the ``restic``
+command will read making it simple for us to enter the
+container running the command directly.
+
+RESTIC_KEEP_DAILY
+~~~~~~~~~~~~~~~~~
+
+**Default value**: ``7``
+
+How many daily snapshots (grouped by path) back in time we
+want to keep. This is passed to restic in the
+``forget --keep-daily`` option.
+
+RESTIC_KEEP_WEEKLY
+~~~~~~~~~~~~~~~~~~
+
+**Default value**: ``4``
+
+How many weeks back we should keep at least one snapshot
+(grouped by path). This is passed to restic in the
+``forget --keep-weekly`` option.
+
+RESTIC_KEEP_MONTHLY
+~~~~~~~~~~~~~~~~~~~
+
+**Default value**: ``12``
+
+How many months back we should keep at least on snapshot
+(grouped by path). This is passed to restic in the
+``forget --keep-monthly`` option.
+
+The schedule parameters only accepts numeric values
+and is validated when the container starts. Providing
+values cron does not understand will stall all backup.
+
+RESTIC_KEEP_YEARLY
+~~~~~~~~~~~~~~~~~~
+
+**Default value**: ``3``
+
+How many years back we should keep at least one snapshot
+(grouped by path). This is passed to restic in the
+``forget --keep-yearly`` option.
+
+CRON_SCHEDULE
+~~~~~~~~~~~~~
+
+**Default value**: ``0 2 * * *`` (daily at 02:00)
+
+The cron schedule parameters. The crontab is generated when the
+container starts from the ``CRON_SCHEDULE`` and ``CRON_COMMAND``
+env variables.
+
+.. code::
+
+    ┌───────────── minute (0 - 59)
+    │ ┌───────────── hour (0 - 23)
+    │ │ ┌───────────── day of the month (1 - 31)
+    │ │ │ ┌───────────── month (1 - 12)
+    │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday)
+    │ │ │ │ │
+    │ │ │ │ │
+    │ │ │ │ │
+    * * * * * command to execute
+
+CRON_COMMAND
+~~~~~~~~~~~~
+
+**Default value**: ``source /env.sh && rcb backup > /proc/1/fd/1``
+
+The command executed in the crontab. A single line is generated when
+the container starts from the ``CRON_SCHEDULE`` and ``CRON_COMMAND``
+environment variables.
+
+The default command sources a dump of all env vars, runs the
+backup command and directs output to pid 1 so it appears in
+docker logs.
+
+By default the crontab will look like this::
+
+    0 2 * * * source /env.sh && rcb backup > /proc/1/fd/1
+
+LOG_LEVEL
+~~~~~~~~~
+
+**Default value**: ``info``
+
+Log level for the ``rcb`` command. Valid values are
+``debug``, ``info``, ``warning``, ``error``.
+
+EMAIL_HOST
+~~~~~~~~~~
+
+The email host to use.
+
+Alerts can be tested using the ``rcb alerts`` command.
+This will send a test message to all configured alert
+backends.
+
+EMAIL_PORT
+~~~~~~~~~~
+
+The port to connect to
+
+Alerts can be tested using the ``rcb alerts`` command.
+This will send a test message to all configured alert
+backends.
+
+EMAIL_HOST_USER
+~~~~~~~~~~~~~~~
+
+The user of the sender account
+
+Alerts can be tested using the ``rcb alerts`` command.
+This will send a test message to all configured alert
+backends.
+
+EMAIL_HOST_PASSWORD
+~~~~~~~~~~~~~~~~~~~
+
+The password for the sender account
+
+Alerts can be tested using the ``rcb alerts`` command.
+This will send a test message to all configured alert
+backends.
+
+EMAIL_SEND_TO
+~~~~~~~~~~~~~
+
+The email address to send alerts
+
+Alerts can be tested using the ``rcb alerts`` command.
+This will send a test message to all configured alert
+backends.
+
+DISCORD_WEBHOOK
+~~~~~~~~~~~~~~~
+
+The discord webhook url. And administrator can quickly set this up
+by going to server settings in the discord client and create
+a webhook that will post embedded messages to a specific channel.
+
+The url usually looks like this: ``https://discordapp.com/api/webhooks/...```
+
+DOCKER_HOST
+~~~~~~~~~~~
+
+**Default value**: ``unix://tmp/docker.sock``
+
+The socket or host of the docker service.
+
+DOCKER_TLS_VERIFY
+~~~~~~~~~~~~~~~~~
+
+If defined verify the host against a CA certificate.
+Path to certs is defined in ``DOCKER_CERT_PATH``
+and can be copied or mapped into this backup container.
+
+DOCKER_CERT_PATH
+~~~~~~~~~~~~~~~~
+
+A path to a directory containing TLS certificates to use when
+connecting to the Docker host. Combined with ``DOCKER_TLS_VERIFY``
+this can be used to talk to docker through TLS in cases
+were we cannot map in the docker socket.
+
+Compose Labels
+--------------
+
+What is backed up is controlled by simple labels in the compose
+yaml file. At any point we can verify this configuration
+by running the ``rcb status`` command.
+
+.. code:
+
+    $ docker-compose run --rm backup rcb status
+    INFO: Status for compose project 'myproject'
+    INFO: Repository: '<restic repository>'
+    INFO: Backup currently running?: False
+    INFO: --------------- Detected Config ---------------
+    INFO: service: mysql
+    INFO:  - mysql (is_ready=True)
+    INFO: service: mariadb
+    INFO:  - mariadb (is_ready=True)
+    INFO: service: postgres
+    INFO:  - postgres (is_ready=True)
+    INFO: service: web
+    INFO:  - volume: media
+    INFO:  - volume: /srv/files
+
+Here we can see what volumes and databases are detected for backup.
+
+Volumes
+~~~~~~~
+
+To enable volume backup for a service we simply add the
+`restic-compose-backup.volumes: true` label. The value
+must be ``true``.
+
+Example:
+
+.. code:: yaml
+
+    myservice:
+      image: some_image
+      labels:
+        restic-compose-backup.volumes: true
+      volumes:
+        - uploaded_media:/srv/media
+        - uploaded_files:/srv/files
+        - /srv/data:/srv/data
+
+    volumes:
+      media:
+      files:
+
+This will back up the three volumes mounted to this service.
+Their path in restic will be:
+
+- /volumes/myservice/srv/media
+- /volumes/myservice/srv/files
+- /volumes/myservice/srv/data
+
+A simple `include` and `exclude` filter for what volumes
+should be backed up is also available. Note that this
+includes or excludes entire volumes and are not include/exclude
+patterns for files in the volumes.
+
+.. note:: The ``exclude`` and ``include`` filtering is applied on
+          the source path, not the destination.
+
+Include example including two volumes only:
+
+.. code:: yaml
+
+    myservice:
+      image: some_image
+      labels:
+        restic-compose-backup.volumes: true
+        restic-compose-backup.volumes.include: "uploaded_media,uploaded_files"
+      volumes:
+        - uploaded_media:/srv/media
+        - uploaded_files:/srv/files
+        - /srv/data:/srv/data
+
+    volumes:
+      media:
+      files:
+
+Exclude example achieving the same result as the example above.
+
+.. code:: yaml
+
+    example:
+      image: some_image
+      labels:
+        restic-compose-backup.volumes: true
+        restic-compose-backup.volumes.exclude: "data"
+      volumes:
+        # Excluded by filter
+        - media:/srv/media
+        # Backed up
+        - files:/srv/files
+        - /srv/data:/srv/data
+
+    volumes:
+      media:
+      files:
+
+The ``exclude`` and ``include`` tag can be used together
+in more complex situations.
+
+mariadb
+~~~~~~~
+
+To enable backup of mariadb simply add the
+``restic-compose-backup.mariadb: true`` label.
+
+Credentials are fetched from the following environment
+variables in the mariadb service. This is the standard
+when using the official mariadb_ image.
+
+.. code::
+
+    MYSQL_USER
+    MYSQL_PASSWORD
+
+Backups are done by dumping all databases directly into
+restic through stdin using ``mysqldump``. It will appear
+in restic as a separate snapshot with path
+``/databases/<service_name>/all_databases.sql``.
+
+.. warning: This will only back up the databases the
+            ``MYSQL_USER` has access to. If you have multiple
+            databases this must be taken into consideration.
+
+Example:
+
+.. code:: yaml
+
+    mariadb:
+      image: mariadb:10
+      labels:
+        restic-compose-backup.mariadb: true
+      env_file:
+        mariadb-credentials.env
+      volumes:
+        - mariadb:/var/lib/mysql
+
+    volumes:
+      mariadb:
+
+mysql
+~~~~~
+
+To enable backup of mysql simply add the
+``restic-compose-backup.mysql: true`` label.
+
+Credentials are fetched from the following environment
+variables in the mysql service. This is the standard
+when using the official mysql_ image.
+
+.. code::
+
+    MYSQL_USER
+    MYSQL_PASSWORD
+
+Backups are done by dumping all databases directly into
+restic through stdin using ``mysqldump``. It will appear
+in restic as a separate snapshot with path
+``/databases/<service_name>/all_databases.sql``.
+
+.. warning: This will only back up the databases the
+            ``MYSQL_USER` has access to. If you have multiple
+            databases this must be taken into consideration.
+
+Example:
+
+.. code:: yaml
+
+    mysql:
+      image: mysql:5
+      labels:
+        restic-compose-backup.mysql: true
+      env_file:
+        mysql-credentials.env
+      volumes:
+        - mysql:/var/lib/mysql
+
+volumes:
+  mysql:
+
+postgres
+~~~~~~~~
+
+To enable backup of mysql simply add the
+``restic-compose-backup.postgres: true`` label.
+
+Credentials are fetched from the following environment
+variables in the postgres service. This is the standard
+when using the official postgres_ image.
+
+.. code::
+
+    POSTGRES_USER
+    POSTGRES_PASSWORD
+    POSTGRES_DB
+
+Backups are done by dumping the ``POSTGRES_DB`` directly into
+restic through stdin using ``pg_dump``. It will appear
+in restic as a separate snapshot with path
+``/databases/<service_name>/<POSTGRES_DB>.sql``.
+
+.. warning:: Currently only the ``POSTGRES_DB`` database
+             is dumped.
+
+Example:
+
+.. code:: yaml
+
+    postgres:
+      image: postgres:11
+      labels:
+        # Enables backup of this database
+        restic-compose-backup.postgres: true
+      env_file:
+        postgres-credentials.env
+      volumes:
+        - pgdata:/var/lib/postgresql/data
+
+    volumes:
+      pgdata:
+
+.. _mariadb: https://hub.docker.com/_/mariadb
+.. _mysql: https://hub.docker.com/_/mysql
+.. _postgres: https://hub.docker.com/_/postgres

docs/guide/install.rst

@@ -0,0 +1,53 @@
+Introduction
+============
+
+
+Install
+-------
+
+restic-compose-backup is available at docker `docker hub`_.
+
+.. code::
+
+    docker pull restic-compose-backup
+
+Optionally it can be built from source using the github_ repository.
+
+.. code:: bash
+
+    git clone https://github.com/ZettaIO/restic-compose-backup.git
+    cd restic-compose-backup
+    # Build and tag the image locally
+    docker build src/ --tag restic-compose-backup
+
+Bug reports and issues
+----------------------
+
+Please report bugs an issues on github_
+
+Development setup
+-----------------
+
+Getting started with local development is fairly simple.
+The github_ repository contains a simple ``docker-compose.yaml``
+
+.. code:: bash
+
+    docker-compose up -d
+    # Enter the container in sh
+    docker-compose run --rm backup sh
+
+The dev compose setup maps in the source from the host
+and the spawned backup container will inherit all
+the volumes from the backup service ensuring code changes
+propagates during development.
+
+Set up a local venv and install the package in development mode::
+
+    python -m venv .venv
+    . .venv/bin/activate
+    pip install -e ./src
+
+
+.. _docker hub: https://hub.docker.com/r/zettaio/restic-compose-backup
+.. _github: https://github.com/ZettaIO/restic-compose-backup

docs/guide/rcb.rst

@@ -0,0 +1,199 @@
+
+The `rcb` command
+-----------------
+
+The ``rcb`` command is is basically what this entire project is.
+It provides useful commands interacting with the compose setup
+and restic.
+
+The command can be executed inside the container or through ``run``.
+
+.. code:: bash
+
+    # Get the current status using run
+    $ docker-compose run --rm backup rcb status
+
+    # by entering the container
+    $ docker-compose exec backup sh
+    /restic-compose-backup # rcb status
+
+Log level can be overridden by using the ``--log-level``
+flag. This can help you better understand what is going on
+for example by using ``--log-level debug``.
+
+version
+~~~~~~~
+
+Displays the version.
+
+Example output::
+
+    /restic-compose-backup # rcb version
+    0.4.0
+
+status
+~~~~~~
+
+Shows the general status of our setup. The command is doing
+the following operations
+
+- Displays the name of the compose setup
+- Displays the repository path
+- Tells us if a backup is currently running
+- Removes stale backup process containers if the exist
+- Checks is the repository is initialized
+- Initializes the repository if this is not already done
+- Displays what volumes and databases are flagged for backup
+
+Example output::
+
+    INFO: Status for compose project 'myproject'
+    INFO: Repository: '<restic repository>'
+    INFO: Backup currently running?: False
+    INFO: --------------- Detected Config ---------------
+    INFO: service: mysql
+    INFO:  - mysql (is_ready=True)
+    INFO: service: mariadb
+    INFO:  - mariadb (is_ready=True)
+    INFO: service: postgres
+    INFO:  - postgres (is_ready=True)
+    INFO: service: web
+    INFO:  - volume: media
+    INFO:  - volume: /srv/files
+
+alert
+~~~~~
+
+Sends a test message to all configured alert backends
+and is there for you to verify that alerts are in
+fact working and configured correctly.
+
+The format of this message::
+
+    subject: myproject: Test Alert
+    body: Test message
+
+snapshots
+~~~~~~~~~
+
+Displays the latest snapshots in restic. This can also
+be done with ``restic snapshots``.
+
+Example output::
+
+    /restic-compose-backup # rcb snapshots
+    repository f325264e opened successfully, password is correct
+    ID        Time                 Host          Tags        Paths
+    ---------------------------------------------------------------------------------------------
+    19928e1c  2019-12-09 02:07:44  b3038db04ec1              /volumes
+    7a642f37  2019-12-09 02:07:45  b3038db04ec1              /databases/mysql/all_databases.sql
+    883dada4  2019-12-09 02:07:46  b3038db04ec1              /databases/mariadb/all_databases.sql
+    76ef2457  2019-12-09 02:07:47  b3038db04ec1              /databases/postgres/test.sql
+    ---------------------------------------------------------------------------------------------
+    4 snapshots
+
+backup
+~~~~~~
+
+Starts a backup process by spawning a new docker container.
+The network stack, mounted volumes, env vars etc. from the
+backup service are copied to this container.
+
+We attach to this container and stream the logs and delete
+the container with the backup process is completed. If the
+container for any reason should not be deleted, it will
+be in next backup run as these containers are tagged with
+a unique label and detected.
+
+If anything goes wrong the exist status of the container
+is non-zero and the logs from this backup run will be sent
+to the user through the configure alerts.
+
+This command is by default called by cron every
+day at 02:00 unless configured otherwise. We can also run this
+manually is needed.
+
+Running this command will do the following:
+
+* Checks if a backup process is already running.
+  If so, we alert the user and abort
+* Gathers all the volumes configured for backup and starts
+  the backup process with these volumes mounted into ``/volumes``
+* Checks the status of the process and reports to the user
+  if anything failed
+
+The backup process does the following:
+
+* ``status`` is first called to ensure everything is ok
+* Backs up ``/volumes`` if any volumes were mounted
+* Backs up each configured database
+* Runs ``cleanup`` purging snapshots based on the configured policy
+* Checks the health of the repository
+
+Example::
+
+        $ docker-compose exec backup sh
+        /restic-compose-backup # rcb backup
+        INFO: Starting backup container
+        INFO: Backup process container: loving_jepsen
+        INFO: 2019-12-09 04:50:22,817 - INFO: Status for compose project 'restic-compose-backup'
+        INFO: 2019-12-09 04:50:22,817 - INFO: Repository: '/restic_data'
+        INFO: 2019-12-09 04:50:22,817 - INFO: Backup currently running?: True
+        INFO: 2019-12-09 04:50:23,701 - INFO: ------------------------- Detected Config -------------------------
+        INFO: 2019-12-09 04:50:23,701 - INFO: service: mysql
+        INFO: 2019-12-09 04:50:23,718 - INFO:  - mysql (is_ready=True)
+        INFO: 2019-12-09 04:50:23,718 - INFO: service: mariadb
+        INFO: 2019-12-09 04:50:23,726 - INFO:  - mariadb (is_ready=True)
+        INFO: 2019-12-09 04:50:23,727 - INFO: service: postgres
+        INFO: 2019-12-09 04:50:23,734 - INFO:  - postgres (is_ready=True)
+        INFO: 2019-12-09 04:50:23,735 - INFO: service: web
+        INFO: 2019-12-09 04:50:23,736 - INFO:  - volume: /some/volume
+        INFO: 2019-12-09 04:50:23,736 - INFO: -------------------------------------------------------------------
+        INFO: 2019-12-09 04:50:23,736 - INFO: Backing up volumes
+        INFO: 2019-12-09 04:50:24,661 - INFO: Backing up databases
+        INFO: 2019-12-09 04:50:24,661 - INFO: Backing up mysql in service mysql
+        INFO: 2019-12-09 04:50:25,643 - INFO: Backing up mariadb in service mariadb
+        INFO: 2019-12-09 04:50:26,580 - INFO: Backing up postgres in service postgres
+        INFO: 2019-12-09 04:50:27,555 - INFO: Forget outdated snapshots
+        INFO: 2019-12-09 04:50:28,457 - INFO: Prune stale data freeing storage space
+        INFO: 2019-12-09 04:50:31,547 - INFO: Checking the repository for errors
+        INFO: 2019-12-09 04:50:32,869 - INFO: Backup completed
+        INFO: Backup container exit code: 0
+
+crontab
+~~~~~~~
+
+Generates and verifies the crontab. This is done automatically when
+the container starts. It can be user to verify the configuration.
+
+Example output::
+
+    /restic-compose-backup # rcb crontab
+    10 2 * * * source /env.sh && rcb backup > /proc/1/fd/1
+
+cleanup
+~~~~~~~
+
+Purges all snapshots based on the configured policy. (``RESTIC_KEEP_*``
+env variables). It runs ``restic forget`` and ``restic purge``.
+
+Example output::
+
+    /restic-compose-backup # rcb cleanup
+    2019-12-09 05:09:52,892 - INFO: Forget outdated snapshots
+    2019-12-09 05:09:53,776 - INFO: Prune stale data freeing storage space
+
+start-backup-process
+~~~~~~~~~~~~~~~~~~~~
+
+This can only be executed by the backup process container.
+Attempting to run this command in the backup service
+will simply tell you it's not possible.
+
+The backup process is doing the following:
+
+* ``status`` is first called to ensure everything is ok
+* Backs up ``/volumes`` if any volumes were mounted
+* Backs up each configured database
+* Runs ``cleanup`` purging snapshots based on the configured policy
+* Checks the health of the repository

docs/index.rst

@@ -6,12 +6,16 @@
 Welcome to restic-compose-backup's documentation!
 =================================================
 
-Simple backup with restic for small to medium docker-compose setups.
+Simple backup with restic_ for docker-compose setups.
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
    :caption: Contents:
 
+   guide/install
+   guide/configuration
+   guide/rcb
+   guide/advanced
 
 Indices and tables
 ==================
@@ -19,3 +23,5 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
+
+.. _restic: https://restic.net/

extras/release.md

@@ -13,9 +13,9 @@ When releasing a bugfix version we need to update the
 main image as well.
 
 ```bash
-docker build src --tag zettaio/restic-compose-backup:0.3
-docker build src --tag zettaio/restic-compose-backup:0.3.3
+docker build src --tag zettaio/restic-compose-backup:0.4
+docker build src --tag zettaio/restic-compose-backup:0.4.1
 
-docker push zettaio/restic-compose-backup:0.3
-docker push zettaio/restic-compose-backup:0.3.3
+docker push zettaio/restic-compose-backup:0.4
+docker push zettaio/restic-compose-backup:0.4.1
 ```

restic_compose_backup.env

@@ -1,6 +1,9 @@
 # DON'T COMMIT THIS FILE IF YOU MODIFY IN DEV
 
-DOCKER_BASE_URL=unix://tmp/docker.sock
+# DOCKER_HOST=unix://tmp/docker.sock
+# DOCKER_TLS_VERIFY=1
+# DOCKER_CERT_PATH=''
+
 RESTIC_REPOSITORY=/restic_data
 RESTIC_PASSWORD=password
 

src/restic_compose_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.4.0'
+__version__ = '0.4.2'

src/restic_compose_backup/backup_runner.py

@@ -17,7 +17,7 @@ def run(image: str = None, command: str = None, volumes: dict = None,
         labels=labels,
         # auto_remove=True,  # We remove the container further down
         detach=True,
-        environment=environment,
+        environment=environment + ['BACKUP_PROCESS_CONTAINER=true'],
         volumes=volumes,
         network_mode=f'container:{source_container_id}',  # Reuse original container's network stack.
         working_dir=os.getcwd(),

src/restic_compose_backup/cli.py

@@ -57,6 +57,9 @@ def status(config, containers):
     logger.info("Status for compose project '%s'", containers.project_name)
     logger.info("Repository: '%s'", config.repository)
     logger.info("Backup currently running?: %s", containers.backup_process_running)
+    logger.info("Checking docker availability")
+
+    utils.list_containers()
 
     if containers.stale_backup_process_containers:
         utils.remove_containers(containers.stale_backup_process_containers)
@@ -117,6 +120,7 @@ def backup(config, containers):
     mounts = containers.generate_backup_mounts('/volumes')
     volumes.update(mounts)
 
+    logger.debug('Starting backup container with image %s', containers.this_container.image)
     try:
         result = backup_runner.run(
             image=containers.this_container.image,
@@ -151,12 +155,18 @@ def backup(config, containers):
 
 def start_backup_process(config, containers):
     """The actual backup process running inside the spawned container"""
-    if (not containers.backup_process_container
-       or containers.this_container == containers.backup_process_container is False):
+    if not utils.is_true(os.environ.get('BACKUP_PROCESS_CONTAINER')):
         logger.error(
             "Cannot run backup process in this container. Use backup command instead. "
             "This will spawn a new container with the necessary mounts."
         )
+        alerts.send(
+            subject="Cannot run backup process in this container",
+            body=(
+                "Cannot run backup process in this container. Use backup command instead. "
+                "This will spawn a new container with the necessary mounts."
+            )
+        )
         exit(1)
 
     status(config, containers)

src/restic_compose_backup/config.py

@@ -10,7 +10,6 @@ class Config:
         # Mandatory values
         self.repository = os.environ.get('RESTIC_REPOSITORY')
         self.password = os.environ.get('RESTIC_REPOSITORY')
-        self.docker_base_url = os.environ.get('DOCKER_BASE_URL') or "unix://tmp/docker.sock"
         self.cron_schedule = os.environ.get('CRON_SCHEDULE') or self.default_crontab_schedule
         self.cron_command = os.environ.get('CRON_COMMAND') or self.default_backup_command
 

src/restic_compose_backup/containers.py

@@ -1,9 +1,11 @@
 import os
+import logging
 from pathlib import Path
 from typing import List
 
 from restic_compose_backup import enums, utils
 
+logger = logging.getLogger(__name__)
 
 VOLUME_TYPE_BIND = "bind"
 VOLUME_TYPE_VOLUME = "volume"
@@ -355,7 +357,7 @@ class RunningContainers:
             # Detect containers belonging to the current compose setup
             if (container.project_name == self.this_container.project_name
                and not container.is_oneoff):
-                if container.id != self.this_container.id:
+                if container != self.backup_process_container:
                     self.containers.append(container)
 
     @property

src/restic_compose_backup/restic.py

@@ -78,6 +78,8 @@ def is_initialized(repository: str) -> bool:
 def forget(repository: str, daily: str, weekly: str, monthly: str, yearly: str):
     return commands.run(restic(repository, [
         'forget',
+        '--group-by',
+        'paths',
         '--keep-daily',
         daily,
         '--keep-weekly',

src/restic_compose_backup/utils.py

@@ -3,7 +3,6 @@ import logging
 from typing import List
 from contextlib import contextmanager
 import docker
-from restic_compose_backup.config import Config
 
 logger = logging.getLogger(__name__)
 
@@ -11,8 +10,18 @@ TRUE_VALUES = ['1', 'true', 'True', True, 1]
 
 
 def docker_client():
-    config = Config()
-    return docker.DockerClient(base_url=config.docker_base_url)
+    """
+    Create a docker client from the following environment variables::
+
+        DOCKER_HOST=unix://tmp/docker.sock
+        DOCKER_TLS_VERIFY=1
+        DOCKER_CERT_PATH=''
+    """
+    # NOTE: Remove this fallback in 1.0
+    if not os.environ.get('DOCKER_HOST'):
+        os.environ['DOCKER_HOST'] = 'unix://tmp/docker.sock'
+
+    return docker.from_env()
 
 
 def list_containers() -> List[dict]:

src/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_namespace_packages
 setup(
     name="restic-compose-backup",
     url="https://github.com/ZettaIO/restic-compose-backup",
-    version="0.4.0",
+    version="0.4.2",
     author="Einar Forselv",
     author_email="eforselv@gmail.com",
     packages=find_namespace_packages(include=['restic_compose_backup']),

src/tests/container.json

@@ -160,7 +160,7 @@
     "OpenStdin": true,
     "StdinOnce": true,
     "Env": [
-      "DOCKER_BASE_URL=unix://tmp/docker.sock",
+      "DOCKER_HOST=unix://tmp/docker.sock",
       "RESTIC_REPOSITORY=/tmp/backup",
       "RESTIC_PASSWORD=password",
       "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

src/tests/tests.py

@@ -77,7 +77,7 @@ class ResticBackupTests(unittest.TestCase):
         ]
         with mock.patch(list_containers_func, fixtures.containers(containers=containers)):
             result = RunningContainers()
-            self.assertEqual(len(result.containers), 3, msg="Three containers expected")
+            self.assertEqual(len(result.containers), 4, msg="Three containers expected")
             self.assertNotEqual(result.this_container, None, msg="No backup container found")
             web_service = result.get_service('web')
             self.assertNotEqual(web_service, None)