add example - rclone remote backend for restic using storj (aka 'uplink' API)

* Use --no-tablespaces in mysqldump
* Dump to 0.7.0
* pin docker version
* Include missing packages in setup.py

.readthedocs.yaml

@@ -0,0 +1,24 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+python:
+  install:
+    - requirements: docs/requirements.txt

docker-compose.yaml

@@ -1,10 +1,9 @@
-version: '3.7'
 services:
   backup:
     build: ./src
     env_file:
       - restic_compose_backup.env
-      - alerts.env
+      # - alerts.env
     labels:
       restic-compose-backup.volumes: true
       restic-compose-backup.volumes.include: 'src'
@@ -32,7 +31,7 @@ services:
       - SOME_VALUE=test
       - ANOTHER_VALUE=1
 
-  mysql:
+  mysql5:
     image: mysql:5
     labels:
       restic-compose-backup.mysql: true
@@ -42,7 +41,19 @@ services:
       - MYSQL_USER=myuser
       - MYSQL_PASSWORD=mypassword
     volumes:
-      - mysqldata:/var/lib/mysql
+      - mysqldata5:/var/lib/mysql
+
+  mysql8:
+    image: mysql:8
+    labels:
+      restic-compose-backup.mysql: true
+    environment:
+      - MYSQL_ROOT_PASSWORD=my-secret-pw
+      - MYSQL_DATABASE=mydb
+      - MYSQL_USER=myuser
+      - MYSQL_PASSWORD=mypassword
+    volumes:
+      - mysqldata8:/var/lib/mysql
 
   mariadb:
     image: mariadb:10
@@ -68,7 +79,8 @@ services:
       - pgdata:/var/lib/postgresql/data
 
 volumes:
-  mysqldata:
+  mysqldata5:
+  mysqldata8:
   mariadbdata:
   pgdata:
 

extras/example-rclone-backend/backup.env

@@ -0,0 +1,69 @@
+
+DOCKER_HOST=unix://var/run/docker.sock
+# DOCKER_TLS_VERIFY=1
+# DOCKER_CERT_PATH=''
+
+INCLUDE_PROJECT_NAME=false
+EXCLUDE_BIND_MOUNTS=false
+
+# storj bucket must exist
+RESTIC_REPOSITORY=rclone:uplink:mybucket
+RESTIC_PASSWORD="password"
+
+RCLONE_CONFIG_UPLINK_TYPE=storj
+# access grant must have permission to use bucket
+RCLONE_CONFIG_UPLINK_ACCESS_GRANT=
+
+RESTIC_KEEP_DAILY=7
+RESTIC_KEEP_WEEKLY=4
+RESTIC_KEEP_MONTHLY=12
+RESTIC_KEEP_YEARLY=3
+
+LOG_LEVEL=info
+CRON_SCHEDULE="10 2 * * *"
+
+# EMAIL_HOST=
+# EMAIL_PORT=
+# EMAIL_HOST_USER=
+# EMAIL_HOST_PASSWORD=
+# EMAIL_SEND_TO=
+
+# DISCORD_WEBHOOK=u
+
+# Various env vars for restic : https://restic.readthedocs.io/en/stable/040_backup.html#environment-variables
+# RESTIC_REPOSITORY                   Location of repository (replaces -r)
+# RESTIC_PASSWORD_FILE                Location of password file (replaces --password-file)
+# RESTIC_PASSWORD                     The actual password for the repository
+# 
+# AWS_ACCESS_KEY_ID                   Amazon S3 access key ID
+# AWS_SECRET_ACCESS_KEY               Amazon S3 secret access key
+# 
+# ST_AUTH                             Auth URL for keystone v1 authentication
+# ST_USER                             Username for keystone v1 authentication
+# ST_KEY                              Password for keystone v1 authentication
+# 
+# OS_AUTH_URL                         Auth URL for keystone authentication
+# OS_REGION_NAME                      Region name for keystone authentication
+# OS_USERNAME                         Username for keystone authentication
+# OS_PASSWORD                         Password for keystone authentication
+# OS_TENANT_ID                        Tenant ID for keystone v2 authentication
+# OS_TENANT_NAME                      Tenant name for keystone v2 authentication
+# 
+# OS_USER_DOMAIN_NAME                 User domain name for keystone authentication
+# OS_PROJECT_NAME                     Project name for keystone authentication
+# OS_PROJECT_DOMAIN_NAME              PRoject domain name for keystone authentication
+# 
+# OS_STORAGE_URL                      Storage URL for token authentication
+# OS_AUTH_TOKEN                       Auth token for token authentication
+# 
+# B2_ACCOUNT_ID                       Account ID or applicationKeyId for Backblaze B2
+# B2_ACCOUNT_KEY                      Account Key or applicationKey for Backblaze B2
+# 
+# AZURE_ACCOUNT_NAME                  Account name for Azure
+# AZURE_ACCOUNT_KEY                   Account key for Azure
+# 
+# GOOGLE_PROJECT_ID                   Project ID for Google Cloud Storage
+# GOOGLE_APPLICATION_CREDENTIALS      Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json)
+# 
+# RCLONE_BWLIMIT                      rclone bandwidth limit
+# RCLONE_VERBOSE=1                    # 2 for -vv

extras/example-rclone-backend/compose.yaml

@@ -0,0 +1,23 @@
+services:
+  backup:
+    build: ../../src
+    env_file:
+      - backup.env
+    labels:
+      restic-compose-backup.volumes: false
+    volumes:
+      # Map in docker socket
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Map restic cache
+      - ./restic_cache:/cache
+  postgres:
+    image: postgres:12
+    env_file:
+      - postgres.env
+    labels:
+      restic-compose-backup.postgres: true
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+
+volumes:
+  pgdata:

extras/example-rclone-backend/postgres.env

@@ -0,0 +1,3 @@
+POSTGRES_USER=pguser
+POSTGRES_PASSWORD=pgpassword
+POSTGRES_DB=test

src/Dockerfile

@@ -1,10 +1,22 @@
-FROM restic/restic:0.9.6
+FROM rclone/rclone:1 AS rclone
 
-RUN apk update && apk add python3 dcron mariadb-client postgresql-client
+FROM restic/restic:0.16.4 AS restic
+
+FROM python:3.12-alpine
+
+COPY --from=rclone /usr/local/bin/rclone /usr/local/bin/rclone
+COPY --from=restic /usr/bin/restic /usr/local/bin/restic
+
+RUN apk update \
+    && apk add \
+        dcron \
+        mariadb-client \
+        postgresql-client \
+        mariadb-connector-c-dev
 
 ADD . /restic-compose-backup
 WORKDIR /restic-compose-backup
-RUN pip3 install -U pip setuptools wheel && pip3 install -e .
+RUN pip install -U pip setuptools wheel && pip install .
 ENV XDG_CACHE_HOME=/cache
 
 ENTRYPOINT []

src/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # Dump all env vars so we can source them in cron jobs
-printenv | sed 's/^\(.*\)$/export \1/g' > /env.sh
+rcb dump-env > /env.sh
 
 # Write crontab
 rcb crontab > crontab

src/restic_compose_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.6.0'
+__version__ = '0.7.1'

src/restic_compose_backup/alerts/smtp.py

@@ -15,7 +15,7 @@ class SMTPAlert(BaseAlert):
         self.host = host
         self.port = port
         self.user = user
-        self.password = password
+        self.password = password or ""
         self.to = to
 
     @classmethod
@@ -34,7 +34,7 @@ class SMTPAlert(BaseAlert):
 
     @property
     def properly_configured(self) -> bool:
-        return self.host and self.port and self.user and self.password and len(self.to) > 0
+        return self.host and self.port and self.user and len(self.to) > 0
 
     def send(self, subject: str = None, body: str = None, alert_type: str = 'INFO'):
         # send_mail("Hello world!")

src/restic_compose_backup/cli.py

@@ -51,6 +51,9 @@ def main():
     elif args.action == "crontab":
         crontab(config)
 
+    elif args.action == "dump-env":
+        dump_env()
+
     # Random test stuff here
     elif args.action == "test":
         nodes = utils.get_swarm_nodes()
@@ -290,6 +293,14 @@ def crontab(config):
     print(cron.generate_crontab(config))
 
 
+def dump_env():
+    """Dump all environment variables to a script that can be sourced from cron"""
+    print("#!/bin/bash")
+    print("# This file was generated by restic-compose-backup")
+    for key, value in os.environ.items():
+        print("export {}='{}'".format(key, value))
+
+
 def parse_args():
     parser = argparse.ArgumentParser(prog='restic_compose_backup')
     parser.add_argument(
@@ -303,6 +314,7 @@ def parse_args():
             'cleanup',
             'version',
             'crontab',
+            'dump-env',
             'test',
         ],
     )

src/restic_compose_backup/containers_db.py

@@ -41,6 +41,7 @@ class MariadbContainer(Container):
             f"--port={creds['port']}",
             f"--user={creds['username']}",
             "--all-databases",
+            "--no-tablespaces",
         ]
 
     def backup(self):
@@ -100,6 +101,7 @@ class MysqlContainer(Container):
             f"--port={creds['port']}",
             f"--user={creds['username']}",
             "--all-databases",
+            "--no-tablespaces",
         ]
 
     def backup(self):

src/setup.py

@@ -3,12 +3,15 @@ from setuptools import setup, find_namespace_packages
 setup(
     name="restic-compose-backup",
     url="https://github.com/ZettaIO/restic-compose-backup",
-    version="0.6.0",
+    version="0.7.1",
     author="Einar Forselv",
     author_email="eforselv@gmail.com",
-    packages=find_namespace_packages(include=['restic_compose_backup']),
+    packages=find_namespace_packages(include=[
+        'restic_compose_backup',
+        'restic_compose_backup.*',
+    ]),
     install_requires=[
-        'docker==4.1.*',
+        'docker~=7.1.0',
     ],
     entry_points={'console_scripts': [
         'restic-compose-backup = restic_compose_backup.cli:main',