contrib: added the common mixins library

contrib/mixins/README.md

@@ -0,0 +1,65 @@
+# Common mixins library
+
+A library of reusable mixins to simplify common tasks needed when taking backups.
+
+Currently, this library contains mixins for:
+* Dumping databases to restic stdin (mysql, mariadb, pgsql)
+* Taking temporary snapshots (lvm, btrfs)
+* Temporarily freezing VM disk images (virsh/kvm)
+
+## Example Usage
+
+`profiles.toml`
+
+```toml
+version = "2"
+
+# downloaded by "https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh" 
+includes = ["mixins-*.yaml"]
+
+[profiles.__base]
+repo = "..."
+
+[proflies.default]
+inherit = "__base"
+[proflies.default.backup]
+use = {name = "snapshot-btrfs", FROM = "/opt/data", TO = "/opt/data_snapshot"}
+source = "/opt/data_snapshot"
+
+[proflies.mysql]
+inherit = "__base"
+[proflies.mysql.backup]
+use = {name = "database-mysql", DATABASE = "dbname", USER="dbuser", PASSWORD_FILE="/path/to/password.txt"}
+
+[proflies.vms]
+inherit = "__base"
+[proflies.vms.backup]
+use = [
+ {name = "snapshot-virsh", DOMAIN = "vmname1", DUMPXML = "/opt/vms/vmname1.xml"},
+ {name = "snapshot-virsh", DOMAIN = "vmname2", DUMPXML = "/opt/vms/vmname2.xml"},
+ {name = "snapshot-virsh", DOMAIN = "vmname2", DUMPXML = "/opt/vms/vmname2.xml"},
+]
+source = "/opt/vms/"
+includes = ["*.qcow2", "*.xml"]
+```
+
+## Setup
+
+### Posix environment:
+
+```shell
+cd /etc/resticprofile \
+&& curl -sL https://raw.githubusercontent.com/creativeprojects/resticprofile/master/contrib/mixins/get.sh | sh -
+```
+
+### Powershell environment:
+
+At the moment the mixins library doesn't support powershell. Contributions are welcome.
+
+## Disclaimer
+
+Please note that the actions that some of the mixins perform can lead to data loss. This lies in the nature of 
+creating and removing snapshots with system privileges. You should carefully read through the actions inside 
+the mixins and consider if the mixin is safe for your use case. Please report bugs when you find any.
+
+This library is under the [GPL3](../../LICENSE) license like all other parts of the project.

contrib/mixins/database.yaml

@@ -0,0 +1,107 @@
+---
+title: "Mixins for database backups in posix shell environments"
+github_repo: "github.com/creativeprojects/resticprofile"
+license: "GPL3"
+copyright: |-
+ This file is part of resticprofile (github.com/creativeprojects/resticprofile).
+ Copyright (c) 2024 resticprofile authors.
+
+ This program is free software: you can redistribute it and/or modify 
+ it under the terms of the GNU General Public License as published by 
+ the Free Software Foundation, version 3.
+
+ This program is distributed in the hope that it will be useful, but 
+ WITHOUT ANY WARRANTY; without even the implied warranty of 
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License 
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+---
+mixins:
+
+ ###
+ # MySQL / MariaDB dump from stdin
+ #
+ # Usage
+ #
+ # backup:
+ # use: {name: "database-mysql", DATABASE: "dbname", HOST: "dbhost", USER: "dbuser", PASSWORD_FILE: "dbpassword.txt"}
+ #
+ database-mysql:
+ default-vars:
+ HOST: "${MYSQLDUMP_HOST}"
+ PORT: "${MYSQLDUMP_PORT}"
+ USER: "${MYSQLDUMP_USER}"
+ PASSWORD: "${MYSQLDUMP_PASSWORD}"
+ PASSWORD_FILE: "${MYSQLDUMP_PASSWORD_FILE}"
+ DATABASE: "--all-databases"
+ FILENAME: "mysql-dump.sql"
+ DEFAULT_OPTS: "${MYSQLDUMP_DEFAULT_OPTS:-'--order-by-primary'}"
+ OPTS: ""
+
+ # creating a defaults-file for mysqldump if the PASSWORD is non-empty
+ run-before...: |
+ defaults=""
+ __pw="${PASSWORD}"
+ if [ -n "${__pw}" ] || [ -n "${PASSWORD_FILE}" ] ; then 
+ defaults="{{ tempFile("mysql-default.conf") }}"
+ chmod 0600 "$defaults"
+ fi
+ [ -f "$defaults" ] && cat > "$defaults" <<-EOF
+ [mysqldump]
+ password = "${__pw:-"$(cat "${PASSWORD_FILE}")"}"
+ EOF
+ echo "MYSQL_DEFAULTS_FILE=${defaults}" >> "{{ env }}"
+
+ # defining stdin command to that writes the mysql dump
+ stdin-filename: "${FILENAME}"
+ stdin-command: >
+ host="$( [ -z "${HOST}" ] || echo "--host='${HOST}'" )"
+ port="$( [ -z "${PORT}" ] || echo "--port='${PORT}'" )"
+ user="$( [ -z "${USER}" ] || echo "-u '${USER}'" )"
+ pass="$( [ -z "${MYSQL_DEFAULTS_FILE}" ] || echo "--defaults-file='${MYSQL_DEFAULTS_FILE}'" )"
+ mysqldump $user $pass $host $port ${DEFAULT_OPTS} ${OPTS} ${DATABASE}
+ source: []
+
+
+ ###
+ # Postgres dump from stdin
+ #
+ # Usage
+ #
+ # backup:
+ # use: {name: "database-pgsql", DATABASE: "dbname", HOST: "dbhost", USER: "dbuser", PASSWORD_FILE: "dbpassword.txt"}
+ #
+ database-pgsql:
+ default-vars:
+ HOST: "${PGDUMP_HOST}"
+ PORT: "${PGDUMP_PORT}"
+ USER: "${PGDUMP_USER}"
+ PASSWORD: "${PGDUMP_PASSWORD}"
+ PASSWORD_FILE: "${PGDUMP_PASSWORD_FILE}"
+ DATABASE: ""
+ FILENAME: "pgsql-dump.sql"
+ DEFAULT_OPTS: "${PGDUMP_DEFAULT_OPTS:-'--clean --if-exists --quote-all-identifiers --serializable-deferrable'}"
+ OPTS: ""
+
+ # creating a .pgpass file for pg_dump if the PASSWORD is non-empty
+ run-before...: |
+ pgpass=""
+ __pw="${PASSWORD}"
+ if [ -n "${__pw}" ] || [ -n "${PASSWORD_FILE}" ] ; then 
+ pgpass="{{ tempFile(".pgpass") }}"
+ echo "*:*:*:*:${__pw:-"$(cat "${PASSWORD_FILE}")"}" > "$pgpass"
+ chmod 0600 "$pgpass" 
+ fi
+ echo "PGPASS_FILE=${pgpass}" >> "{{ env }}" 
+
+ # defining stdin command to that writes the mysql dump
+ stdin-filename: "${FILENAME}"
+ stdin-command: >
+ [ -z "${PGPASS_FILE}" ] || export HOME=$(dirname "${PGPASS_FILE}") 
+ ; host="$( [ -z "${HOST}" ] || echo "--host='${HOST}'" )"
+ port="$( [ -z "${PORT}" ] || echo "--port='${PORT}'" )"
+ user="$( [ -z "${USER}" ] || echo "--username='${USER}'" )"
+ pg_dump $user --no-password $host $port ${DEFAULT_OPTS} ${OPTS} ${DATABASE}
+ source: []

contrib/mixins/get.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -e
+
+MIXINS="
+ database
+ snapshot
+"
+
+PREFIX="${RESTICPROFILE_MIXINS_PREFIX:-"mixins"}"
+TEMP_FILE="${TMPDIR:-/tmp}/.rp-mixin.tmp"
+GIT_BRANCH="${RESTICPROFILE_BRANCH:-"master"}"
+GIT_BASE_URL="${RESTICPROFILE_GIT_URL:-"https://raw.githubusercontent.com/creativeprojects/resticprofile/${GIT_BRANCH}/contrib/mixins"}"
+
+move_download_to() {
+ [ -s "$TEMP_FILE" ] && mv -f "$TEMP_FILE" "$1"
+ return $?
+}
+
+download() {
+ result=0
+ if which -s curl ; then
+ curl -fsL "$1" > "$TEMP_FILE" && move_download_to "$2"
+ result=$?
+ elif which -s wget ; then
+ wget -nv -O "$TEMP_FILE" "$1" && move_download_to "$2"
+ result=$?
+ else
+ echo "neither curl nor wget found, cannot load $1"
+ result=1
+ fi
+
+ [ -e "$TEMP_FILE" ] && rm "$TEMP_FILE"
+ return $result
+}
+
+download_all() {
+ dir=""
+ if [ -n "$1" ] && [ -d "$1" ] ; then
+ dir="$1/"
+ echo "downloading to $dir"
+ fi
+ for m in $MIXINS ; do
+ url="$GIT_BASE_URL/${m}.yaml"
+ dest="${dir}${PREFIX}-${m}.yaml"
+ echo "getting $url > $dest"
+ download "$url" "$dest" || echo "failed"
+ done
+}
+
+download_all "$1"

contrib/mixins/snapshot.yaml

@@ -0,0 +1,151 @@
+---
+title: "Mixins for snapshot creation in posix shell environments"
+github_repo: "github.com/creativeprojects/resticprofile"
+license: "GPL3"
+copyright: |-
+ This file is part of resticprofile (github.com/creativeprojects/resticprofile).
+ Copyright (c) 2024 resticprofile authors.
+
+ This program is free software: you can redistribute it and/or modify 
+ it under the terms of the GNU General Public License as published by 
+ the Free Software Foundation, version 3.
+
+ This program is distributed in the hope that it will be useful, but 
+ WITHOUT ANY WARRANTY; without even the implied warranty of 
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License 
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+---
+mixins:
+ ##
+ # Creates a temporary snapshot of a btrfs volume
+ # Usage
+ #
+ # backup:
+ # use:
+ # - {name: "snapshot-btrfs", FROM: "/opt/data", TO: "/opt/data_snapshot"}
+ # source: "/opt/data_snapshot"
+ #
+ snapshot-btrfs:
+ default-vars:
+ FROM: ""
+ TO: ""
+ IF_EXISTS: "fail" # one of (delete, continue, fail)
+
+ ...run-before: |
+ if [ ! -d "${FROM}" ] ; then echo "source volume (${FROM}) is not existing" ; exit 1 ; fi 
+ if [ -z "${TO}" ] ; then echo "snapshot destination is not specified" ; exit 1 ; fi
+ if [ "${TO}" == "${FROM}" ] ; then echo "snapshot source & destination must differ" ; exit 1 ; fi
+ ;
+ if [ -d "${TO}" ] ; then
+ if [ "${IF_EXISTS}" == "delete" ] ; then
+ btrfs subvolume delete "${TO}"
+ elif [ "${IF_EXISTS}" == "continue" ] ; then
+ echo "${TO} already existing, continuing without creating a new snapshot" ; exit 0
+ else
+ echo "${TO} already existing, snapshot failed" ; exit 1
+ fi
+ fi
+ ;
+ btrfs subvolume snapshot -r "${FROM}" "${TO}" \
+ && echo "btrfs:${FROM}:${TO}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}" 
+
+ run-finally...: |
+ if [ -d "${TO}" ] && grep -q "btrfs:${FROM}:${TO}--" "{{ tempFile "mixins-lib-snapshots.list" }}" ; then
+ btrfs subvolume delete "${TO}"
+ fi
+
+ ##
+ # Creates a temporary snapshot of a lvm volume
+ # Usage
+ #
+ # backup:
+ # use:
+ # - {name: "snapshot-lvm", FROM: "/dev/vg00/data", TO: "/mnt/data_snapshot"}
+ # source: "/mnt/data_snapshot"
+ #
+ snapshot-lvm:
+ default-vars:
+ FROM: ""
+ TO: ""
+ SNAPSHOT_NAME: ""
+ DEFAULT_OPTS: "-l100%FREE"
+ OPTS: ""
+
+ ...run-before: |
+ if [ ! -e "${FROM}" ] ; then echo "source volume (${FROM}) is not existing" ; exit 1 ; fi 
+ if [ -z "${TO}" ] ; then echo "snapshot destination is not specified" ; exit 1 ; fi 
+ if [ "${TO}" == "${FROM}" ] ; then echo "snapshot source & destination must differ" ; exit 1 ; fi
+ ( [ -d "${TO}" ] || mkdir -p "${TO}" ) || exit 1
+ ;
+ export snap_name="$( [ -z "${SNAPSHOT_NAME}" ] || echo "${SNAPSHOT_NAME}" )"
+ snap_name="restic_${snap_name:-"$(basename "${TO}")"}"
+ export snap_dev="$(dirname "${FROM}")/${snap_name}"
+ ;
+ lvcreate ${DEFAULT_OPTS} ${OPTS} --name "${snap_name}" --snapshot "${FROM}" \
+ && mount "${snap_dev}" "${TO}" \
+ && echo "lvm:${snap_dev}:${TO}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}"
+
+ run-finally...: |
+ export snap_name="$( [ -z "${SNAPSHOT_NAME}" ] || echo "${SNAPSHOT_NAME}" )" 
+ snap_name="restic_${snap_name:-"$(basename "${TO}")"}" 
+ export snap_dev="$(dirname "${FROM}")/${snap_name}"
+ ;
+ if [ -d "${TO}" ] && grep -q "lvm:${snap_dev}:${TO}--" "{{ tempFile "mixins-lib-snapshots.list" }}" ; then 
+ umount "${TO}" \
+ && lvremove --force "${snap_dev}" 
+ fi
+
+
+ ##
+ # Temporarily freezes the image of a VM managed by virsh so that it can be backed-up while the keeps VM running
+ # Usage
+ #
+ # backup:
+ # use:
+ # - {name: "snapshot-virsh", DOMAIN: "vmname", DUMPXML: "/opt/vms/vmname-definition.xml"}
+ # - {name: "snapshot-virsh", DOMAIN: "vm2", DUMPXML: "/opt/vms/vm2-definition.xml"}
+ # - {name: "snapshot-virsh", DOMAIN: "vm3"}
+ # - {name: "snapshot-virsh", DOMAIN: "vm4"}
+ # - {name: "snapshot-virsh", DOMAIN: "vm4-without-quest-additions", OPTS: ""}
+ # # OPTIONAL: - "snapshot-virsh-aa-teardown" # place last if snapshot restores block with apparmor)
+ # source:
+ # - /opt/vms
+ #
+ snapshot-virsh:
+ default-vars:
+ DOMAIN: ""
+ DUMPXML: ""
+ LD_PATH: "/var/db"
+ LD_SUFFIX: "livedata.qcow2"
+ LD_DISCSPEC: "vda"
+ SNAPSHOT_SUFFIX: "restic-backup"
+ DEFAULT_OPTS: "--atomic --no-metadata"
+ OPTS: "--quiesce" # (fsync in guest OS, needs guest additions, highly recommended)
+
+ excludes...:
+ - "*-${LD_SUFFIX}"
+
+ ...run-before: >
+ virsh snapshot-create-as --domain "${DOMAIN}" --name "${DOMAIN}-${SNAPSHOT_SUFFIX}"
+ --diskspec "${LD_DISCSPEC},file=${LD_PATH}/${DOMAIN}-${LD_SUFFIX}"
+ --disk-only ${DEFAULT_OPTS} ${OPTS}
+ && echo "virsh:${LD_PATH}/${DOMAIN}-${LD_SUFFIX}--" >> "{{ tempFile "mixins-lib-snapshots.list" }}"
+ && ( [ -z "${DUMPXML}" ] 
+ || virsh dumpxml --domain "${DOMAIN}" --inactive --migratable > "${DUMPXML}" ) 
+
+ run-finally...: >
+ grep -q "virsh:${LD_PATH}/${DOMAIN}-${LD_SUFFIX}--" "{{ tempFile "mixins-lib-snapshots.list" }}"
+ && virsh blockcommit --domain "${DOMAIN}" ${LD_DISCSPEC} --wait --active
+ && virsh blockjob --domain "${DOMAIN}" "${LD_PATH}/${DOMAIN}-${LD_SUFFIX}" --pivot
+ && rm -f "${LD_PATH}/${DOMAIN}-${LD_SUFFIX}"
+
+ #
+ # Utility to teardown apparmor before other run finals (e.g. blockcommit / blockjob) and restart it afterward
+ # See "snapshot-virsh"
+ #
+ snapshot-virsh-aa-teardown:
+ ...run-finally: aa-teardown
+ run-finally...: service apparmor restart