From acd4ff4962244f9c88a5b97618a7e429759cce2e Mon Sep 17 00:00:00 2001 From: Ruben van Staveren Date: Tue, 19 Mar 2024 11:33:12 +0100 Subject: [PATCH] Store backup scripts --- client/README.md | 19 ++++++++++ client/zfs-receive.sh | 35 ++++++++++++++++++ jail/README.md | 25 +++++++++++++ jail/zfs-receive.sh | 2 ++ server/README.md | 6 ++++ server/etc/periodic/hourly/500.zfs-backup | 7 ++++ server/zfs-send.sh | 44 +++++++++++++++++++++++ 7 files changed, 138 insertions(+) create mode 100644 client/README.md create mode 100755 client/zfs-receive.sh create mode 100644 jail/README.md create mode 100755 jail/zfs-receive.sh create mode 100644 server/README.md create mode 100755 server/etc/periodic/hourly/500.zfs-backup create mode 100755 server/zfs-send.sh diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..42b3db4 --- /dev/null +++ b/client/README.md @@ -0,0 +1,19 @@ +# client portion of remote zfs pool backup + +* Uses hardened ssh access +* Uses a geli encrypted zvol to receive the pool + * the geli zvol is only used on demand, the backup pool is imported but not mounted + * _using geli also allows for having a zpool on a zvol which is normally not possible_ + +# ssh configuration + +Add the following to your sshd configuration. The connection comes from a jail that functions as an indermediate agent + +``` +Match user root Address 2a10:3781:3e9:1::da7a:caf3 + AllowTcpForwarding no + ForceCommand /root/zfs-receive.sh + PermitRootLogin prohibit-password + PermitTTY no + X11Forwarding no +``` diff --git a/client/zfs-receive.sh b/client/zfs-receive.sh new file mode 100755 index 0000000..56d5b53 --- /dev/null +++ b/client/zfs-receive.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -e +POOL=backup-chassis +ZVOL=system/backup/chassis.verweg.com/store +MAILTO=root + +if [ -n "$SSH_ORIGINAL_COMMAND" ]; then + set -- $SSH_ORIGINAL_COMMAND +fi + +if [ $# -lt 1 ];then + echo "No destination dataset given" + exit 1 +fi +# zfs destroy backup-chassis/system/%recv +( +zfs snapshot system/backup/chassis.verweg.com/store@backup +geli attach -k /opt/backup/key -j /opt/backup/pass /dev/zvol/${ZVOL} +zpool import -R /backup/chassis.verweg.com/ -N ${POOL} +if zstdcat -vf | time zfs receive -v -Fu ${POOL}/${1}; then + zpool export ${POOL} + geli detach /dev/zvol/${ZVOL}.eli + zfs destroy system/backup/chassis.verweg.com/store@backup + logger -t zfs-receive "succesful backup for ${POOL}/${1}" + echo "OK" +else + logger -t zfs-receive "failed backup for ${POOL}/${1}" + echo "zfs-receive: failed backup for ${POOL}/${1}" | mail $MAILTO + zpool export ${POOL} + geli detach /dev/zvol/${ZVOL}.eli + echo "ERROR" + exit 1 +fi +) 2>&1 | tee -a /var/log/zfs-receive.log +exit 0 diff --git a/jail/README.md b/jail/README.md new file mode 100644 index 0000000..a32b4ac --- /dev/null +++ b/jail/README.md @@ -0,0 +1,25 @@ +# intermediate backup agent + +* Server "sends" the backup to the backup jail +* The backup jail is ipv6 only, mostly empty, and uses an hardened ssh configuration +* the receive script immediatly reconnects to the system actually receiving the backup + +# Setup jail sshd + +Add the following to the sshd of the jail. To maximise security ssh certificates are used (but you can do without ymmv) + + +``` +AcceptEnv LANG LC_* +ChallengeResponseAuthentication no +PasswordAuthentication no +PrintMotd no +RevokedKeys /etc/ssh/ssh_revoked_keys +Subsystem sftp /usr/libexec/sftp-server +TrustedUserCAKeys /etc/ssh/backup-ca.pub +UsePAM no +X11Forwarding yes +Match User root Address 2a02:898::96:1 + ForceCommand /root/zfs-receive.sh + PermitRootLogin forced-commands-only +``` diff --git a/jail/zfs-receive.sh b/jail/zfs-receive.sh new file mode 100755 index 0000000..3cf1947 --- /dev/null +++ b/jail/zfs-receive.sh @@ -0,0 +1,2 @@ +#! /bin/sh +ssh helium.niet.verweg.com "$SSH_ORIGINAL_COMMAND" diff --git a/server/README.md b/server/README.md new file mode 100644 index 0000000..61da9df --- /dev/null +++ b/server/README.md @@ -0,0 +1,6 @@ +# server off site zfs backup + +* uses the `/root/zfs-send.sh` called from `/usr/local/etc/periodic/hourly/500.zfs-backup` + * the script is quiet, unless it is run from a controlling terminal for diagnostic purposes + * Uses the file `/var/db/backup/last-${backup_host}-${pool}` to determine which incremental backup to make + * when not present, will do a full backup diff --git a/server/etc/periodic/hourly/500.zfs-backup b/server/etc/periodic/hourly/500.zfs-backup new file mode 100755 index 0000000..b3ea812 --- /dev/null +++ b/server/etc/periodic/hourly/500.zfs-backup @@ -0,0 +1,7 @@ +#! /bin/sh +if [ -t 1 ]; then + env HOME=/root lockf -st 300 /var/run/zfs-send.lock /root/zfs-send.sh +else + env HOME=/root lockf -st 300 /var/run/zfs-send.lock /root/zfs-send.sh 2>&1 > /dev/null +fi +find /var/tmp/ -name backup-backup.niet.verweg.com-\* -ctime +14 -delete diff --git a/server/zfs-send.sh b/server/zfs-send.sh new file mode 100755 index 0000000..a9df617 --- /dev/null +++ b/server/zfs-send.sh @@ -0,0 +1,44 @@ +#! /bin/sh +# vim:ts=4:sw=4: +set -e +HOSTS="backup.niet.verweg.com" +POOLS="system" +BACKUP_PATH="${1}" +for backup_host in $HOSTS; do + if nc -w 5 $backup_host 22 2>&1 > /dev/null; then + for pool in $POOLS; do + last_snapshot=$(zfs list -H -o name -t snap -r -d 1 $pool | tail -1) + last_send_snapshot_file="/var/db/backup/last-${backup_host}-${pool}" + test -f ${last_send_snapshot_file} && last_send_snapshot=$(cat ${last_send_snapshot_file}) || last_send_snapshot="" + if [ "${last_snapshot}" != "${pool}@${last_send_snapshot}" ]; then + test -n "${last_send_snapshot}" && incremental_opt="-I ${pool}@${last_send_snapshot}" + echo zfs send -n -vR ${incremental_opt} ${last_snapshot} + zfs send -n -vR ${incremental_opt} ${last_snapshot} 2>&1 | \ + tail -1 | \ + tee /var/tmp/backup-${backup_host}-$$.log + if [ -n "${BACKUP_PATH}" -a -d "${BACKUP_PATH}" ]; then + zfs send -vR ${incremental_opt} ${last_snapshot} | \ + zstd -v -6 --long > ${BACKUP_PATH}/${pool}-${backup_host}-${last_snapshot}.zstd | \ + tee -a /var/tmp/backup-${backup_host}-$$.log + echo "Stored backup in ${BACKUP_PATH}/${pool}-${backup_host}-${last_snapshot}.zstd" + echo "rsync this to the other server and use as input" + else + zfs send -vR ${incremental_opt} ${last_snapshot} | \ + zstd -v -6 --long | \ + ssh ${backup_host} $pool | \ + tee -a /var/tmp/backup-${backup_host}-$$.log + fi + status=$(tail -1 /var/tmp/backup-${backup_host}-$$.log) + echo ${last_snapshot} | sed -e "s/${pool}@//" + if [ "${status}" = "OK" ]; then + echo ${last_snapshot} | sed -e "s/${pool}@//" > ${last_send_snapshot_file} + mv /var/tmp/backup-${backup_host}-$$.log /var/log/zfs-backup-${backup_host}-${pool}-$(date +%a).log + fi + else + echo "Backup for $pool up to date for ${backup_host}" + fi + done + else + echo "WARNING: $backup_host not reachable over IPv6" + fi +done