Store backup scripts

client/README.md

@@ -0,0 +1,19 @@
+# client portion of remote zfs pool backup
+
+* Uses hardened ssh access
+* Uses a geli encrypted zvol to receive the pool
+  * the geli zvol is only used on demand, the backup pool is imported but not mounted
+  * _using geli also allows for having a zpool on a zvol which is normally not possible_
+
+# ssh configuration
+
+Add the following to your sshd configuration. The connection comes from a jail that functions as an indermediate agent
+
+```
+Match user root Address 2a10:3781:3e9:1::da7a:caf3
+    AllowTcpForwarding no
+    ForceCommand /root/zfs-receive.sh
+    PermitRootLogin prohibit-password
+    PermitTTY no
+    X11Forwarding no
+```

client/zfs-receive.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+set -e
+POOL=backup-chassis
+ZVOL=system/backup/chassis.verweg.com/store
+MAILTO=root
+
+if [ -n "$SSH_ORIGINAL_COMMAND" ]; then
+	set -- $SSH_ORIGINAL_COMMAND
+fi
+
+if [ $# -lt 1 ];then
+	echo "No destination dataset given"
+	exit 1
+fi
+# zfs destroy backup-chassis/system/%recv
+(
+zfs snapshot system/backup/chassis.verweg.com/store@backup
+geli attach -k /opt/backup/key -j /opt/backup/pass /dev/zvol/${ZVOL}
+zpool import -R /backup/chassis.verweg.com/ -N ${POOL}
+if zstdcat -vf | time zfs receive -v -Fu ${POOL}/${1}; then
+	zpool export ${POOL}
+	geli detach /dev/zvol/${ZVOL}.eli
+	zfs destroy system/backup/chassis.verweg.com/store@backup
+	logger -t zfs-receive "succesful backup for ${POOL}/${1}"
+	echo "OK"
+else
+	logger -t zfs-receive "failed backup for ${POOL}/${1}"
+	echo "zfs-receive: failed backup for ${POOL}/${1}" | mail $MAILTO
+	zpool export ${POOL}
+	geli detach /dev/zvol/${ZVOL}.eli
+	echo "ERROR"
+	exit 1
+fi
+) 2>&1 | tee -a /var/log/zfs-receive.log
+exit 0

jail/README.md

@@ -0,0 +1,25 @@
+# intermediate backup agent
+
+* Server "sends" the backup to the backup jail
+* The backup jail is ipv6 only, mostly empty, and uses an hardened ssh configuration
+* the receive script immediatly reconnects to the system actually receiving the backup
+
+# Setup jail sshd
+
+Add the following to the sshd of the jail. To maximise security ssh certificates are used (but you can do without ymmv)
+
+
+```
+AcceptEnv LANG LC_*
+ChallengeResponseAuthentication no
+PasswordAuthentication no
+PrintMotd no
+RevokedKeys /etc/ssh/ssh_revoked_keys
+Subsystem sftp /usr/libexec/sftp-server
+TrustedUserCAKeys /etc/ssh/backup-ca.pub
+UsePAM no
+X11Forwarding yes
+Match User root Address 2a02:898::96:1
+    ForceCommand /root/zfs-receive.sh
+    PermitRootLogin forced-commands-only
+```

jail/zfs-receive.sh

@@ -0,0 +1,2 @@
+#! /bin/sh
+ssh helium.niet.verweg.com "$SSH_ORIGINAL_COMMAND"

server/README.md

@@ -0,0 +1,6 @@
+# server off site zfs backup
+
+* uses the `/root/zfs-send.sh` called from `/usr/local/etc/periodic/hourly/500.zfs-backup`
+  * the script is quiet, unless it is run from a controlling terminal for diagnostic purposes
+  * Uses the file `/var/db/backup/last-${backup_host}-${pool}` to determine which incremental backup to make
+    * when not present, will do a full backup

server/etc/periodic/hourly/500.zfs-backup

@@ -0,0 +1,7 @@
+#! /bin/sh
+if [ -t 1 ]; then
+	env HOME=/root lockf -st 300 /var/run/zfs-send.lock /root/zfs-send.sh 
+else
+	env HOME=/root lockf -st 300 /var/run/zfs-send.lock /root/zfs-send.sh 2>&1 > /dev/null
+fi
+find /var/tmp/ -name backup-backup.niet.verweg.com-\* -ctime +14 -delete

server/zfs-send.sh

@@ -0,0 +1,44 @@
+#! /bin/sh
+# vim:ts=4:sw=4:
+set -e
+HOSTS="backup.niet.verweg.com"
+POOLS="system"
+BACKUP_PATH="${1}"
+for backup_host in $HOSTS; do
+	if nc -w 5 $backup_host 22 2>&1 > /dev/null; then
+		for pool in $POOLS; do
+			last_snapshot=$(zfs list -H -o name -t snap -r -d 1 $pool | tail -1)
+			last_send_snapshot_file="/var/db/backup/last-${backup_host}-${pool}"
+			test -f ${last_send_snapshot_file} && last_send_snapshot=$(cat ${last_send_snapshot_file}) || last_send_snapshot=""
+			if [ "${last_snapshot}" != "${pool}@${last_send_snapshot}" ]; then
+				test -n "${last_send_snapshot}" && incremental_opt="-I ${pool}@${last_send_snapshot}"
+				echo zfs send -n -vR ${incremental_opt} ${last_snapshot}
+				zfs send -n -vR ${incremental_opt} ${last_snapshot} 2>&1 | \
+						tail -1 | \
+						tee /var/tmp/backup-${backup_host}-$$.log
+				if [ -n "${BACKUP_PATH}" -a -d "${BACKUP_PATH}" ]; then
+					zfs send -vR ${incremental_opt} ${last_snapshot} | \
+							zstd -v -6 --long > ${BACKUP_PATH}/${pool}-${backup_host}-${last_snapshot}.zstd | \
+							tee -a /var/tmp/backup-${backup_host}-$$.log
+					echo "Stored backup in ${BACKUP_PATH}/${pool}-${backup_host}-${last_snapshot}.zstd" 
+					echo "rsync this to the other server and use as input"
+				else
+					zfs send -vR ${incremental_opt} ${last_snapshot} | \
+							zstd -v -6 --long | \
+							ssh ${backup_host} $pool | \
+							tee -a /var/tmp/backup-${backup_host}-$$.log
+				fi
+				status=$(tail -1 /var/tmp/backup-${backup_host}-$$.log)
+				echo ${last_snapshot} | sed -e "s/${pool}@//"
+				if [ "${status}" = "OK" ]; then
+					echo ${last_snapshot} | sed -e "s/${pool}@//" >  ${last_send_snapshot_file}
+					mv /var/tmp/backup-${backup_host}-$$.log /var/log/zfs-backup-${backup_host}-${pool}-$(date +%a).log
+				fi
+			else
+				echo "Backup for $pool up to date for ${backup_host}"
+			fi
+		done
+	else
+		echo "WARNING: $backup_host not reachable over IPv6"
+	fi
+done