refactor(borgbackup): implement shared staging with defense-in-depth
Major improvements to borgbackup configuration for better reliability and
maintainability:
**Shared staging directory:**
- Use single /btrfs-subvolumes directory (was /subvolumes-{onsite,offsite})
- Eliminates redundant path suffixes in archive structure
- Archive paths now semantic: /btrfs-subvolumes/srv-forgejo clearly indicates
BTRFS subvolume content without redundant backup job metadata
**Defense-in-depth protection:**
- Layer 1: Systemd ordering - offsite waits for onsite completion
- Layer 2: Self-healing preHook - auto-cleanup orphaned snapshots from
crashes/power loss
- Prevents cascading failures from race conditions or abnormal terminations
**Code quality improvements:**
- Extract subvolume lists to reduce duplication (DRY principle)
- Add /* sh */ syntax hints for proper editor highlighting
- Silent operation for consistency with existing hooks
- Improved readability with clearer comments and formatting
- All lines ≤ 100 characters
**Timing:**
- Offsite: *-*-* 00:15:00 (daily at 12:15 AM, waits for onsite)
- Onsite: hourly (unchanged)
This commit is contained in:
parent
37924375a2
commit
15b4851e8e
GPG key ID:
8563E358D4E8040E
4 changed files with 134 additions and 74 deletions
|
|
@ -10,35 +10,44 @@
|
|||
|
||||
# Create staging directory before borg service starts
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /subvolumes-offsite 0755 root root -"
|
||||
"d /btrfs-subvolumes 0755 root root -"
|
||||
];
|
||||
|
||||
# Wait for onsite backup to complete before starting offsite
|
||||
systemd.services."borgbackup-job-offsite" = {
|
||||
after = ["borgbackup-job-onsite.service"];
|
||||
};
|
||||
|
||||
services.borgbackup.jobs."offsite" = {
|
||||
# Allow writing to staging directory
|
||||
readWritePaths = [ "/subvolumes-offsite" ];
|
||||
readWritePaths = [ "/btrfs-subvolumes" ];
|
||||
|
||||
# Create staging snapshots before backup (independent from onsite)
|
||||
preHook = ''
|
||||
# Create read-only staging snapshots for home directory
|
||||
preHook = /* sh */ ''
|
||||
# Clean up orphaned snapshots from failed runs (crash/power loss)
|
||||
[ -d "/btrfs-subvolumes/hm-sajenim" ] && \
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/btrfs-subvolumes/hm-sajenim" 2>/dev/null || true
|
||||
|
||||
# Create read-only BTRFS snapshot for backup
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r \
|
||||
"/home/sajenim" "/subvolumes-offsite/hm-sajenim"
|
||||
"/home/sajenim" "/btrfs-subvolumes/hm-sajenim"
|
||||
'';
|
||||
|
||||
# Backup explicit home directories and persistent files
|
||||
paths = [
|
||||
# Home directories (valuable user data only)
|
||||
"/subvolumes-offsite/hm-sajenim/Documents"
|
||||
"/subvolumes-offsite/hm-sajenim/Pictures"
|
||||
"/subvolumes-offsite/hm-sajenim/Videos"
|
||||
"/subvolumes-offsite/hm-sajenim/Music"
|
||||
"/subvolumes-offsite/hm-sajenim/Downloads"
|
||||
"/subvolumes-offsite/hm-sajenim/Academics"
|
||||
"/subvolumes-offsite/hm-sajenim/Notes"
|
||||
"/subvolumes-offsite/hm-sajenim/Library"
|
||||
"/btrfs-subvolumes/hm-sajenim/Documents"
|
||||
"/btrfs-subvolumes/hm-sajenim/Pictures"
|
||||
"/btrfs-subvolumes/hm-sajenim/Videos"
|
||||
"/btrfs-subvolumes/hm-sajenim/Music"
|
||||
"/btrfs-subvolumes/hm-sajenim/Downloads"
|
||||
"/btrfs-subvolumes/hm-sajenim/Academics"
|
||||
"/btrfs-subvolumes/hm-sajenim/Notes"
|
||||
"/btrfs-subvolumes/hm-sajenim/Library"
|
||||
|
||||
# Dotfiles (critical user configuration)
|
||||
"/subvolumes-offsite/hm-sajenim/.ssh"
|
||||
"/subvolumes-offsite/hm-sajenim/.gnupg"
|
||||
"/btrfs-subvolumes/hm-sajenim/.ssh"
|
||||
"/btrfs-subvolumes/hm-sajenim/.gnupg"
|
||||
|
||||
# Persistent files (actual storage location)
|
||||
"/persist/etc/machine-id"
|
||||
|
|
@ -54,10 +63,10 @@
|
|||
"/persist/etc/NetworkManager/system-connections"
|
||||
];
|
||||
|
||||
# Remove staging snapshots after backup completes
|
||||
postHook = ''
|
||||
postHook = /* sh */ ''
|
||||
# Clean up snapshots after successful backup
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/subvolumes-offsite/hm-sajenim"
|
||||
"/btrfs-subvolumes/hm-sajenim"
|
||||
'';
|
||||
|
||||
# Remote repository configuration
|
||||
|
|
@ -70,7 +79,7 @@
|
|||
|
||||
environment.BORG_RSH = "ssh -i /etc/ssh/ssh_host_ed25519_key";
|
||||
compression = "zstd,9";
|
||||
startAt = "daily";
|
||||
startAt = "*-*-* 00:15:00"; # Daily at 12:15 AM
|
||||
|
||||
# Ensure backup runs on next boot if system was asleep
|
||||
persistentTimer = true;
|
||||
|
|
|
|||
|
|
@ -8,35 +8,39 @@ in {
|
|||
|
||||
# Create staging directory before borg service starts
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /subvolumes-onsite 0755 root root -"
|
||||
"d /btrfs-subvolumes 0755 root root -"
|
||||
];
|
||||
|
||||
services.borgbackup.jobs."onsite" = {
|
||||
# Allow writing to staging directory
|
||||
readWritePaths = [ "/subvolumes-onsite" ];
|
||||
readWritePaths = [ "/btrfs-subvolumes" ];
|
||||
|
||||
# Create staging snapshots before backup (independent from offsite)
|
||||
preHook = ''
|
||||
# Create read-only staging snapshots for home directory
|
||||
preHook = /* sh */ ''
|
||||
# Clean up orphaned snapshots from failed runs (crash/power loss)
|
||||
[ -d "/btrfs-subvolumes/hm-sajenim" ] && \
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/btrfs-subvolumes/hm-sajenim" 2>/dev/null || true
|
||||
|
||||
# Create read-only BTRFS snapshot for backup
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r \
|
||||
"/home/sajenim" "/subvolumes-onsite/hm-sajenim"
|
||||
"/home/sajenim" "/btrfs-subvolumes/hm-sajenim"
|
||||
'';
|
||||
|
||||
# Backup explicit home directories and persistent files
|
||||
paths = [
|
||||
# Home directories (valuable user data only)
|
||||
"/subvolumes-onsite/hm-sajenim/Documents"
|
||||
"/subvolumes-onsite/hm-sajenim/Pictures"
|
||||
"/subvolumes-onsite/hm-sajenim/Videos"
|
||||
"/subvolumes-onsite/hm-sajenim/Music"
|
||||
"/subvolumes-onsite/hm-sajenim/Downloads"
|
||||
"/subvolumes-onsite/hm-sajenim/Academics"
|
||||
"/subvolumes-onsite/hm-sajenim/Notes"
|
||||
"/subvolumes-onsite/hm-sajenim/Library"
|
||||
"/btrfs-subvolumes/hm-sajenim/Documents"
|
||||
"/btrfs-subvolumes/hm-sajenim/Pictures"
|
||||
"/btrfs-subvolumes/hm-sajenim/Videos"
|
||||
"/btrfs-subvolumes/hm-sajenim/Music"
|
||||
"/btrfs-subvolumes/hm-sajenim/Downloads"
|
||||
"/btrfs-subvolumes/hm-sajenim/Academics"
|
||||
"/btrfs-subvolumes/hm-sajenim/Notes"
|
||||
"/btrfs-subvolumes/hm-sajenim/Library"
|
||||
|
||||
# Dotfiles (critical user configuration)
|
||||
"/subvolumes-onsite/hm-sajenim/.ssh"
|
||||
"/subvolumes-onsite/hm-sajenim/.gnupg"
|
||||
"/btrfs-subvolumes/hm-sajenim/.ssh"
|
||||
"/btrfs-subvolumes/hm-sajenim/.gnupg"
|
||||
|
||||
# Persistent files (actual storage location)
|
||||
"/persist/etc/machine-id"
|
||||
|
|
@ -52,10 +56,10 @@ in {
|
|||
"/persist/etc/NetworkManager/system-connections"
|
||||
];
|
||||
|
||||
# Remove staging snapshots after backup completes
|
||||
postHook = ''
|
||||
postHook = /* sh */ ''
|
||||
# Clean up snapshots after successful backup
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/subvolumes-onsite/hm-sajenim"
|
||||
"/btrfs-subvolumes/hm-sajenim"
|
||||
'';
|
||||
|
||||
# Onsite repository configuration (backup to viridian over SSH)
|
||||
|
|
|
|||
|
|
@ -10,35 +10,53 @@
|
|||
|
||||
# Create staging directory before borg service starts
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /subvolumes-offsite 0755 root root -"
|
||||
"d /btrfs-subvolumes 0755 root root -"
|
||||
];
|
||||
|
||||
# Wait for onsite backup to complete before starting offsite
|
||||
systemd.services."borgbackup-job-offsite" = {
|
||||
after = ["borgbackup-job-onsite.service"];
|
||||
};
|
||||
|
||||
services.borgbackup.jobs."offsite" = {
|
||||
# Allow writing to staging directory
|
||||
readWritePaths = [ "/subvolumes-offsite" ];
|
||||
readWritePaths = [ "/btrfs-subvolumes" ];
|
||||
|
||||
# Create staging snapshots before backup (independent from onsite)
|
||||
preHook = ''
|
||||
# Create read-only staging snapshots for each service
|
||||
for subvol in srv-containers srv-forgejo srv-lighttpd srv-minecraft srv-opengist; do
|
||||
# Map subvolume names to actual paths
|
||||
preHook = let
|
||||
subvolumes = [
|
||||
"srv-containers"
|
||||
"srv-forgejo"
|
||||
"srv-lighttpd"
|
||||
"srv-minecraft"
|
||||
"srv-opengist"
|
||||
];
|
||||
in /* sh */ ''
|
||||
# Clean up orphaned snapshots from failed runs (crash/power loss)
|
||||
for subvol in ${toString subvolumes}; do
|
||||
[ -d "/btrfs-subvolumes/$subvol" ] && \
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/btrfs-subvolumes/$subvol" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Create read-only BTRFS snapshots for backup
|
||||
for subvol in ${toString subvolumes}; do
|
||||
case "$subvol" in
|
||||
srv-containers) src="/srv/multimedia/containers" ;;
|
||||
srv-*) src="/srv/''${subvol#srv-}" ;;
|
||||
esac
|
||||
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r \
|
||||
"$src" "/subvolumes-offsite/$subvol"
|
||||
"$src" "/btrfs-subvolumes/$subvol"
|
||||
done
|
||||
'';
|
||||
|
||||
# Backup staging snapshots and explicit persistent files
|
||||
paths = [
|
||||
"/subvolumes-offsite/srv-containers"
|
||||
"/subvolumes-offsite/srv-forgejo"
|
||||
"/subvolumes-offsite/srv-lighttpd"
|
||||
"/subvolumes-offsite/srv-minecraft"
|
||||
"/subvolumes-offsite/srv-opengist"
|
||||
"/btrfs-subvolumes/srv-containers"
|
||||
"/btrfs-subvolumes/srv-forgejo"
|
||||
"/btrfs-subvolumes/srv-lighttpd"
|
||||
"/btrfs-subvolumes/srv-minecraft"
|
||||
"/btrfs-subvolumes/srv-opengist"
|
||||
|
||||
# Persistent files (actual storage location)
|
||||
"/persist/etc/machine-id"
|
||||
|
|
@ -54,11 +72,19 @@
|
|||
"/persist/etc/NetworkManager/system-connections"
|
||||
];
|
||||
|
||||
# Remove staging snapshots after backup completes
|
||||
postHook = ''
|
||||
for subvol in srv-containers srv-forgejo srv-lighttpd srv-minecraft srv-opengist; do
|
||||
postHook = let
|
||||
subvolumes = [
|
||||
"srv-containers"
|
||||
"srv-forgejo"
|
||||
"srv-lighttpd"
|
||||
"srv-minecraft"
|
||||
"srv-opengist"
|
||||
];
|
||||
in /* sh */ ''
|
||||
# Clean up snapshots after successful backup
|
||||
for subvol in ${toString subvolumes}; do
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/subvolumes-offsite/$subvol"
|
||||
"/btrfs-subvolumes/$subvol"
|
||||
done
|
||||
'';
|
||||
|
||||
|
|
@ -72,7 +98,7 @@
|
|||
|
||||
environment.BORG_RSH = "ssh -i /etc/ssh/ssh_host_ed25519_key";
|
||||
compression = "zstd,9";
|
||||
startAt = "daily";
|
||||
startAt = "*-*-* 00:15:00"; # Daily at 12:15 AM
|
||||
|
||||
# Ensure backup runs on next boot if system was asleep
|
||||
persistentTimer = true;
|
||||
|
|
|
|||
|
|
@ -17,35 +17,48 @@ in {
|
|||
|
||||
# Create staging directory before borg service starts
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /subvolumes-onsite 0755 root root -"
|
||||
"d /btrfs-subvolumes 0755 root root -"
|
||||
];
|
||||
|
||||
services.borgbackup.jobs."onsite" = {
|
||||
# Allow writing to staging directory
|
||||
readWritePaths = [ "/subvolumes-onsite" ];
|
||||
readWritePaths = [ "/btrfs-subvolumes" ];
|
||||
|
||||
# Create staging snapshots before backup (independent from offsite)
|
||||
preHook = ''
|
||||
# Create read-only staging snapshots for each service
|
||||
for subvol in srv-containers srv-forgejo srv-lighttpd srv-minecraft srv-opengist; do
|
||||
# Map subvolume names to actual paths
|
||||
preHook = let
|
||||
subvolumes = [
|
||||
"srv-containers"
|
||||
"srv-forgejo"
|
||||
"srv-lighttpd"
|
||||
"srv-minecraft"
|
||||
"srv-opengist"
|
||||
];
|
||||
in /* sh */ ''
|
||||
# Clean up orphaned snapshots from failed runs (crash/power loss)
|
||||
for subvol in ${toString subvolumes}; do
|
||||
[ -d "/btrfs-subvolumes/$subvol" ] && \
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/btrfs-subvolumes/$subvol" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Create read-only BTRFS snapshots for backup
|
||||
for subvol in ${toString subvolumes}; do
|
||||
case "$subvol" in
|
||||
srv-containers) src="/srv/multimedia/containers" ;;
|
||||
srv-*) src="/srv/''${subvol#srv-}" ;;
|
||||
esac
|
||||
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r \
|
||||
"$src" "/subvolumes-onsite/$subvol"
|
||||
"$src" "/btrfs-subvolumes/$subvol"
|
||||
done
|
||||
'';
|
||||
|
||||
# Backup staging snapshots and explicit persistent files
|
||||
paths = [
|
||||
"/subvolumes-onsite/srv-containers"
|
||||
"/subvolumes-onsite/srv-forgejo"
|
||||
"/subvolumes-onsite/srv-lighttpd"
|
||||
"/subvolumes-onsite/srv-minecraft"
|
||||
"/subvolumes-onsite/srv-opengist"
|
||||
"/btrfs-subvolumes/srv-containers"
|
||||
"/btrfs-subvolumes/srv-forgejo"
|
||||
"/btrfs-subvolumes/srv-lighttpd"
|
||||
"/btrfs-subvolumes/srv-minecraft"
|
||||
"/btrfs-subvolumes/srv-opengist"
|
||||
|
||||
# Persistent files (actual storage location)
|
||||
"/persist/etc/machine-id"
|
||||
|
|
@ -61,11 +74,19 @@ in {
|
|||
"/persist/etc/NetworkManager/system-connections"
|
||||
];
|
||||
|
||||
# Remove staging snapshots after backup completes
|
||||
postHook = ''
|
||||
for subvol in srv-containers srv-forgejo srv-lighttpd srv-minecraft srv-opengist; do
|
||||
postHook = let
|
||||
subvolumes = [
|
||||
"srv-containers"
|
||||
"srv-forgejo"
|
||||
"srv-lighttpd"
|
||||
"srv-minecraft"
|
||||
"srv-opengist"
|
||||
];
|
||||
in /* sh */ ''
|
||||
# Clean up snapshots after successful backup
|
||||
for subvol in ${toString subvolumes}; do
|
||||
${pkgs.btrfs-progs}/bin/btrfs subvolume delete \
|
||||
"/subvolumes-onsite/$subvol"
|
||||
"/btrfs-subvolumes/$subvol"
|
||||
done
|
||||
'';
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue