diff --git a/.gitignore b/.gitignore index 3730a4cdb5e8863782ef22a4418dbb856adbe5e1..0279560e9a998ac839a707e1ee7b0b0e3cff2dd2 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,8 @@ handlers/pgsql handlers/pgsql.helper handlers/rdiff handlers/rdiff.helper +handlers/restic +handlers/restic.helper handlers/rsync handlers/sh handlers/svn diff --git a/AUTHORS b/AUTHORS index c3d48b6fe64b691303a957ca71a40a00467877fb..a8dec1abec0abba5e1e26b7b19bdbf9cb1c81f6e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,7 +7,7 @@ micah@riseup.net -- debian package, vserver support, bug fixes stefani@riseup.net -- makecd handler, man pages intrigeri@boum.org -- dup handler, pgsql handler, vserver support, bug fixes Charles Lepple -- trac handler -Petr Kl�ma <petr.klima@madeta-group.cz> -- autotools, RPM support and sys checks +Petr Kl�ma <petr.klima@madeta-group.cz> -- autotools, RPM support and sys checks paulv@bikkel.org -- rsnap handler Robert Napier -- improved RPM build rhatto -- rub handler and patches @@ -65,3 +65,6 @@ David Gasaway <dave@gasaway.org> -- Fixes for configuration files without suffix Hugh Nowlan <nosmo@nosmo.me> -- dup check for archive dir Lyz <lyz@riseup.net> -- sys support for LUKS in disk partitions Glandos <bugs-0xacab@antipoul.fr> -- sys excludes zram devices +Nicolas Karolak <nicolas@karolak.fr> -- Add restic support +Derek Laventure -- Add restic helper +Colan Schwartz -- Fix restic options handler diff --git a/ChangeLog b/ChangeLog index 3adc1dd3264614c1bb4f689eede2afd6978a7b4f..ce6b06496dea50feaebe5fd2a8ca063d46488df1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ version 1.1.0 -- June 29, 2018 . Add initial support for the borgbackup program Thanks to Ben <ben@wainei.net> and Thomas Preissler <thomas@preissler.co.uk> for contributing patches + restic: + . Add support for restic. dup: · Fix symmetric encryption Thanks to Matthijs Wensveen <matthijs.wensveen@gmail.com> for diff --git a/INSTALL.md b/INSTALL.md index 4ea811273161000a9485250a4035cda63d43978f..175fe1fd9c42ea6f64b8991be11d45d359400aba 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -15,7 +15,7 @@ Requirements: Recommended: - rdiff-backup duplicity rsync borgbackup gzip hwinfo sfdisk cryptsetup flashrom hwinfo + borgbackup cryptsetup duplicity flashrom gzip hwinfo rdiff-backup restic rsync sfdisk To install backupninja, simply do the following: diff --git a/README.md b/README.md index 4de6ae8d89ad6e4bdf3ed4f9e77fc4f8143ce098..8867041dc382a2910b44e276b74916e798730835 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ file in `/etc/backup.d` according to the file's suffix: - `.sh`: run this file as a shell script. - `.rdiff`: filesystem backup (using rdiff-backup) + - `.restic`: filesystem backup (using restic) - `.dup`: filesystem backup (using duplicity) - `.borg`: filesystem backup (using borg) - `.mysql`: backup mysql databases diff --git a/Vagrantfile b/Vagrantfile index fc64ec07b72ca93da4a15232c4bacb6ff1085c49..eb824cc43f9fd5d7851b0eb61b20651463655848 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -27,6 +27,7 @@ Vagrant.configure("2") do |config| systemctl reload sshd echo -e "vagrant\nvagrant" | passwd vagrant chown vagrant: /var/backups + wget -q https://github.com/restic/rest-server/releases/download/v0.10.0/rest-server_0.10.0_linux_amd64.tar.gz -O - | tar -xz -C /usr/local/bin --strip-components=1 SHELL end diff --git a/examples/Makefile.am b/examples/Makefile.am index c33a8fbe66d6f195f0f0258ef92e4efdce318c39..5a5c48261e53388b2a9230767e9ca06cf9514782 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -1,7 +1,7 @@ EXAMPLES = example.borg example.dup example.maildir example.makecd \ example.mysql example.pgsql example.rdiff example.rsync \ - example.sh example.svn example.sys example.trac + example.sh example.svn example.sys example.trac example.restic EXTRA_DIST = $(EXAMPLES) diff --git a/examples/example.restic b/examples/example.restic new file mode 100644 index 0000000000000000000000000000000000000000..9fbb124d3eb5974f347858f04d5021aa7fbf6d85 --- /dev/null +++ b/examples/example.restic @@ -0,0 +1,244 @@ +# +# restic handler example file +# +# Mandatory options are un-commented with suggested values +# Other options are commented out with their default values +# + +# when = everyday at 01 + +[general] + +# Create a new backup of files and/or directories [yes/no] +run_backup = yes + +# Remove snapshots from the repository [yes/no] +#run_forget = no + +# Check the repository for errors [yes/no] +#run_check = no + +# Remove unneeded data from the repository [yes/no] +#run_prune = no + +# Retry to run the command [integer] +#retry_run = 1 + +# Seconds to wait between each retry attempts [integer] +#retry_wait = 5 + +# Repository to backup to or restore from [path] +repository = /mnt/backup + +# Repository password [string] +password = secret + +# File to load root certificates from (default: use system certificates) [path] +#cacert = + +# Set the cache directory [path] +#cache_dir = + +# Auto remove old cache directories [yes/no] +#cleanup_cache = + +# Set output mode to JSON for commands that support it [yes/no] +#json = + +# Limits downloads to a maximum rate in KiB/s. (default: unlimited) [integer] +#limit_download = + +# Limits uploads to a maximum rate in KiB/s. (default: unlimited) [integer] +#limit_upload = + +# Do not use a local cache [yes/no] +#no_cache = + +# Do not lock the repo, this allows some operations on read-only repos [yes/no] +#no_lock = + +# Set extended option (can be specified multiple times) [key=value] +#option = + +# Read the repository password from a file [path] +#password_file = + +# Do not output comprehensive progress report [yes/no] +#quiet = + +# Path to a file containing PEM encoded TLS client certificate and private key [path] +#tls_client_cert = + +## Adjust process scheduling priority. When set to to an integer value between +## -19 and 20, the backup command will be started with the desired scheduling +## priority. A positive integer indicates lower priority, the default priority +## being 0. [integer] +## +## Default: +# nicelevel = + +## Adjust I/O scheduling class and priority. When set to to an integer value +## between 0 and 7, the backup command will be started with the best-effort +## class and desired priority level. [integer] +## +## See the ionice(1) man page for more details about available levels. +## +## Default: +# ionicelevel = + +[s3] + +#aws_access_key_id = + +#aws_secret_access_key = + +#aws_session_token = + +[swift] + +#os_auth_url = + +#os_tenant_id = + +#os_tenant_name = + +#os_username = + +#os_password = + +#os_region_name = + +[b2] + +#b2_account_id = + +#b2_account_key = + +[azure] + +#azure_account_name = + +#azure_account_key = + +[gs] + +#google_project_id = + +#google_application_credentials = + +[backup] + +# Initialize the repository if it doesn't exist [yes/no] +#init = yes + +# Files adn directories to backup (can be specified multiple times) [path] +#include = / + +# Exclude a pattern (can be specified multiple times) [pattern] +#exclude = /dev +#exclude = /lost+found +#exclude = /media +#exclude = /mnt +#exclude = /proc +#exclude = /run +#exclude = /sys +#exclude = /tmp +#exclude = /var/cache +#exclude = /var/lock +#exclude = /var/spool +#exclude = /var/run +#exclude = /var/tmp + +# Excludes cache directories that are marked with a CACHEDIR.TAG file [yes/no] +#exclude_caches = + +# Read exclude patterns from a file (can be specified multiple times) [path] +#exclude_file = + +# Takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times) [stringArray] +#exclude_if_present = + +# Read the files to backup from file (can be combined with file args) [path] +#files_from = + +# Force re-reading the target files/directories (overrides the "parent" flag) [yes/no] +#force = + +# Set the hostname for the snapshot manually. To prevent an expensive rescan use the "parent" flag [string] +#hostname = + +# Exclude other file systems [yes/no] +#one_file_system = + +# Use this parent snapshot (default: last snapshot in the repo that has the same target files/directories) [string] +#parent = + +# Add a tag for the new snapshot (can be specified multiple times) [string] +#tag = + +# Time of the backup (ex. '2012-11-01 22:08:41') (default: now) [string] +#time = + +# Store the atime for all files and directories [yes/no] +#with_atime = + +[forget] + +# Keep the last n snapshots [integer] +#keep_last = 7 + +# Keep the last n hourly snapshots [integer] +#keep_hourly = + +# Keep the last n daily snapshots [integer] +#keep_daily = + +# Keep the last n weekly snapshots [integer] +#keep_weekly = + +# Keep the last n monthly snapshots [integer] +#keep_monthly = + +# Keep the last n yearly snapshots [integer] +#keep_yearly = + +# Keep snapshots that were created within duration before the newest (e.g. 1y5m7d) [string] +#keep_within = + +# Keep snapshots with this tag (can be specified multiple times) [string] +#keep_tag = + +# Only consider snapshots with the given host [string] +#host = + +# Only consider snapshots which include this tag (can be specified multiple times) [string] +#tag = + +# Only consider snapshots which include this (absolute) path (can be specified multiple times) [string] +#path = + +# Use compact format [yes/no] +#compact = + +# String for grouping snapshots by host,paths,tags (default "host,paths") [string] +#group_by = + +# Do not delete anything, just print what would be done [yes/no] +dry_run = + +# Automatically run the 'prune' command if snapshots have been removed [yes/no] +#prune = + +[check] + +# Find unused blobs [yes/no] +#check_unused = + +# Read all data blobs [yes/no] +#read_data = + +# Read subset of data packs [string] +#read_data_subset = + +# Use the cache [yes/no] +#with_cache = diff --git a/handlers/Makefile.am b/handlers/Makefile.am index 43ba5d18abeea50185448cb935d9704dc4a0b937..65275e5738c0605dd59e9788788d1b8c2d551337 100644 --- a/handlers/Makefile.am +++ b/handlers/Makefile.am @@ -1,11 +1,12 @@ HANDLERS = borg borg.helper dup dup.helper maildir makecd \ - makecd.helper mysql mysql.helper pgsql pgsql.helper rdiff \ - rdiff.helper rsync sh svn sys sys.helper trac tar tar.helper + makecd.helper mysql mysql.helper pgsql pgsql.helper restic \ + restic.helper rdiff rdiff.helper rsync sh svn sys sys.helper trac tar tar.helper DIST_HANDLERS = borg.in borg.helper.in dup.in dup.helper.in maildir.in makecd.in \ makecd.helper.in mysql.in mysql.helper.in pgsql.in pgsql.helper.in rdiff.in \ - rdiff.helper.in rsync.in sh.in svn.in sys.in sys.helper.in trac.in tar.in tar.helper.in wget + rdiff.helper.in restic.in restic.helper.in rsync.in sh.in svn.in sys.in sys.helper.in \ + trac.in tar.in tar.helper.in wget CLEANFILES = $(HANDLERS) @@ -78,6 +79,14 @@ rdiff.helper: $(srcdir)/rdiff.helper.in rm -f rdiff.helper $(edit) $(srcdir)/rdiff.helper.in > rdiff.helper +restic: $(srcdir)/restic.in + rm -f restic + $(edit) $(srcdir)/restic.in > restic + +restic.helper: $(srcdir)/restic.helper.in + rm -f restic.helper + $(edit) $(srcdir)/restic.helper.in > restic.helper + rsync: $(srcdir)/rsync.in rm -f rsync $(edit) $(srcdir)/rsync.in > rsync diff --git a/handlers/restic.helper.in b/handlers/restic.helper.in new file mode 100644 index 0000000000000000000000000000000000000000..1e40cd04655c558a0b6a19cdda4d3f05c8bbeec8 --- /dev/null +++ b/handlers/restic.helper.in @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# -*- mode: sh; sh-basic-offset: 2; indent-tabs-mode: nil; -*- +# vim: set filetype=sh sw=2 sts=2 expandtab autoindent: +# +# restic helper for backupninja +# + +HELPERS="$HELPERS restic:fast_secure_efficient_backup" + +declare -a restic_includes +declare -a restic_excludes + +# FUNCTIONS + +function do_restic_repository() { + REPLY= + while [ -z "$REPLY" -o -z "$restic_repository" ]; do + inputBox "$restic_title - Repository" "Enter Repository (eg. rclone:remote:bucket):" "$restic_repository" + [ $? = 0 ] || return 1 + restic_repository="$REPLY" + done +} + +function do_restic_password_file() { + REPLY= + while [ -z "$REPLY" -o -z "$restic_password_file" ]; do + inputBox "$restic_title - Password File" "Enter password-file (eg. /etc/restic.passwd) containing repository password:" "$restic_password_file" + [ $? = 0 ] || return 1 + restic_password_file="$REPLY" + done +} + +function do_restic_general() { + # set restic_repository + do_restic_repository ; [ $? = 0 ] || return 1 + + # set restic_password_file + do_restic_password_file ; [ $? = 0 ] || return 1 + + _gen_done="(DONE)" + setDefault back +} + +function do_restic_includes() { + set -o noglob + # choose the files to backup + REPLY= + while [ -z "$REPLY" ]; do + formBegin "$restic_title - host system: includes" + for ((i=0; i < ${#restic_includes[@]} ; i++)); do + formItem include ${restic_includes[$i]} + done + formItem include + formItem include + formItem include + formItem include + formItem include + formItem include + formItem include + formItem include + formDisplay + [ $? = 0 ] || return + unset restic_includes + restic_includes=($REPLY) + done + set +o noglob +} + +function do_restic_excludes() { + set -o noglob + formBegin "$restic_title: host system: excludes" + for ((i=0; i < ${#restic_excludes[@]} ; i++)) + do + formItem exclude ${restic_excludes[$i]} + done + formItem exclude + formItem exclude + formItem exclude + formItem exclude + formItem exclude + formItem exclude + formItem exclude + formItem exclude + formDisplay + [ $? = 0 ] || return + unset restic_excludes + restic_excludes=($REPLY) + set +o noglob +} + +function do_restic_backup() { + do_restic_includes + [ $? = 0 ] || return 1 + + do_restic_excludes + [ $? = 0 ] || return 1 + + _back_done="(DONE)" + setDefault finish +} + + +function do_restic_finish() { + get_next_filename $configdirectory/90.restic + + cat > $next_filename <<EOF +## for more options see +## - example.restic +## - $restic_docs + +# Minimal output +[general] +run_backup = yes +EOF + + echo "repository = $restic_repository" >> $next_filename + echo "password_file = $restic_password_file" >> $next_filename + + cat >> $next_filename <<EOF + +[backup] +EOF + + ## includes ## + set -o noglob + for ((i=0; i < ${#restic_includes[@]} ; i++)); do + echo "include = ${restic_includes[$i]}" >> $next_filename + done + set +o noglob + + ## excludes ## + set -o noglob + for ((i=0; i < ${#restic_excludes[@]} ; i++)); do + echo exclude = ${restic_excludes[$i]} >> $next_filename + done + set +o noglob + + chmod 600 $next_filename +} + +function restic_main_menu() { + while true; do + genitem="General Restic settings $_gen_done" + backitem="Backup settings $_back_done" + menuBox "$restic_title" "choose a step:" \ + gen "$genitem" \ + back "$backitem" \ + finish "finish and create config file" + [ $? = 0 ] || return + result="$REPLY" + case "$result" in + "gen") do_restic_general;; + "back") do_restic_backup;; + "finish") + if [[ "$_gen_done$_back_done" != "(DONE)(DONE)" ]]; then + msgBox "$restic_title" "You cannot create the config file until all steps are completed." + else + do_restic_finish + return + fi + ;; + esac + done +} + +function restic_wizard { + require_packages rclone + + # global variables + restic_title="restic action wizard" + restic_docs="https://restic.readthedocs.io/en/latest" + + _gen_done= + _back_done= + + restic_main_menu +} diff --git a/handlers/restic.in b/handlers/restic.in new file mode 100644 index 0000000000000000000000000000000000000000..664547cd3e990c839b94d127b85497453e4682e9 --- /dev/null +++ b/handlers/restic.in @@ -0,0 +1,583 @@ +# -*- mode: sh; sh-basic-offset: 2; indent-tabs-mode: nil; -*- +# vim: set filetype=sh sw=2 sts=2 expandtab autoindent: +# +# restic script for backupninja +# + +### GETCONF ################################################################### + +setsection general + +getconf nicelevel +getconf ionicelevel + +getconf run_backup "no" +getconf run_forget "no" +getconf run_check "no" +getconf run_prune "no" + +getconf cacert +getconf cache_dir +getconf cleanup_cache +getconf json +getconf limit_download +getconf limit_upload +getconf no_cache +getconf no_lock +getconf option +getconf password +getconf password_file +getconf quiet +getconf repository +getconf tls_client_cert + +setsection s3 + +getconf aws_access_key_id +getconf aws_secret_access_key +getconf aws_session_token + +setsection swift + +getconf os_auth_url +getconf os_tenant_id +getconf os_tenant_name +getconf os_username +getconf os_password +getconf os_region_name + +setsection b2 + +getconf b2_account_id +getconf b2_account_key + +setsection azure + +getconf azure_account_name +getconf azure_account_key + +setsection gs + +getconf google_project_id +getconf google_application_credentials + +# Check that the ionicelevel is valid +if [ -n "$nicelevel" ] && { [ "$nicelevel" -lt -20 ] || [ "$nicelevel" -gt 19 ]; }; then + fatal "The value of nicelevel is expected to be either empty or an integer from -20 to 19. Got: $nicelevel" +fi + +# Check that the ionicelevel is valid +if [ -n "$ionicelevel" ] && echo "$ionicelevel" | grep -vq "^[0-7]$"; then + fatal "The value of ionicelevel is expected to be either empty or an integer from 0 to 7. Got: $ionicelevel" +fi + +### HELPERS ################################################################### + +function export_debug { + export "$1"="$2" + debug "$1=${!1}" +} + +### PRE-COMMANDS ############################################################## + +[ -n "$nicelevel" ] && \ + precmd+="nice -n $nicelevel " + +[ -n "$ionicelevel" ] && \ + precmd+="ionice -c2 -n $ionicelevel " + +### GLOBAL OPTIONS ############################################################ + +[ -z "$repository" ] && \ + fatal "The repository option must be set." + +[ -z "$password" ] && [ -z "$password_file" ] && \ + fatal "The password must be set by option 'password' or 'password_file'." + +[ -n "$repository" ] && \ + cmd_global_options+="--repo $repository " + +[ -n "$password" ] && \ + export_debug RESTIC_PASSWORD "$password" + +[ -n "$password_file" ] && \ + cmd_global_options+="--password-file $password_file " + +[ -n "$cacert" ] && \ + cmd_global_options+="--cacert $cacert " + +[ -n "$cache_dir" ] && \ + cmd_global_options+="--cache-dir $cache_dir " + +[ -n "$cleanup_cache" ] && \ + cmd_global_options+="--cleanup-cache " + +[ -n "$json" ] && \ + cmd_global_options+="--json " + +[ -n "$limit_download" ] && \ + cmd_global_options+="--limit-download $limit_download " + +[ -n "$limit_upload" ] && \ + cmd_global_options+="--limit-upload $limit_upload " + +[ -n "$no_cache" ] && \ + cmd_global_options+="--no-cache " + +[ -n "$no_lock" ] && \ + cmd_global_options+="--no-lock " + +[ -n "$option" ] && \ + cmd_global_options+="$(for i in "${option[@]}"; do echo "--option $i "; done)" + +[ -n "$quiet" ] && \ + cmd_global_options+="--quiet " + +[ -n "$tls_client_cert" ] && \ + cmd_global_options+="--tls-client-cert $tls_client_cert " + +### REPOSITORY ################################################################ + +# Amazon S3 repository +if [ "$(echo "$repository" | @AWK@ -F ':' '{print $1}')" == "s3" ]; then + + ( [ -z "$aws_access_key_id" ] || [ -z "$aws_secret_access_key" ] ) && \ + fatal "Missing some S3 credentials." + + export_debug AWS_ACCESS_KEY_ID "$aws_access_key_id" + export_debug AWS_SECRET_ACCESS_KEY "$aws_secret_access_key" + [ -n "$aws_session_token" ] && \ + export_debug AWS_SESSION_TOKEN "$aws_session_token" + +fi + +# OpenStack Swift repository +if [ "$(echo "$repository" | @AWK@ -F ':' '{print $1}')" == "swift" ]; then + + ( + [ -z "$os_auth_url" ] || [ -z "$os_tenant_id" ] || [ -z "$os_tenant_name" ] || \ + [ -z "$os_username" ] || [ -z "$os_password" ] || [ -z "$os_region_name" ] + ) && \ + fatal "Missing some Swift credentials." + + export_debug OS_AUTH_URL "$os_auth_url" + export_debug OS_TENANT_ID "$os_tenant_id" + export_debug OS_TENANT_NAME "$os_tenant_name" + export_debug OS_USERNAME "$os_username" + export_debug OS_PASSWORD "$os_password" + export_debug OS_REGION_NAME "$os_region_name" + +fi + +# Backblaze B2 repository +if [ "$(echo "$repository" | @AWK@ -F ':' '{print $1}')" == "b2" ]; then + + ( [ -z "$b2_account_id" ] || [ -z "$b2_account_key" ] ) && \ + fatal "Missing some B2 credentials." + + export_debug B2_ACCOUNT_ID "$b2_account_id" + export_debug B2_ACCOUNT_KEY "$b2_account_key" + +fi + +# Microsoft Azure Blob Storage repository +if [ "$(echo "$repository" | @AWK@ -F ':' '{print $1}')" == "azure" ]; then + + ( [ -z "$azure_account_name" ] || [ -z "$azure_account_key" ] ) && \ + fatal "Missing some Azure credentials." + + export_debug AZURE_ACCOUNT_NAME "$azure_account_name" + export_debug AZURE_ACCOUNT_KEY "$azure_account_key" + +fi + +# Google Cloud Storage repository +if [ "$(echo "$repository" | @AWK@ -F ':' '{print $1}')" == "gs" ]; then + + ( [ -z "$google_project_id" ] || [ -z "$google_application_credentials" ] ) && \ + fatal "Missing some Google credentials." + + export_debug GOOGLE_PROJECT_ID "$google_project_id" + export_debug GOOGLE_APPLICATION_CREDENTIALS "$google_application_credentials" + +fi + +### TEST ####################################################################### + +info "Attempting to connect to repository at ${repository}" + +cmd="restic snapshots" +execstr="${cmd} ${cmd_global_options//$'\n'}" + +debug "executing restic snapshots" +debug "$execstr" +output=$(eval $execstr 2>&1) +ret=$? + +if [ $ret -eq 0 ]; then + debug $output + info "Connected successfully." +else + setsection backup + getconf init yes + if [ "$init" = "yes" ]; then + debug $output + info "Unable to find a repository at ${repository}, will attempt to create one." + need_init="yes" + else + error $output + fatal "The specified repository is absent or unusable!" + fi +fi + +### INIT ####################################################################### + +if [ "$need_init" = "yes" ]; then + + cmd="restic init" + execstr="${cmd} ${cmd_global_options//$'\n'}" + + debug "executing restic init" + debug "$execstr" + + if [ $test -eq 1 ]; then + info "Test mode enabled, skipping restic init." + else + info "Initializing repository at $repository" + + output=$(eval $execstr 2>&1) + ret=$? + + if [ $ret -eq 0 ]; then + warning $output + warning "Repository has been initialized." + else + error $output + fatal "Unable to initialize repository, aborting!" + fi + fi + +fi + +### BACKUP ##################################################################### + +if [ "$run_backup" == "yes" ]; then + + setsection backup + + getconf include + getconf exclude + getconf exclude_caches + getconf exclude_file + getconf exclude_if_present + getconf files_from + getconf force + getconf hostname + getconf one_file_system + getconf parent + getconf tag + getconf time + getconf with_atime + + set -o noglob + SAVEIFS=$IFS + IFS=$(echo -en "\n\b") + + [ -z "$include" ] && [ -z "$files_from" ] && \ + fatal "No files or directories specified for backup." + + [ -n "$include" ] && \ + cmd_options+="$(for i in $include; do echo "'$i' "; done)" + + [ -n "$files_from" ] && \ + cmd_options+="$(for i in $files_from; do echo "--files-from '$i' "; done)" + + [ -d "$repository" ] && \ + cmd_options+="--exclude $repository " + + [ -n "$exclude" ] && \ + cmd_options+="$(for i in $exclude; do echo "--exclude '$i' "; done)" + + [ "$exclude_caches" == "yes" ] && \ + cmd_options+="--exclude-caches " + + [ -n "$exclude_file" ] && \ + cmd_options+="$(for i in $exclude_file; do echo "--exclude-file '$i' "; done)" + + [ -n "$exclude_if_present" ] && \ + cmd_options+="$(for i in $exclude_if_present; do echo "--exclude-if-present '$i' "; done)" + + [ "$force" == "yes" ] && \ + cmd_options+="--force " + + [ -n "$hostname" ] && \ + cmd_options+="--hostname $hostname " + + [ -n "$one_file_system" ] && \ + cmd_options+="--one-file-system " + + [ -n "$parent" ] && \ + cmd_options+="--parent $parent " + + [ -n "$tag" ] && \ + cmd_options+="$(for i in $tag; do echo "--tag='$i' "; done)" + + [ -n "$time" ] && \ + cmd_options+="--time $time " + + [ "$with_atime" == "yes" ] && \ + cmd_options+="--with_atime " + + IFS=$SAVEIFS + set +o noglob + + # format command + cmd="restic backup" + execstr="${precmd}${cmd} ${cmd_global_options//$'\n'}${cmd_options//$'\n'}" + + # debug + debug "executing restic backup" + debug "$execstr" + + # execute + if [ $test -eq 1 ]; then + info "Test mode enabled, skipping restic backup." + else + info "Creating new backup snapshot." + output=$(eval $execstr 2>&1) + ret=$? + if [ $ret -eq 0 ]; then + debug $output + info "Restic backup successful." + else + error $output + fatal "Restic backup failed." + fi + fi + + debug "Unsetting variables" + unset cmd_options + unset execstr + unset output + unset ret + +fi + +### FORGET ##################################################################### + +if [[ "$run_forget" == "yes" ]]; then + + setsection forget + + getconf keep_last "7" + getconf keep_hourly + getconf keep_daily + getconf keep_weekly + getconf keep_monthly + getconf keep_yearly + getconf keep_within + getconf keep_tag + getconf host + getconf tag + getconf path + getconf compact + getconf group_by + getconf dry_run + getconf prune + + set -o noglob + SAVEIFS=$IFS + IFS=$(echo -en "\n\b") + + [ -n "$keep_last" ] && \ + cmd_options+="--keep-last $keep_last " + + [ -n "$keep_hourly" ] && \ + cmd_options+="--keep-hourly $keep_hourly " + + [ -n "$keep_daily" ] && \ + cmd_options+="--keep-daily $keep_daily " + + [ -n "$keep_weekly" ] && \ + cmd_options+="--keep-weekly $keep_weekly " + + [ -n "$keep_monthly" ] && \ + cmd_options+="--keep-monthly $keep_monthly " + + [ -n "$keep_yearly" ] && \ + cmd_options+="--keep-yearly $keep_yearly " + + [ -n "$keep_within" ] && \ + cmd_options+="--keep-within $keep_within " + + [ -n "$keep_tag" ] && \ + cmd_options+="$(for i in $keep_tag; do echo "--keep-tag=$i "; done)" + + [ -n "$host" ] && \ + cmd_options+="--host $host " + + [ -n "$tag" ] && \ + cmd_options+="$(for i in $tag; do echo "--tag=$i "; done)" + + [ -n "$path" ] && \ + cmd_options+="$(for i in $path; do echo "--path=$i "; done)" + + [ -n "$compact" ] && \ + cmd_options+="--compact " + + [ -n "$group_by" ] && \ + cmd_options+="--group-by $group_by " + + [ -n "$dry_run" ] && \ + cmd_options+="--dry-run " + + [ -n "$prune" ] && \ + cmd_options+="--prune " + + IFS=$SAVEIFS + set +o noglob + + # format command + cmd="restic forget" + execstr="${precmd}${cmd} ${cmd_global_options//$'\n'}${cmd_options//$'\n'}" + + # debug + debug "executing restic forget" + debug "$execstr" + + # execute + if [ $test -eq 1 ]; then + info "Test mode enabled, skipping restic forget." + else + info "Removing old snapshots based on defined retention policy." + output=$(eval $execstr 2>&1) + ret=$? + if [ $ret -eq 0 ]; then + debug $output + info "Restic forget successful." + else + error $output + fatal "Restic forget failed." + fi + fi + + debug "Unsetting variables" + unset cmd_options + unset execstr + unset output + unset ret + +fi + +### PRUNE ###################################################################### + +if [ "$run_prune" == "yes" ]; then + + # format command + cmd="restic prune" + execstr="${precmd}${cmd} ${cmd_global_options//$'\n'}" + + # debug + debug "executing restic prune" + debug "$execstr" + + # execute + if [ $test -eq 1 ]; then + info "Test mode enabled, skipping restic prune." + else + info "Removing unreferenced data from repository." + output=$(eval $execstr 2>&1) + ret=$? + if [ $ret -eq 0 ]; then + debug $output + info "Restic prune successful." + else + error $output + fatal "Restic prune failed." + fi + fi + + debug "Unsetting variables" + unset execstr + unset output + unset ret + +fi + +### CHECK ###################################################################### + +if [ "$run_check" == "yes" ]; then + + setsection check + + getconf check_unused + getconf read_data + getconf read_data_subset + getconf with_cache + + [ -n "$check_unused" ] && \ + cmd_options+="--check-unused " + + [ -n "$read_data" ] && \ + cmd_options+="--read-data " + + [ -n "$read_data_subset" ] && \ + cmd_options+="--read-data-subset $read_data_subset " + + [ -n "$with_cache" ] && \ + cmd_options+="--with-cache " + + # format command + cmd="restic check" + execstr="${precmd}${cmd} ${cmd_global_options//$'\n'}${cmd_options//$'\n'}" + + # debug + debug "executing restic check" + debug "$execstr" + + # execute + if [ $test -eq 1 ]; then + info "Test mode enabled, skipping restic check." + else + info "Checking repository integrity and consistency." + output=$(eval $execstr 2>&1) + ret=$? + if [ $ret -eq 0 ]; then + debug $output + info "Restic check successful." + else + error $output + fatal "Restic check failed." + fi + fi + + debug "Unsetting variables" + unset cmd_options + unset execstr + unset output + unset ret + +fi + +### CLEAN UP ################################################################### + +debug "Unsetting environment variables" +unset RESTIC_PASSWORD +unset AWS_ACCESS_KEY_ID +unset AWS_SECRET_ACCESS_KEY +unset AWS_SESSION_TOKEN +unset OS_AUTH_URL +unset OS_TENANT_ID +unset OS_TENANT_NAME +unset OS_USERNAME +unset OS_PASSWORD +unset OS_REGION_NAME +unset B2_ACCOUNT_ID +unset B2_ACCOUNT_KEY +unset AZURE_ACCOUNT_NAME +unset AZURE_ACCOUNT_KEY +unset GOOGLE_PROJECT_ID +unset GOOGLE_APPLICATION_CREDENTIALS +unset cmd_global_options + +return 0 diff --git a/man/backup.d.5 b/man/backup.d.5 index b2e66a22378c91fee44c276230d2928bf8b85856..5178067178d7dc1e574f60cc69f82b0758d81e1d 100644 --- a/man/backup.d.5 +++ b/man/backup.d.5 @@ -31,6 +31,8 @@ To preform the actual backup actions, backupninja processes each action configur run this file as a shell script. .IP .rdiff backup action for rdiff-backup. +.IP .restic +backup action for restic. .IP .dup backup action for duplicity. .IP .borg diff --git a/test/common.bash b/test/common.bash index 7016e2004ac47cb631df92b18e697be8cbf7eead..28d85837185749a6d3bab02ecd37dc06f9e4edf9 100644 --- a/test/common.bash +++ b/test/common.bash @@ -141,6 +141,10 @@ remote_command() { ssh "${BN_REMOTEUSER}@${BN_REMOTEHOST}" "$1" } +remote_background_command() { + ssh -f "${BN_REMOTEUSER}@${BN_REMOTEHOST}" "$1" +} + # remove backup test artifacts cleanup_backups() { for c in "$@"; do diff --git a/test/restic.bats b/test/restic.bats new file mode 100644 index 0000000000000000000000000000000000000000..0ead5caa229bcb70b3a5949cafed28c1f01c8b9e --- /dev/null +++ b/test/restic.bats @@ -0,0 +1,219 @@ +load common + +begin_restic() { + if [ ! -d "$BN_SRCDIR" ]; then + apt -qq install debootstrap + debootstrap --variant=minbase testing "$BN_SRCDIR" + fi + + [ -x "$(which restic)" ] || apt -qq install restic + + remote_background_command "/usr/local/bin/rest-server --no-auth --path ${BN_BACKUPDIR}" +} + +setup_restic() { + export RESTICREPO="${BN_BACKUPDIR}/testrestic" + + cat << EOF > "${BATS_TMPDIR}/backup.d/test.restic" +when = manual + +[general] +nicelevel = +ionicelevel = +repository = ${RESTICREPO} +password = 123test +run_backup = yes + +[backup] +init = yes +include = ${BN_SRCDIR} +exclude = ${BN_SRCDIR}/var +EOF + + chmod 0640 "${BATS_TMPDIR}/backup.d/test.restic" + rm -rf /root/.cache/restic +} + +finish_restic() { + remote_command 'kill $(pgrep -u 1000 rest-server)' + cleanup_backups local remote + rm -rf /root/.cache/restic +} + +init_repo() { + if [ "$1" = "remote" ]; then + cleanup_backups remote + remote_command "restic init --repo ${RESTICREPO} <<<123test" + else + cleanup_backups local + restic init --repo "${RESTICREPO}" <<<123test + fi +} + +@test "check connection test, local repository" { + local badrepo="${BN_BACKUPDIR}foo/bar" + local goodrepo="${RESTICREPO}" + + # invalid repo + setconfig general repository "$badrepo" + setconfig backup init no + testaction + greplog "Info: Attempting to connect to repository at ${badrepo}$" + greplog "Fatal: The specified repository is absent or unusable!$" + + # valid repo + init_repo local + setconfig general repository "${goodrepo}" + setconfig backup init no + testaction + greplog "Info: Attempting to connect to repository at ${goodrepo}$" + greplog "Info: Connected successfully.$" +} + +@test "check connection test, rest repository" { + local badrepo="rest:http://foo@bar:8000/testrestic" + local okrepo="rest:http://${BN_REMOTEUSER}@${BN_REMOTEHOST}:8000/testrestic" + + # invalid repo + setconfig general repository "$badrepo" + setconfig backup init no + testaction + greplog "Info: Attempting to connect to repository at ${badrepo}$" + greplog "Fatal: The specified repository is absent or unusable!$" + + # valid repo + init_repo remote + setconfig general repository "${okrepo}" + setconfig backup init no + testaction + greplog "Info: Attempting to connect to repository at ${okrepo}$" + greplog "Info: Connected successfully.$" +} + +@test "check config parameter general/nicelevel" { + # nicelevel is 0 by default + delconfig general nicelevel + testaction + not_greplog 'Debug: executing restic backup$' '\bnice -n\b' + + # nicelevel is defined + setconfig general nicelevel -19 + testaction + greplog 'Debug: executing restic backup$' '\bnice -n -19\b' +} + +@test "check config parameter general/ionicelevel" { + # no ionice by default + delconfig general ionicelevel + testaction + not_greplog 'Debug: executing restic backup$' '\bionice -c2\b' + + # acceptable value + setconfig general ionicelevel 7 + testaction + greplog 'Debug: executing restic backup$' '\bionice -c2 -n 7\b' + + # unacceptable value + setconfig general ionicelevel 10 + testaction + greplog 'Fatal: The value of ionicelevel is expected to be either empty or an integer from 0 to 7. Got: 10$' +} + +@test "check config parameter backup/include" { + local cmd="restic backup --repo ${RESTICREPO}" + + # undefined, raises fatal error + delconfig backup include + testaction + greplog "Fatal: No files or directories specified for backup.$" + + # single value + setconfig backup include "$BN_SRCDIR" + testaction + greplog 'Debug: executing restic backup$' "\b${cmd} '${BN_SRCDIR}'\s" + + # mutliple values + setconfig_repeat backup include "$BN_SRCDIR" "/home" + testaction + greplog 'Debug: executing restic backup$' "\b${cmd} '${BN_SRCDIR}' '/home'\s" + + # with spaces + delconfig backup include + setconfig backup include "/home/foo/My Documents" + testaction + cat ${BATS_TMPDIR}/backup.d/test.restic + greplog 'Debug: executing restic backup$' "\b${cmd} '/home/foo/My Documents'\s" + + # with glob (though restic doesn't support it) + delconfig backup include + setconfig backup include "/etc/*" + testaction + cat ${BATS_TMPDIR}/backup.d/test.restic + greplog 'Debug: executing restic backup$' "\b${cmd} '/etc/\*'\s" +} + +@test "check config parameter backup/exclude" { + local cmd="restic backup --repo ${RESTICREPO} '${BN_SRCDIR}' --exclude ${RESTICREPO}" + + # undefined + delconfig backup exclude + testaction + greplog 'Debug: executing restic backup$' "${cmd}" + + # single value + setconfig backup exclude "${BN_SRCDIR}/var" + testaction + greplog 'Debug: executing restic backup$' "\b${cmd} --exclude '${BN_SRCDIR}/var'" + + # mutliple values + setconfig_repeat backup exclude "${BN_SRCDIR}/var" "/home" + testaction + greplog 'Debug: executing restic backup$' "\b${cmd} --exclude '${BN_SRCDIR}/var' --exclude '/home'" + + # with spaces + delconfig backup exclude + setconfig backup exclude "/home/foo/My Documents" + testaction + cat ${BATS_TMPDIR}/backup.d/test.restic + greplog 'Debug: executing restic backup$' "\b${cmd} --exclude '/home/foo/My Documents'" + + # with glob (though restic doesn't support it) + delconfig backup exclude + setconfig backup exclude "/etc/*" + testaction + cat ${BATS_TMPDIR}/backup.d/test.restic + greplog 'Debug: executing restic backup$' "\b${cmd} --exclude '/etc/\*'" +} + +@test "run local backup" { + init_repo + setconfig general run_backup yes + runaction + greplog 'Info: Restic backup successful.$' +} + +@test "run local forget" { + setconfig general run_forget yes + runaction + greplog 'Info: Restic forget successful.$' +} + +@test "run local prune" { + setconfig general run_prune yes + runaction + greplog 'Info: Restic prune successful.$' +} + +@test "run local check" { + setconfig general run_check yes + runaction + greplog 'Info: Restic check successful.$' +} + +@test "run remote backup" { + local remote="http://${BN_REMOTEUSER}@${BN_REMOTEHOST}:8000/testrestic" + setconfig general repository "rest:${remote}" + setconfig general run_backup yes + runaction + greplog 'Info: Restic backup successful.$' +}