.gitignore

@@ -35,6 +35,8 @@ handlers/pgsql
 handlers/pgsql.helper
 handlers/rdiff
 handlers/rdiff.helper
+handlers/restic
+handlers/restic.helper
 handlers/rsync
 handlers/sh
 handlers/svn
@@ -47,9 +49,10 @@ lib/Makefile
 lib/easydialog
 lib/parseini
 lib/tools
-lib/vserver
 man/Makefile
 src/Makefile
 src/backupninja
 src/ninjahelper
 .vagrant
+*.swp
+*.swo

AUTHORS

@@ -7,7 +7,7 @@ micah@riseup.net   -- debian package, vserver support, bug fixes
 stefani@riseup.net -- makecd handler, man pages
 intrigeri@boum.org -- dup handler, pgsql handler, vserver support, bug fixes
 Charles Lepple 	   -- trac handler
-Petr Klma <petr.klima@madeta-group.cz>	-- autotools, RPM support and sys checks
+Petr Kl�ma <petr.klima@madeta-group.cz>	-- autotools, RPM support and sys checks
 paulv@bikkel.org   -- rsnap handler
 Robert Napier 	   -- improved RPM build
 rhatto 		   -- rub handler and patches
@@ -65,3 +65,8 @@ David Gasaway <dave@gasaway.org> -- Fixes for configuration files without suffix
 Hugh Nowlan <nosmo@nosmo.me> -- dup check for archive dir
 Lyz <lyz@riseup.net> -- sys support for LUKS in disk partitions
 Glandos <bugs-0xacab@antipoul.fr> -- sys excludes zram devices
+Nicolas Karolak <nicolas@karolak.fr> -- Add restic support
+Derek Laventure -- Add restic helper
+Colan Schwartz -- Fix restic options handler
+
+... and other contributors, thank you!

INSTALL.md

@@ -15,7 +15,7 @@ Requirements:
 
 Recommended:
 
-        rdiff-backup duplicity rsync borgbackup gzip hwinfo sfdisk cryptsetup flashrom hwinfo
+        borgbackup cryptsetup duplicity flashrom gzip hwinfo rdiff-backup restic rsync sfdisk
 
 To install backupninja, simply do the following:
 

Makefile.am

 # vi: noexpandtab softtabstop=0
 ## Process this file with automake to produce Makefile.in
 
-EXTRA_DIST = FAQ.md README.md COPYING AUTHORS INSTALL.md NEWS ChangeLog \
+EXTRA_DIST = FAQ.md README.md COPYING AUTHORS INSTALL.md NEWS CHANGELOG.md \
              backupninja.spec backupninja.spec.in autogen.sh
 
 SUBDIRS = etc examples handlers lib man src

README.md

@@ -24,7 +24,6 @@ The key features of backupninja are:
  - console-based wizard (ninjahelper) makes it easy to create
    backup action configuration files
  - passwords are never sent via the command line to helper programs
- - works with [Linux-Vservers](http://linux-vserver.org/)
 
 The following backup types are supported:
 
@@ -82,8 +81,7 @@ To add an additional 'wizard' to ninjahelper, follow these steps:
            }
 
 3. Look at the existing helpers to see how they are written. Try to re-use
-   functions, such as the dialog functions that are defined in `easydialog.sh`,
-   or the vserver functions defined in `lib/vserver`.
+   functions, such as the dialog functions that are defined in `easydialog.sh`.
 
 4. Test, re-test, and test again. Try to break the helper by going backwards,
    try to think like someone who has no idea how to configure your handler
@@ -106,6 +104,7 @@ file in `/etc/backup.d` according to the file's suffix:
 
  - `.sh`: run this file as a shell script.
  - `.rdiff`: filesystem backup (using rdiff-backup)
+ - `.restic`: filesystem backup (using restic)
  - `.dup`: filesystem backup (using duplicity)
  - `.borg`: filesystem backup (using borg)
  - `.mysql`: backup mysql databases
@@ -215,25 +214,6 @@ Since it performs incremental backups it minimizes the number of request per
 operation therefore reducing the costs. The boto Python interface to Amazon
 Web Services is needed to use duplicity with S3 (Debian package: `python-boto`).
 
-
-Vservers
---------
-
-If you are using [Linux-Vservers](http://linux-vserver.org/) there are some
-special capabilities that different handlers have to make vserver
-backups easier.
-
-Set the variable `vservers` to be `yes` in `/etc/backupninja.conf` and see the
-example configuration files for each handler to configure the vserver specific
-variables.
-
-Additional vserver variables that can be configured in `/etc/backupninja.conf`,
-but they probably don't need to be changed:
-
- - `VSERVERINFO` (default: `/usr/sbin/vserver-info`)
- - `VSERVER` (default: `/usr/sbin/vserver`)
- - `VROOTDIR` (default: `$VSERVERINFO info SYSINFO |grep vserver-Rootdir | awk '{print $2}'`)
-
 .sh configuration files
 -----------------------
 

README_tests.md

+When developping fixes or new features for backupninja, it is highly recommended
+to run the test suite to help spot potential problems.
+
+The test suite is based on Vagrant, and is configured to rely on the VirtualBox
+provider. The required package may be installed using the following command:
+
+    apt install vagrant virtualbox
+
+On Debian 10 (buster) these packages aren't available in the default upstream
+repositories, so you will need to use an alternative such as the one provided
+by an individual Debian developper here:
+
+    https://people.debian.org/~lucas/virtualbox-buster/
+
+Once the requirements are in place, the test suite may be run in this manner:
+
+    git clone git@0xacab.org:liberate/backupninja.git
+    cd backupninja
+    vagrant up
+    vagrant ssh -c "sudo /vagrant/test/test.sh"
+
+It's possible to only test a specific handler with:
+
+    vagrant ssh -c "sudo /vagrant/test/test.sh rdiff"
+
+To synchronise changes in the source code and rebuild backupninja:
+
+    vagrant rsync local && vagrant ssh -c "build-backupninja.sh"
+
+Please report any problems with the test suite on the issue tracker at:
+
+    https://0xacab.org/liberate/backupninja/-/issues

RELEASE.md

 Upstream
 ========
 
+* run the full testsuite
+
+        vagrant rsync local && \
+        vagrant ssh -c "build-backupninja.sh && sudo /vagrant/test/test.sh"
+
 * prepare the environment:
 
         export VERSION=x.y.z
 
-* update `configure.ac` and `ChangeLog`
+* update `configure.ac` and `CHANGELOG.md`
 
         perl -pi -E \
-           "s{^AC_INIT\(\[backupninja\],\[[0-9.]+\],}{AC_INIT([backupninja],[$VERSION],}" \
-           configure.ac && \
-        RELEASE_DATE=$(LC_ALL=C date '+%B %d, %Y') perl -pi -E \
-           "s{^version\s+[0-9.]+\s+--\s+UNRELEASED$}{version $VERSION -- $RELEASE_DATE}" \
-           ChangeLog
+           "s{^AC_INIT\(\[backupninja\],\[[0-9.\-rc]+\],}{AC_INIT([backupninja],[$VERSION],}" \
+           configure.ac
+
+        RELEASE_DATE=$(LC_ALL=C date '+%Y-%m-%d'); perl -pi -E \
+           "s{^## \[Unreleased\].*}{## [$VERSION] - $RELEASE_DATE}" \
+           CHANGELOG.md
 
-* commit, tag and create the tarball:
+* commit and created signed tag
 
-        git commit configure.ac ChangeLog \
+        git commit configure.ac CHANGELOG.md \
             -m "Releasing backupninja $VERSION" && \
-        git clean -fdx && \
+        git clean -fdx -e .vagrant && \
         git tag -s "backupninja-$VERSION" \
             -m "Releasing backupninja $VERSION" && \
         ./autogen.sh && \
@@ -28,15 +34,15 @@ Upstream
 * compare the content of the generated tarball with the content of the
   previous one
 
-* move the tarball outside of the Git working copy and clean up:
+        diffoscope --text-color=always ../tarballs/backupninja-x.y.z.tar.gz \
+            backupninja-$VERSION.tar.gz | less -R
+
+* move the tarball outside of the Git working copy and clean up
 
         mkdir -p ../tarballs && \
         mv backupninja-$VERSION.tar.gz ../tarballs/ && \
         make distclean && \
-        git clean -fdx
-
-* Install (extract tarball, `.configure && make && sudo make install`)
-  and test.
+        git clean -fdx -e .vagrant
 
 Debian
 ======
@@ -58,10 +64,8 @@ Install the `.deb` and test.
 Release
 =======
 
-* sign the release and push it to Git:
+* push the release to GitLab
 
-        gpg --armor --detach-sign \
-            ../tarballs/backupninja-$VERSION.tar.gz && \
         git checkout debian && \
         gbp buildpackage --git-tag-only --git-sign-tags && \
         git push --follow-tags origin \
@@ -70,10 +74,11 @@ Release
             pristine-tar:pristine-tar \
             upstream:upstream
 
-* upload the upstream tarball and detached signature to the GitLab
-  milestone page with *Edit* → *Attach a file*
+* create a new GitLab release
+
 * announce the release on the backupninja mailing-list,
   pointing to the milestone web page
+
 * upload to Debian or ask someone listed in the `Uploaders` control
   field to review and upload
 
@@ -81,4 +86,5 @@ Open the next development cycle
 ===============================
 
 * `git checkout master`
-* Add an empty new section in `ChangeLog`, commit and push.
+
+* Add an empty new section in `CHANGELOG.md`, commit and push.

TODO

@@ -6,15 +6,13 @@ you are working on it!
   http://bugs.debian.org/cgi-bin/pkgreport.cgi?which=pkg&data=backupninja&archive=no
 
 . Fix all bugs reported on our Gitlab instance:
-  https://0xacab.org/riseuplabs/backupninja/issues
+  https://0xacab.org/liberate/backupninja/issues
 
 . Make ninjahelper allow you to pick what type of backup you want (instead
   of just assuming you want local-to-remote, or push backups. Some people
   want local-to-local, or remote-to-local, or pull backups). This has been
   reported for the duplicity handler as Debian bug #346040.
 
-. Allow vsnames "all" in the msyql handler.
-
 . Factorize the rdiff.helper's connection-related functions into a lib, so
   that they can be used by dup.helper too. (NB: don't forget that the dup
   handler has a sshoptions configuration setting, that is often used to

Vagrantfile

@@ -3,13 +3,89 @@
 
 ENV["LC_ALL"] = "en_US.UTF-8"
 
+base_box = "debian/testing64"
+
+empty_disk = '.vagrant/tmp/empty.vdi'
+lvm_disk = '.vagrant/tmp/lvm.vdi'
+lukspart_disk = '.vagrant/tmp/lukspart.vdi'
+luksdev_disk = '.vagrant/tmp/luksdev.vdi'
+
 Vagrant.configure("2") do |config|
-  config.vm.box = "debian/stretch64"
-  config.vm.provision "shell", inline: <<-SHELL
+
+  config.vm.define "remote" do |remote|
+    remote.vm.box = base_box
+    remote.vm.hostname = "bntest1"
+    remote.vm.network "private_network", ip: "192.168.181.5"
+    remote.vm.provision "shell", inline: <<-SHELL
+      export DEBIAN_FRONTEND=noninteractive
+      echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
       locale-gen
       apt-get update
-    apt-get install -y git automake make
-    git clone https://0xacab.org/riseuplabs/backupninja.git
-    chown vagrant: backupninja -R
+      apt-get install -y borgbackup duplicity rdiff-backup restic rsync
+      sed -i 's/^PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
+      echo "Port 22" >> /etc/ssh/sshd_config
+      echo "Port 7722" >> /etc/ssh/sshd_config
+      systemctl reload sshd
+      echo -e "vagrant\nvagrant" | passwd vagrant
+      chown vagrant: /var/backups
+      wget -q https://github.com/restic/rest-server/releases/download/v0.10.0/rest-server_0.10.0_linux_amd64.tar.gz -O - | tar -xz -C /usr/local/bin --strip-components=1
     SHELL
   end
+
+  config.vm.define "local", primary: true do |local|
+    local.vm.box = base_box
+    local.vm.hostname = "bntest0"
+    local.vm.network "private_network", ip: "192.168.181.4"
+    local.vm.provision "shell", inline: <<-SHELL
+      export DEBIAN_FRONTEND=noninteractive
+      echo "root: vagrant" >> /etc/aliases
+      echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
+      locale-gen
+      apt-get update
+      apt-get install -y automake make dialog sshpass
+      BUILDSCRIPT="/usr/local/bin/build-backupninja.sh"
+      echo "#!/bin/sh" >> $BUILDSCRIPT
+      echo "cd /vagrant" >> $BUILDSCRIPT
+      echo "make clean" >> $BUILDSCRIPT
+      echo "./autogen.sh" >> $BUILDSCRIPT
+      echo "./configure --prefix=/usr --mandir=/usr/share/man --sysconfdir=/etc --localstatedir=/var --libdir=/usr/lib --libexecdir=/usr/lib" >> $BUILDSCRIPT
+      echo "make" >> $BUILDSCRIPT
+      echo "sudo make install" >> $BUILDSCRIPT
+      chmod +x $BUILDSCRIPT
+      $BUILDSCRIPT
+      mkdir -p /root/.ssh
+      yes y | ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N ''
+      echo "StrictHostKeyChecking accept-new" >> /root/.ssh/config
+      echo "192.168.181.5 bntest1" >> /etc/hosts
+      sshpass -p vagrant scp /root/.ssh/id_ed25519.pub vagrant@bntest1:/tmp/bntest.pub
+      sshpass -p vagrant ssh vagrant@bntest1 "cat /tmp/bntest.pub >> /home/vagrant/.ssh/authorized_keys"
+      sshpass -p vagrant ssh vagrant@bntest1 "chmod 400 /home/vagrant/.ssh/authorized_keys"
+      ssh vagrant@bntest1 "sudo sed -i 's/^PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config"
+      ssh vagrant@bntest1 "sudo systemctl reload sshd"
+    SHELL
+
+    local.vm.provider :virtualbox do |vb|
+      unless File.exist?(empty_disk)
+        vb.customize ['createhd', '--filename', empty_disk, '--size', 100 ]
+      end
+      unless File.exist?(empty_disk)
+        vb.customize ['createhd', '--filename', lvm_disk, '--size', 100 ]
+      end
+      unless File.exist?(lukspart_disk)
+        vb.customize ['createhd', '--filename', lukspart_disk, '--size', 100 ]
+      end
+      unless File.exist?(luksdev_disk)
+        vb.customize ['createhd', '--filename', luksdev_disk, '--size', 100 ]
+      end
+      vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', empty_disk]
+      vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', lvm_disk]
+      vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 3, '--device', 0, '--type', 'hdd', '--medium', lukspart_disk]
+      vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 4, '--device', 0, '--type', 'hdd', '--medium', luksdev_disk]
+    end
+  end
+
+  config.vm.synced_folder ".", "/vagrant", type: "rsync",
+    rsync__exclude: ".git/",
+    rsync__args: ["--recursive", "--delete"]
+
+end

backupninja.spec.in

@@ -7,11 +7,11 @@ Version:    %{version}
 Release:    1
 License:    GPL
 Group:      Applications/System
-URL:        https://0xacab.org/riseuplabs/backupninja
+URL:        https://0xacab.org/liberate/backupninja
 Source:     %{name}-%{version}.tar.gz
 Requires:   bash, gawk, rdiff-backup, gzip
 Provides:   %{name}
-Packager:   Petr Klima <Petr.Klima@madeta-group.cz>
+Packager:   Petr Klima <Petr.Klima@madeta.cz>
 BuildRoot:  %{_tmppath}/%{name}-%{version}
 Prefix:     %{_prefix}
 
@@ -50,7 +50,7 @@ rm -fr %{buildroot}
 
 %ghost %{_localstatedir}/log/backupninja.log
 
-%doc AUTHORS COPYING ChangeLog INSTALL.md NEWS README.md
+%doc AUTHORS COPYING CHANGELOG.md INSTALL.md NEWS README.md
 %{_mandir}/man1/*
 %{_mandir}/man5/*
 

configure.ac

@@ -3,15 +3,12 @@
 
 # The maintainer mode is causing me grief with newest versions of autotools
 #AM_MAINTAINER_MODE
-AC_INIT([backupninja],[1.1.0],[backupninja@lists.riseup.net])
+AC_INIT([backupninja],[1.2.2],[backupninja@lists.riseup.net])
 AC_CONFIG_SRCDIR([src/backupninja.in])
 AM_INIT_AUTOMAKE([foreign])
 
 # Checks for programs.
 
-# BASH may already be set in the shell, if the admin then changes the
-# the /bin/sh symlink to a non-bash shell, all hell will break lose.
-unset BASH
 AC_PATH_PROGS(BASH, bash, "no", [$PATH:/bin:/usr/bin:/usr/sbin])
 if test x$BASH = "xno"; then
     AC_MSG_ERROR([bash is required])

etc/backupninja.conf.in

@@ -13,6 +13,10 @@
 # 1 -- Fatal errors           (only)
 loglevel = 4
 
+# Produce prometheus metrics of backup status (default = no).
+# Requires `prometheus-node-exporter` to be installed
+reportprom = false
+
 # send a summary of the backup status to
 # this email address:
 reportemail = root
@@ -46,6 +50,9 @@ reportuser = ninja
 # use a globally unique name, preferably the hostname
 reportdirectory = /var/lib/backupninja/reports
 
+# number of columns the report email body should wrap to
+#reportwrap = 80
+
 # set to the administration group that is allowed to
 # read/write configuration files in /etc/backup.d
 admingroup = root
@@ -72,9 +79,6 @@ usecolors = yes
 # default value for 'when'
 when = everyday at 01:00
 
-# if running vservers, set to yes
-vservers = no
-
 # programs paths
 # SLAPCAT=/usr/sbin/slapcat
 # LDAPSEARCH=/usr/bin/ldapsearch
@@ -89,6 +93,3 @@ vservers = no
 # GZIP=/bin/gzip
 # GZIP_OPTS='--rsyncable'
 # RSYNC=/usr/bin/rsync
-# VSERVERINFO=/usr/sbin/vserver-info
-# VSERVER=/usr/sbin/vserver
-# VROOTDIR=/var/lib/vservers

examples/Makefile.am

 
 EXAMPLES = example.borg example.dup example.maildir example.makecd \
 		example.mysql example.pgsql example.rdiff example.rsync \
-		example.sh example.svn example.sys example.trac
+		example.sh example.svn example.sys example.trac example.restic
 
 EXTRA_DIST = $(EXAMPLES)
 

examples/example.borg

@@ -11,6 +11,13 @@
 ## destination host and user.
 ##
 
+## passed directly to borg commands, e.g. to use a particular remote
+## borg executable (here: 'borg12'), set this to:
+## options = --remote-path=borg12
+##
+## Default:
+# options =
+
 ## default is 0, but set to 19 if you want to lower the priority.
 ## an example setting would be:
 ## nicelevel = 19
@@ -18,6 +25,16 @@
 ## Default
 # nicelevel = 0
 
+## ionicelevel is optional. If it is left undefined or empty, ionice will not
+## be used at all. If it is set to to an integer value from 0 to 7, then ionice
+## will be used with the best effort class (-c2) and with the ionicelevel as
+## the class data (-n).
+##
+## See the ionice(1) man page for more details about available levels.
+##
+## Default:
+# ionicelevel =
+
 ## default is yes. set to no to skip the test if the remote host is alive
 ##
 ## Default:
@@ -29,6 +46,22 @@
 ## Default:
 # bwlimit = 0
 
+## export "borg info last_archive" to a given file
+## this is usefull for monitoring without using borg
+##
+## Example:
+# borginfo = /var/backups/borginfo.json
+## Default:
+# borginfo =
+
+## export "borg list repository" to a given file
+## this is usefull for monitoring without using borg
+##
+## Example
+# borglist = /var/backups/borglist.json
+## Default:
+# borglist =
+
 ######################################################
 ## source section
 ## (where the files to be backed up are coming from)
@@ -97,6 +130,16 @@ exclude = /var/lib/mysql
 ## Default:
 # keep = 30d
 
+## define hourly, daily, weekly and monthly retention for the "borg prune" operation.
+##
+## theses options will be ignored if set to 0
+##
+## Default:
+## keephourly = 0
+## keepdaily = 0
+## keepweekly = 0
+## keepmonthly = 0
+
 ## define extra command-line options for the "borg prune" operation.
 ##
 ## Example:
@@ -107,6 +150,33 @@ exclude = /var/lib/mysql
 ## Default:
 # prune_options =
 
+## Path to the directory that will hold borg's cache files. By default this is
+## empty, which will let borg use its default path of "~/.cache/borg".
+##
+## Default:
+# cache_directory =
+
+## by default borg emits various warnings that are impossible to check on large
+## infrastructures.
+## - when some files/repositories included in borg create does not exists
+## - when some files have changed during the backup (happens a lot on log files)
+## This option allows to disable these warning.
+##
+## Default:
+# filter_warnings = yes
+
+## when filter_warning == yes, allows to choose to disable warning if
+## file changed during backup
+##
+## Default:
+# warning_if_file_changed_during_backup = yes
+
+## when warning_if_file_changed_during_backup == yes, allows to ignore some
+## paths or filenames.
+##
+## Default:
+# file_changes_to_ignore = /
+
 ######################################################
 ## destination section
 ## (where the files are copied to)
@@ -181,8 +251,10 @@ exclude = /var/lib/mysql
 ## Default:
 # passphrase =
 
-## Path to the directory that will hold borg's cache files. By default this is
-## empty, which will let borg use its default path of "~/.cache/borg".
+## command-line options to use with ssh
+##
+## an example setting would be:
+## sshoptions = -i /root/.ssh/id_rsa_borg
 ##
 ## Default:
-# cache_directory =
+# sshoptions =

examples/example.dup

@@ -19,6 +19,16 @@
 ## Default:
 # nicelevel = 0
 
+## ionicelevel is optional. If it is left undefined or empty, ionice will not
+## be used at all. If it is set to to an integer value from 0 to 7, then ionice
+## will be used with the best effort class (-c2) and with the ionicelevel as
+## the class data (-n).
+##
+## See the ionice(1) man page for more details about available levels.
+##
+## Default:
+# ionicelevel =
+
 ## test the connection? set to no to skip the test if the remote host is alive.
 ## if 'desturl' is set below, 'testconnect' must be set to 'no' for now.
 ##
@@ -101,7 +111,7 @@
 [source]
 
 ## A few notes about includes and excludes:
-## 1. include, exclude and vsinclude statements support globbing with '*'
+## 1. include and exclude statements support globbing with '*'
 ## 2. Symlinks are not dereferenced. Moreover, an include line whose path
 ##    contains, at any level, a symlink to a directory, will only have the
 ##    symlink backed-up, not the target directory's content. Yes, you have to
@@ -127,21 +137,6 @@ include = /usr/local/sbin
 include = /var/lib/dpkg/status
 include = /var/lib/dpkg/status-old
 
-## If vservers = yes in /etc/backupninja.conf then the following variables can
-## be used:
-## vsnames = all | <vserver1> <vserver2> ... (default = all)
-## vsinclude = <path>
-## vsinclude = <path>
-## ...
-## Any path specified in vsinclude is added to the include list for each vserver
-## listed in vsnames (or all if vsnames = all, which is the default).
-##
-## For example, vsinclude = /home will backup the /home directory in every
-## vserver listed in vsnames. If you have 'vsnames = foo bar baz', this
-## vsinclude will add to the include list /vservers/foo/home, /vservers/bar/home
-## and /vservers/baz/home.
-## Vservers paths are derived from $VROOTDIR.
-
 # files to exclude from the backup
 exclude = /home/*/.gnupg
 exclude = /var/cache/backupninja/duplicity