Add optional compression to mariabackup
As database backups can grow substantially in size, compressing backups helps to preserve disk space. While the mariabackup utility offers no compression by itself, we can stream the backup into a compression tool to create an archive [1]. The xtrabackup_checkpoints file, which contains metadata on a backup, gets stored alongside the archive, allowing to create incremental backups from non-compressed backups and vice-versa [2]. One thing to note, is that compressed backups cannot be prepared in advance, this step must be manually carried out by the user. Backup compression is disabled by default and different compressors can be chosen (zstd, xz, ...), with gzip being the default. [1] https://mariadb.com/kb/en/using-encryption-and-compression-tools-with-mariabackup/ [2] https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/#combining-with-stream-output Change-Id: I28c6a0e0b41d4d29c3e79e601de45ea373dee4fb Signed-off-by: Simon Hensel <simon.hensel@inovex.de>
This commit is contained in:
Simon Hensel
parent
92b5711b94
commit
60009ed7ce
@ -319,6 +319,8 @@ galera_mariadb_backups_user: galera_mariadb_backup
|
||||
galera_mariadb_backups_suffix: "{{ inventory_hostname }}"
|
||||
galera_mariadb_backups_cnf_file: "/etc/mysql/mariabackup.cnf"
|
||||
galera_mariadb_backups_nodes: ["{{ galera_cluster_members[0] }}"]
|
||||
galera_mariadb_backups_compress: False
|
||||
galera_mariadb_backups_compressor: gzip
|
||||
|
||||
galera_mariadb_encryption_enabled: false
|
||||
galera_mariadb_encryption_plugin: "file_key_management"
|
||||
|
||||
@ -0,0 +1,28 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Adds optional compression for backups created with mariabackup. Adds two
|
||||
new CLI parameters to the mariabackup script that are used to enable
|
||||
compression and to choose a compression tool.
|
||||
|
||||
* ``--compress=True|False``
|
||||
* ``--compressor=<compressor>``
|
||||
|
||||
Also introduces new Ansible variables that control the above mentioned
|
||||
parameters.
|
||||
|
||||
* ``galera_mariadb_backups_compress``
|
||||
* ``galera_mariadb_backups_compressor``
|
||||
|
||||
Each backup archive is stored in a dedicated directory, alongside the
|
||||
backup metadata.
|
||||
upgrade:
|
||||
- |
|
||||
Backup compression is disabled by default, so no changes need to be made
|
||||
for existing deployments. Should compression be desired, set
|
||||
``galera_mariadb_backups_compress`` to ``True``. Choose a compression tool
|
||||
with ``galera_mariadb_backups_compressor``, default is ``gzip``.
|
||||
others:
|
||||
- |
|
||||
Compressed backups cannot be prepared in advance, this step must be
|
||||
manually carried out by the user before importing it into MariaDB.
|
||||
@ -39,6 +39,7 @@
|
||||
- /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }}
|
||||
--full-backup --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }}
|
||||
--defaults-file={{ galera_mariadb_backups_cnf_file }}
|
||||
--compress={{ galera_mariadb_backups_compress }} --compressor={{ galera_mariadb_backups_compressor }}
|
||||
environment:
|
||||
UMASK: '0640'
|
||||
UMASK_DIR: '0750'
|
||||
@ -66,6 +67,7 @@
|
||||
- /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }}
|
||||
--increment --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }}
|
||||
--defaults-file={{ galera_mariadb_backups_cnf_file }}
|
||||
--compress={{ galera_mariadb_backups_compress }} --compressor={{ galera_mariadb_backups_compressor }}
|
||||
environment:
|
||||
UMASK: '0640'
|
||||
UMASK_DIR: '0750'
|
||||
|
||||
@ -1,11 +1,10 @@
|
||||
#!/usr/bin/python3
|
||||
# {{ ansible_managed }}
|
||||
from subprocess import Popen, PIPE
|
||||
from subprocess import Popen, PIPE, check_output, run
|
||||
from argparse import ArgumentParser
|
||||
from shutil import rmtree
|
||||
from time import strftime, mktime, sleep
|
||||
from datetime import datetime, timedelta
|
||||
import socket
|
||||
import os
|
||||
|
||||
def get_opts():
|
||||
@ -35,6 +34,21 @@ def get_opts():
|
||||
default=False,
|
||||
help="Flag to make incremental backup, based on the latest backup",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compress",
|
||||
dest="compress_flag",
|
||||
default=False,
|
||||
type=eval,
|
||||
choices=[True, False],
|
||||
help="Flag to compress created backups",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compressor",
|
||||
dest="compressor",
|
||||
default="gzip",
|
||||
type=str,
|
||||
help="The compressor to use when compressing backups (default: gzip)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--copies",
|
||||
@ -102,30 +116,44 @@ def check_backups(dest, warning, critical, full_backup_filename):
|
||||
raise SystemExit(0)
|
||||
|
||||
|
||||
def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args):
|
||||
def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args, compress, compressor):
|
||||
check_lock_file()
|
||||
get_lock_file()
|
||||
try:
|
||||
#Creating full backup
|
||||
err = open(os.path.normpath(dest+"/backup.log"), "w")
|
||||
mariabackup_run = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err
|
||||
)
|
||||
mariabackup_run.wait()
|
||||
mariabackup_res = mariabackup_run.communicate()
|
||||
if mariabackup_run.returncode:
|
||||
print(mariabackup_res[1])
|
||||
if compress:
|
||||
#Creating compressed full backup
|
||||
os.makedirs(dest+"/"+full_backup_filename+curtime, exist_ok=True)
|
||||
mariabackup_run = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--extra-lsndir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=PIPE, stderr=err
|
||||
)
|
||||
compressed_backup = open(os.path.normpath(dest+"/"+full_backup_filename+curtime+"/"+full_backup_filename+curtime), "wb")
|
||||
run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup)
|
||||
mariabackup_run.wait()
|
||||
mariabackup_res = mariabackup_run.communicate()
|
||||
if mariabackup_run.returncode:
|
||||
print(mariabackup_res[1])
|
||||
compressed_backup.close()
|
||||
else:
|
||||
#Creating full backup
|
||||
mariabackup_run = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err
|
||||
)
|
||||
mariabackup_run.wait()
|
||||
mariabackup_res = mariabackup_run.communicate()
|
||||
if mariabackup_run.returncode:
|
||||
print(mariabackup_res[1])
|
||||
#Preparing full backup
|
||||
err_p = open(os.path.normpath(dest+"/prepare.log"), "w")
|
||||
mariabackup_prep = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p
|
||||
)
|
||||
mariabackup_prep.wait()
|
||||
mariabackup_prep_res = mariabackup_prep.communicate()
|
||||
if mariabackup_prep.returncode:
|
||||
print(mariabackup_prep_res[1])
|
||||
err_p.close()
|
||||
err.close()
|
||||
#Preparing full backup
|
||||
err_p = open(os.path.normpath(dest+"/prepare.log"), "w")
|
||||
mariabackup_prep = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p
|
||||
)
|
||||
mariabackup_prep.wait()
|
||||
mariabackup_prep_res = mariabackup_prep.communicate()
|
||||
if mariabackup_prep.returncode:
|
||||
print(mariabackup_prep_res[1])
|
||||
err_p.close()
|
||||
except OSError:
|
||||
print("Please, check that Mariabackup is installed")
|
||||
except Exception as e:
|
||||
@ -134,7 +162,7 @@ def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_ar
|
||||
os.unlink("/var/run/mariabackup-galera/db_backup.pid")
|
||||
|
||||
|
||||
def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args):
|
||||
def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args, compress, compressor):
|
||||
check_lock_file()
|
||||
get_lock_file()
|
||||
try:
|
||||
@ -145,14 +173,28 @@ def create_increment_backup(dest, curtime, increment_backup_filename, extra_mari
|
||||
raise SystemExit(1)
|
||||
try:
|
||||
err = open(os.path.normpath(dest+"/increment.err"), "w")
|
||||
#Creating incremental backup
|
||||
mariabackup_run = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err
|
||||
)
|
||||
mariabackup_run.wait()
|
||||
mariabackup_res = mariabackup_run.communicate()
|
||||
if mariabackup_run.returncode:
|
||||
print(mariabackup_res[1])
|
||||
if compress:
|
||||
#Creating compressed incremental backup
|
||||
os.makedirs(dest+"/"+increment_backup_filename+curtime, exist_ok=True)
|
||||
mariabackup_run = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--incremental-basedir="+basedir, "--extra-lsndir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime)], stdout=PIPE, stderr=err
|
||||
)
|
||||
compressed_backup = open(os.path.normpath(dest+"/"+increment_backup_filename+curtime+"/"+increment_backup_filename+curtime), "wb")
|
||||
run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup)
|
||||
mariabackup_run.wait()
|
||||
mariabackup_res = mariabackup_run.communicate()
|
||||
if mariabackup_run.returncode:
|
||||
print(mariabackup_res[1])
|
||||
compressed_backup.close()
|
||||
else:
|
||||
#Creating incremental backup
|
||||
mariabackup_run = Popen(
|
||||
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err
|
||||
)
|
||||
mariabackup_run.wait()
|
||||
mariabackup_res = mariabackup_run.communicate()
|
||||
if mariabackup_run.returncode:
|
||||
print(mariabackup_res[1])
|
||||
err.close()
|
||||
except OSError:
|
||||
print("Please, check that Mariabackup is installed")
|
||||
@ -227,11 +269,11 @@ def main():
|
||||
if opts.fullbackup_flag and opts.increment_flag:
|
||||
raise NameError("Only one flag can be specified per operation")
|
||||
elif opts.fullbackup_flag:
|
||||
create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args)
|
||||
create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor)
|
||||
rotate_backups(opts.destdir, opts.copies_flag, full_backup_filename, increment_backup_filename)
|
||||
raise SystemExit()
|
||||
elif opts.increment_flag:
|
||||
create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args)
|
||||
create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor)
|
||||
raise SystemExit()
|
||||
elif opts.check_flag:
|
||||
pass
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user