diff --git a/etc/reddwarf/reddwarf-guestagent.conf.sample b/etc/reddwarf/reddwarf-guestagent.conf.sample index da473a4f9b..e256477649 100644 --- a/etc/reddwarf/reddwarf-guestagent.conf.sample +++ b/etc/reddwarf/reddwarf-guestagent.conf.sample @@ -72,6 +72,8 @@ storage_strategy = SwiftStorage storage_namespace = reddwarf.guestagent.strategies.storage.swift backup_swift_container = database_backups backup_use_gzip_compression = True +backup_use_openssl_encryption = True +backup_aes_cbc_key = "default_aes_cbc_key" backup_use_snet = False backup_chunk_size = 65536 backup_segment_max_size = 2147483648 diff --git a/reddwarf/common/cfg.py b/reddwarf/common/cfg.py index c5994b4459..ed3eb45999 100644 --- a/reddwarf/common/cfg.py +++ b/reddwarf/common/cfg.py @@ -140,6 +140,10 @@ common_opts = [ cfg.StrOpt('backup_swift_container', default='database_backups'), cfg.BoolOpt('backup_use_gzip_compression', default=True, help='Compress backups using gzip.'), + cfg.BoolOpt('backup_use_openssl_encryption', default=True, + help='Encrypt backups using openssl.'), + cfg.StrOpt('backup_aes_cbc_key', default='default_aes_cbc_key', + help='default openssl aes_cbc key.'), cfg.BoolOpt('backup_use_snet', default=False, help='Send backup files over snet.'), cfg.IntOpt('backup_chunk_size', default=2 ** 16, diff --git a/reddwarf/guestagent/strategies/backup/base.py b/reddwarf/guestagent/strategies/backup/base.py index 75d553c17c..53e47e4c4b 100644 --- a/reddwarf/guestagent/strategies/backup/base.py +++ b/reddwarf/guestagent/strategies/backup/base.py @@ -30,6 +30,9 @@ CHUNK_SIZE = CONF.backup_chunk_size MAX_FILE_SIZE = CONF.backup_segment_max_size BACKUP_CONTAINER = CONF.backup_swift_container BACKUP_USE_GZIP = CONF.backup_use_gzip_compression +BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption +BACKUP_ENCRYPT_KEY = CONF.backup_aes_cbc_key + LOG = logging.getLogger(__name__) @@ -49,6 +52,8 @@ class BackupRunner(Strategy): # The actual system call to run the backup cmd = None is_zipped = BACKUP_USE_GZIP + is_encrypted = BACKUP_USE_OPENSSL + encrypt_key = BACKUP_ENCRYPT_KEY def __init__(self, filename, **kwargs): self.filename = filename @@ -119,6 +124,15 @@ class BackupRunner(Strategy): def zip_manifest(self): return '.gz' if self.is_zipped else '' + @property + def encrypt_cmd(self): + return (' | openssl enc -aes-256-cbc -salt -pass pass:%s' % + self.encrypt_key) if self.is_encrypted else '' + + @property + def encrypt_manifest(self): + return '.enc' if self.is_encrypted else '' + def read(self, chunk_size): """Wrap self.process.stdout.read to allow for segmentation.""" if self.end_of_segment: diff --git a/reddwarf/guestagent/strategies/backup/impl.py b/reddwarf/guestagent/strategies/backup/impl.py index cb143ea2b9..8a2c0bc7b9 100644 --- a/reddwarf/guestagent/strategies/backup/impl.py +++ b/reddwarf/guestagent/strategies/backup/impl.py @@ -32,12 +32,12 @@ class MySQLDump(base.BackupRunner): ' --opt'\ ' --password=%(password)s'\ ' -u %(user)s' - return cmd + self.zip_cmd + return cmd + self.zip_cmd + self.encrypt_cmd @property def manifest(self): - manifest = '%s' + self.zip_manifest - return manifest % self.filename + manifest = '%s' % self.filename + return manifest + self.zip_manifest + self.encrypt_manifest class InnoBackupEx(base.BackupRunner): @@ -49,9 +49,9 @@ class InnoBackupEx(base.BackupRunner): cmd = 'sudo innobackupex'\ ' --stream=xbstream'\ ' /var/lib/mysql 2>/tmp/innobackupex.log' - return cmd + self.zip_cmd + return cmd + self.zip_cmd + self.encrypt_cmd @property def manifest(self): - manifest = '%s.xbstream' + self.zip_manifest - return manifest % self.filename + manifest = '%s.xbstream' % self.filename + return manifest + self.zip_manifest + self.encrypt_manifest diff --git a/reddwarf/guestagent/strategies/restore/base.py b/reddwarf/guestagent/strategies/restore/base.py index 8e8f5fb615..d8e023f7ed 100644 --- a/reddwarf/guestagent/strategies/restore/base.py +++ b/reddwarf/guestagent/strategies/restore/base.py @@ -27,6 +27,9 @@ import glob LOG = logging.getLogger(__name__) CONF = cfg.CONF CHUNK_SIZE = CONF.backup_chunk_size +BACKUP_USE_GZIP = CONF.backup_use_gzip_compression +BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption +BACKUP_DECRYPT_KEY = CONF.backup_aes_cbc_key RESET_ROOT_RETRY_TIMEOUT = 100 RESET_ROOT_SLEEP_INTERVAL = 10 RESET_ROOT_MYSQL_COMMAND = """ @@ -65,13 +68,20 @@ class RestoreRunner(Strategy): # The backup format type restore_type = None + # Decryption Parameters + is_zipped = BACKUP_USE_GZIP + is_encrypted = BACKUP_USE_OPENSSL + decrypt_key = BACKUP_DECRYPT_KEY + def __init__(self, restore_stream, **kwargs): self.restore_stream = restore_stream self.restore_location = kwargs.get('restore_location', '/var/lib/mysql') - self.restore_cmd = self.restore_cmd % kwargs - self.prepare_cmd = self.prepare_cmd % kwargs \ - if hasattr(self, 'prepare_cmd') else None + self.restore_cmd = (self.decrypt_cmd + + self.unzip_cmd + + (self.base_restore_cmd % kwargs)) + self.prepare_cmd = self.base_prepare_cmd % kwargs \ + if hasattr(self, 'base_prepare_cmd') else None super(RestoreRunner, self).__init__() def __enter__(self): @@ -164,3 +174,15 @@ class RestoreRunner(Strategy): filelist = glob.glob(self.restore_location + "/ib_logfile*") for f in filelist: os.unlink(f) + + @property + def decrypt_cmd(self): + if self.is_encrypted: + return ('openssl enc -d -aes-256-cbc -salt -pass pass:%s | ' + % self.decrypt_key) + else: + return '' + + @property + def unzip_cmd(self): + return 'gzip -d -c | ' if self.is_zipped else '' diff --git a/reddwarf/guestagent/strategies/restore/impl.py b/reddwarf/guestagent/strategies/restore/impl.py index 26e33d3c15..00d414d7a6 100644 --- a/reddwarf/guestagent/strategies/restore/impl.py +++ b/reddwarf/guestagent/strategies/restore/impl.py @@ -25,10 +25,9 @@ LOG = logging.getLogger(__name__) class MySQLDump(base.RestoreRunner): """ Implementation of Restore Strategy for MySQLDump """ __strategy_name__ = 'mysqldump' - is_zipped = True - restore_cmd = ('mysql ' - '--password=%(password)s ' - '-u %(user)s') + base_restore_cmd = ('mysql ' + '--password=%(password)s ' + '-u %(user)s') def _pre_restore(self): pass @@ -40,11 +39,10 @@ class MySQLDump(base.RestoreRunner): class InnoBackupEx(base.RestoreRunner): """ Implementation of Restore Strategy for InnoBackupEx """ __strategy_name__ = 'innobackupex' - is_zipped = True - restore_cmd = 'sudo xbstream -x -C %(restore_location)s' - prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s ' - '--defaults-file=%(restore_location)s/backup-my.cnf ' - '--ibbackup xtrabackup 2>/tmp/innoprepare.log') + base_restore_cmd = 'sudo xbstream -x -C %(restore_location)s' + base_prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s' + ' --defaults-file=%(restore_location)s/backup-my.cnf' + ' --ibbackup xtrabackup 2>/tmp/innoprepare.log') def _pre_restore(self): app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get()) diff --git a/reddwarf/guestagent/strategies/storage/swift.py b/reddwarf/guestagent/strategies/storage/swift.py index 7a1b4892de..3b8c2b53b1 100644 --- a/reddwarf/guestagent/strategies/storage/swift.py +++ b/reddwarf/guestagent/strategies/storage/swift.py @@ -21,7 +21,6 @@ from reddwarf.common import utils from eventlet.green import subprocess import zlib -UNZIPPER = zlib.decompressobj(16 + zlib.MAX_WBITS) LOG = logging.getLogger(__name__) @@ -107,7 +106,6 @@ class SwiftDownloadStream(object): def __init__(self, **kwargs): self.process = None self.pid = None - self.is_zipped = kwargs.get('is_zipped', False) self.cmd = self.cmd % kwargs def __enter__(self): @@ -128,9 +126,7 @@ class SwiftDownloadStream(object): pass def read(self, *args, **kwargs): - if not self.is_zipped: - return self.process.stdout.read(*args, **kwargs) - return UNZIPPER.decompress(self.process.stdout.read(*args, **kwargs)) + return self.process.stdout.read(*args, **kwargs) def run(self): self.process = subprocess.Popen(self.cmd, shell=True, diff --git a/reddwarf/tests/unittests/guestagent/test_backups.py b/reddwarf/tests/unittests/guestagent/test_backups.py new file mode 100644 index 0000000000..40910777eb --- /dev/null +++ b/reddwarf/tests/unittests/guestagent/test_backups.py @@ -0,0 +1,114 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License + +import reddwarf.guestagent.strategies.backup.base as backupBase +import reddwarf.guestagent.strategies.restore.base as restoreBase +import testtools +from reddwarf.common import utils + +BACKUP_XTRA_CLS = "reddwarf.guestagent.strategies.backup.impl.InnoBackupEx" +RESTORE_XTRA_CLS = "reddwarf.guestagent.strategies.restore.impl.InnoBackupEx" +BACKUP_SQLDUMP_CLS = "reddwarf.guestagent.strategies.backup.impl.MySQLDump" +RESTORE_SQLDUMP_CLS = "reddwarf.guestagent.strategies.restore.impl.MySQLDump" +PIPE = " | " +ZIP = "gzip" +UNZIP = "gzip -d -c" +ENCRYPT = "openssl enc -aes-256-cbc -salt -pass pass:default_aes_cbc_key" +DECRYPT = "openssl enc -d -aes-256-cbc -salt -pass pass:default_aes_cbc_key" +XTRA_BACKUP = "sudo innobackupex --stream=xbstream /var/lib/mysql 2>/" \ + "tmp/innobackupex.log" +SQLDUMP_BACKUP = "/usr/bin/mysqldump --all-databases --opt " \ + "--password=password -u user" +XTRA_RESTORE = "sudo xbstream -x -C /var/lib/mysql" +SQLDUMP_RESTORE = "mysql --password=password -u user" +PREPARE = "sudo innobackupex --apply-log /var/lib/mysql " \ + "--defaults-file=/var/lib/mysql/backup-my.cnf " \ + "--ibbackup xtrabackup 2>/tmp/innoprepare.log" +CRYPTO_KEY = "default_aes_cbc_key" + + +class GuestAgentBackupTest(testtools.TestCase): + def test_backup_decrypted_xtrabackup_command(self): + backupBase.BackupRunner.is_zipped = True + backupBase.BackupRunner.is_encrypted = False + RunnerClass = utils.import_class(BACKUP_XTRA_CLS) + bkup = RunnerClass(12345, user="user", password="password") + self.assertEqual(bkup.command, XTRA_BACKUP + PIPE + ZIP) + self.assertEqual(bkup.manifest, "12345.xbstream.gz") + + def test_backup_encrypted_xtrabackup_command(self): + backupBase.BackupRunner.is_zipped = True + backupBase.BackupRunner.is_encrypted = True + backupBase.BackupRunner.encrypt_key = CRYPTO_KEY + RunnerClass = utils.import_class(BACKUP_XTRA_CLS) + bkup = RunnerClass(12345, user="user", password="password") + self.assertEqual(bkup.command, + XTRA_BACKUP + PIPE + ZIP + PIPE + ENCRYPT) + self.assertEqual(bkup.manifest, "12345.xbstream.gz.enc") + + def test_backup_decrypted_mysqldump_command(self): + backupBase.BackupRunner.is_zipped = True + backupBase.BackupRunner.is_encrypted = False + RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS) + bkup = RunnerClass(12345, user="user", password="password") + self.assertEqual(bkup.command, SQLDUMP_BACKUP + PIPE + ZIP) + self.assertEqual(bkup.manifest, "12345.gz") + + def test_backup_encrypted_mysqldump_command(self): + backupBase.BackupRunner.is_zipped = True + backupBase.BackupRunner.is_encrypted = True + backupBase.BackupRunner.encrypt_key = CRYPTO_KEY + RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS) + bkup = RunnerClass(12345, user="user", password="password") + self.assertEqual(bkup.command, + SQLDUMP_BACKUP + PIPE + ZIP + PIPE + ENCRYPT) + self.assertEqual(bkup.manifest, "12345.gz.enc") + + def test_restore_decrypted_xtrabackup_command(self): + restoreBase.RestoreRunner.is_zipped = True + restoreBase.RestoreRunner.is_encrypted = False + RunnerClass = utils.import_class(RESTORE_XTRA_CLS) + restr = RunnerClass(None, restore_location="/var/lib/mysql") + self.assertEqual(restr.restore_cmd, UNZIP + PIPE + XTRA_RESTORE) + self.assertEqual(restr.prepare_cmd, PREPARE) + + def test_restore_encrypted_xtrabackup_command(self): + restoreBase.RestoreRunner.is_zipped = True + restoreBase.RestoreRunner.is_encrypted = True + restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY + RunnerClass = utils.import_class(RESTORE_XTRA_CLS) + restr = RunnerClass(None, restore_location="/var/lib/mysql") + self.assertEqual(restr.restore_cmd, + DECRYPT + PIPE + UNZIP + PIPE + XTRA_RESTORE) + self.assertEqual(restr.prepare_cmd, PREPARE) + + def test_restore_decrypted_mysqldump_command(self): + restoreBase.RestoreRunner.is_zipped = True + restoreBase.RestoreRunner.is_encrypted = False + RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS) + restr = RunnerClass(None, restore_location="/var/lib/mysql", + user="user", password="password") + self.assertEqual(restr.restore_cmd, UNZIP + PIPE + SQLDUMP_RESTORE) + self.assertIsNone(restr.prepare_cmd) + + def test_restore_encrypted_mysqldump_command(self): + restoreBase.RestoreRunner.is_zipped = True + restoreBase.RestoreRunner.is_encrypted = True + restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY + RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS) + restr = RunnerClass(None, restore_location="/var/lib/mysql", + user="user", password="password") + self.assertEqual(restr.restore_cmd, + DECRYPT + PIPE + UNZIP + PIPE + SQLDUMP_RESTORE) + self.assertIsNone(restr.prepare_cmd)