[DRAFT] Enable Backup/Restore Encryption/Decryption in Simulation Tests

fdbclient/BackupContainerFileSystem.actor.cpp

@@ -1267,6 +1267,12 @@ class BackupContainerFileSystemImpl {
 	}
 
 	ACTOR static Future<Void> createTestEncryptionKeyFile(std::string filename) {
+		if (fileExists(filename)) {
+			// Key file already exists, don't overwrite it -> only for testing between backup and restore workloads to
+			// share the key.
+			TraceEvent("EncryptionKeyFileExists").detail("FileName", filename);
+			return Void();
+		}
 		state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(
 		    filename,
 		    IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE,

fdbclient/FileBackupAgent.actor.cpp

@@ -7726,7 +7726,8 @@ class FileBackupAgentImpl {
 			throw restore_error();
 		}
 
-		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(url.toString(), proxy, {});
+		state Reference<IBackupContainer> bc =
+		    IBackupContainer::openContainer(url.toString(), proxy, encryptionKeyFileName);
 
 		state BackupDescription desc = wait(bc->describeBackup(true));
 		if (cxOrig.present()) {
@@ -7909,6 +7910,7 @@ class FileBackupAgentImpl {
 		}
 
 		state Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx.getReference()));
+
 		bc = fileBackup::getBackupContainerWithProxy(bc);
 
 		if (fastRestore) {

fdbserver/workloads/AtomicRestore.actor.cpp

@@ -21,6 +21,7 @@
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
+#include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/RestoreCommon.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
@@ -37,6 +38,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 	UsePartitionedLog usePartitionedLogs{ false };
 	Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix
 	// CAVEAT: When removePrefix is used, we must ensure every key in backup have the removePrefix
+	Optional<std::string> encryptionKeyFileName;
 
 	AtomicRestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 
@@ -55,6 +57,10 @@ struct AtomicRestoreWorkload : TestWorkload {
 		addPrefix = getOption(options, "addPrefix"_sr, ""_sr);
 		removePrefix = getOption(options, "removePrefix"_sr, ""_sr);
 
+		if (getOption(options, "encrypted"_sr, true)) {
+			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
+		}
+
 		// Correctness is not clean for addPrefix feature yet. Uncomment below to enable the test
 		// Generate addPrefix
 		// if (addPrefix.size() == 0 && removePrefix.size() == 0) {
@@ -95,6 +101,10 @@ struct AtomicRestoreWorkload : TestWorkload {
 		wait(delay(self->startAfter * deterministicRandom()->random01()));
 		TraceEvent("AtomicRestore_Start").detail("UsePartitionedLog", self->usePartitionedLogs);
 
+		if (self->encryptionKeyFileName.present()) {
+			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		}
+
 		state std::string backupContainer = "file://simfdb/backups/";
 		try {
 			wait(backupAgent.submitBackup(cx,
@@ -106,7 +116,9 @@ struct AtomicRestoreWorkload : TestWorkload {
 			                              self->backupRanges,
 			                              true,
 			                              StopWhenDone::False,
-			                              self->usePartitionedLogs));
+			                              self->usePartitionedLogs,
+			                              IncrementalBackupOnly::False,
+			                              self->encryptionKeyFileName));
 		} catch (Error& e) {
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
 				throw;

fdbserver/workloads/Backup.actor.cpp

@@ -74,7 +74,7 @@ struct BackupWorkload : TestWorkload {
 		std::vector<std::string> restorePrefixesToInclude =
 		    getOption(options, "restorePrefixesToInclude"_sr, std::vector<std::string>());
 
-		if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.1)) {
+		if (getOption(options, "encrypted"_sr, true)) {
 			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
 		}
 

fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp

@@ -21,6 +21,7 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbserver/RestoreWorkerInterface.actor.h"
 #include "fdbclient/RunRYWTransaction.actor.h"
@@ -47,6 +48,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 	UsePartitionedLog usePartitionedLogs{ false };
 	Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix
 	// CAVEAT: When removePrefix is used, we must ensure every key in backup have the removePrefix
+	Optional<std::string> encryptionKeyFileName;
 
 	std::map<Standalone<KeyRef>, Standalone<ValueRef>> dbKVs;
 
@@ -85,6 +87,10 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		addPrefix = getOption(options, "addPrefix"_sr, ""_sr);
 		removePrefix = getOption(options, "removePrefix"_sr, ""_sr);
 
+		if (getOption(options, "encrypted"_sr, true)) {
+			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
+		}
+
 		KeyRef beginRange;
 		KeyRef endRange;
 		UID randomID = nondeterministicRandom()->randomUniqueID();
@@ -159,6 +165,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 		TraceEvent(SevInfo, "BARW_Param").detail("DifferentialBackup", differentialBackup);
 		TraceEvent(SevInfo, "BARW_Param").detail("StopDifferentialAfter", stopDifferentialAfter);
 		TraceEvent(SevInfo, "BARW_Param").detail("AgentRequest", agentRequest);
+		TraceEvent(SevInfo, "BARW_Param").detail("Encrypted", encryptionKeyFileName.present());
 
 		return _start(cx, this);
 	}
@@ -230,7 +237,9 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 			                               backupRanges,
 			                               true,
 			                               StopWhenDone{ !stopDifferentialDelay },
-			                               self->usePartitionedLogs));
+			                               self->usePartitionedLogs,
+			                               IncrementalBackupOnly::False,
+			                               self->encryptionKeyFileName));
 		} catch (Error& e) {
 			TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
 			if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
@@ -388,7 +397,11 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				                                  normalKeys,
 				                                  Key(),
 				                                  Key(),
-				                                  self->locked)));
+				                                  self->locked,
+				                                  OnlyApplyMutationLogs::False,
+				                                  InconsistentSnapshotOnly::False,
+				                                  ::invalidVersion,
+				                                  self->encryptionKeyFileName)));
 				TraceEvent(SevError, "BARW_RestoreAllowedOverwrittingDatabase", randomID).log();
 				ASSERT(false);
 			} catch (Error& e) {
@@ -425,6 +438,10 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 			BackupAndParallelRestoreCorrectnessWorkload::backupAgentRequests++;
 		}
 
+		if (self->encryptionKeyFileName.present()) {
+			wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+		}
+
 		try {
 			state Future<Void> startRestore = delay(self->restoreAfter);
 
@@ -489,7 +506,9 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 					                                       self->backupRanges,
 					                                       true,
 					                                       StopWhenDone::True,
-					                                       UsePartitionedLog::False);
+					                                       UsePartitionedLog::False,
+					                                       IncrementalBackupOnly::False,
+					                                       self->encryptionKeyFileName);
 				} catch (Error& e) {
 					TraceEvent("BARW_SubmitBackup2Exception", randomID)
 					    .error(e)
@@ -529,8 +548,9 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload {
 				    .detail("BackupTag", printable(self->backupTag));
 				// start restoring
 
-				auto container =
-				    IBackupContainer::openContainer(lastBackupContainer->getURL(), lastBackupContainer->getProxy(), {});
+				auto container = IBackupContainer::openContainer(lastBackupContainer->getURL(),
+				                                                 lastBackupContainer->getProxy(),
+				                                                 lastBackupContainer->getEncryptionKeyFileName());
 				BackupDescription desc = wait(container->describeBackup());
 				ASSERT(self->usePartitionedLogs == desc.partitioned);
 				ASSERT(desc.minRestorableVersion.present()); // We must have a valid backup now.

fdbserver/workloads/BackupCorrectness.actor.cpp

@@ -93,7 +93,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
 		    getOption(options, "restorePrefixesToInclude"_sr, std::vector<std::string>());
 
 		shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false;
-		if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.1)) {
+		if (getOption(options, "encrypted"_sr, true)) {
 			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
 		}
 

fdbserver/workloads/BackupCorrectnessPartitioned.actor.cpp

@@ -88,7 +88,7 @@ struct BackupAndRestorePartitionedCorrectnessWorkload : TestWorkload {
 		    getOption(options, "restorePrefixesToInclude"_sr, std::vector<std::string>());
 
 		shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false;
-		if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.1)) {
+		if (getOption(options, "encrypted"_sr, true)) {
 			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
 		}
 

fdbserver/workloads/IncrementalBackup.actor.cpp

@@ -27,9 +27,12 @@
 #include "fdbrpc/simulator.h"
 #include "fdbclient/BackupAgent.actor.h"
 #include "fdbclient/BackupContainer.h"
+#include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbserver/Knobs.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/Arena.h"
+#include "flow/Platform.h"
+#include "flow/Trace.h"
 #include "flow/serialize.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -47,6 +50,7 @@ struct IncrementalBackupWorkload : TestWorkload {
 	bool checkBeginVersion;
 	bool clearBackupAgentKeys;
 	Standalone<StringRef> blobManifestUrl;
+	Optional<std::string> encryptionKeyFileName;
 
 	IncrementalBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		backupDir = getOption(options, "backupDir"_sr, "file://simfdb/backups/"_sr);
@@ -59,6 +63,10 @@ struct IncrementalBackupWorkload : TestWorkload {
 		checkBeginVersion = getOption(options, "checkBeginVersion"_sr, false);
 		clearBackupAgentKeys = getOption(options, "clearBackupAgentKeys"_sr, false);
 		blobManifestUrl = getOption(options, "blobManifestUrl"_sr, ""_sr);
+
+		if (getOption(options, "encrypted"_sr, true)) {
+			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
+		}
 	}
 
 	Future<Void> setup(Database const& cx) override { return Void(); }
@@ -99,6 +107,12 @@ struct IncrementalBackupWorkload : TestWorkload {
 				TraceEvent("IBackupWaitContainer").log();
 				wait(success(self->backupAgent.waitBackup(
 				    cx, self->tag.toString(), StopWhenDone::False, &backupContainer, &backupUID)));
+
+				state Optional<std::string> restoreEncryptionKeyFileName;
+				if (self->encryptionKeyFileName.present() && fileExists(self->encryptionKeyFileName.get())) {
+					restoreEncryptionKeyFileName = self->encryptionKeyFileName.get();
+				}
+
 				if (!backupContainer.isValid()) {
 					TraceEvent("IBackupCheckListContainersAttempt").log();
 					state std::vector<std::string> containers =
@@ -107,7 +121,8 @@ struct IncrementalBackupWorkload : TestWorkload {
 					    .detail("Size", containers.size())
 					    .detail("First", containers.front());
 					if (containers.size()) {
-						backupContainer = IBackupContainer::openContainer(containers.front(), {}, {});
+						backupContainer =
+						    IBackupContainer::openContainer(containers.front(), {}, restoreEncryptionKeyFileName);
 					}
 				}
 				state bool e = wait(backupContainer->exists());
@@ -157,6 +172,10 @@ struct IncrementalBackupWorkload : TestWorkload {
 		addDefaultBackupRanges(backupRanges);
 
 		if (self->submitOnly) {
+			if (self->encryptionKeyFileName.present()) {
+				wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
+			}
+
 			TraceEvent("IBackupSubmitAttempt").log();
 			try {
 				Optional<std::string> blobManifestUrl;
@@ -174,7 +193,7 @@ struct IncrementalBackupWorkload : TestWorkload {
 				                                    StopWhenDone::False,
 				                                    UsePartitionedLog::False,
 				                                    IncrementalBackupOnly::True,
-				                                    {},
+				                                    self->encryptionKeyFileName,
 				                                    blobManifestUrl));
 			} catch (Error& e) {
 				TraceEvent("IBackupSubmitError").error(e);
@@ -205,6 +224,12 @@ struct IncrementalBackupWorkload : TestWorkload {
 			state Version beginVersion = invalidVersion;
 			wait(success(self->backupAgent.waitBackup(
 			    cx, self->tag.toString(), StopWhenDone::False, &backupContainer, &backupUID)));
+
+			state Optional<std::string> restoreEncryptionKeyFileName;
+			if (self->encryptionKeyFileName.present() && fileExists(self->encryptionKeyFileName.get())) {
+				restoreEncryptionKeyFileName = self->encryptionKeyFileName.get();
+			}
+
 			if (self->checkBeginVersion) {
 				TraceEvent("IBackupReadSystemKeys").log();
 				state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
@@ -275,7 +300,8 @@ struct IncrementalBackupWorkload : TestWorkload {
 				                                       UnlockDB::True,
 				                                       OnlyApplyMutationLogs::True,
 				                                       InconsistentSnapshotOnly::False,
-				                                       beginVersion)));
+				                                       beginVersion,
+				                                       restoreEncryptionKeyFileName)));
 			}
 			TraceEvent("IBackupRestoreAttempt").detail("BeginVersion", beginVersion);
 			wait(success(self->backupAgent.restore(cx,
@@ -293,7 +319,8 @@ struct IncrementalBackupWorkload : TestWorkload {
 			                                       UnlockDB::True,
 			                                       OnlyApplyMutationLogs::True,
 			                                       InconsistentSnapshotOnly::False,
-			                                       beginVersion)));
+			                                       beginVersion,
+			                                       restoreEncryptionKeyFileName)));
 			TraceEvent("IBackupRestoreSuccess").log();
 		}
 		return Void();

fdbserver/workloads/Restore.actor.cpp

@@ -60,7 +60,7 @@ struct RestoreWorkload : TestWorkload {
 		    getOption(options, "restorePrefixesToInclude"_sr, std::vector<std::string>());
 
 		shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false;
-		if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.1)) {
+		if (getOption(options, "encrypted"_sr, true)) {
 			encryptionKeyFileName = "simfdb/" + getTestEncryptionFileName();
 		}