Adding range_backup_worker_enabled to DB config for recruiting range partitioned backup workers

fdbcli/ConfigureCommand.cpp

@@ -160,9 +160,14 @@ Future<bool> configureCommandActor(Reference<IDatabase> db,
 				result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED;
 				break;
 			}
+			if (it->startsWith("range_backup_worker_enabled:="_sr)) {
+				result = ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED;
+				break;
+			}
 		}
 
-		if (result != ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED) {
+		if (result != ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED &&
+		    result != ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED) {
 			ConfigurationResult r = co_await ManagementAPI::changeConfig(
 			    db, std::vector<StringRef>(tokens.begin() + startToken, tokens.end()), conf, force);
 			result = r;
@@ -287,6 +292,12 @@ Future<bool> configureCommandActor(Reference<IDatabase> db,
 		        "backup system.\n");
 		ret = false;
 		break;
+	case ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED:
+		fprintf(stderr,
+		        "ERROR: range_backup_worker_enabled configuration is restricted in fdbcli and managed "
+		        "automatically by the backup system.\n");
+		ret = false;
+		break;
 	default:
 		ASSERT(false);
 		ret = false;

fdbcli/FileConfigureCommand.cpp

@@ -80,6 +80,8 @@ Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
 	// This setting is now managed automatically by the backup system.
 	if (configString.find(" backup_worker_enabled:=") != std::string::npos) {
 		result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED;
+	} else if (configString.find(" range_backup_worker_enabled:=") != std::string::npos) {
+		result = ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED;
 	} else {
 		ConfigurationResult r = co_await ManagementAPI::changeConfig(db, configString, force);
 		result = r;
@@ -168,6 +170,12 @@ Future<bool> fileConfigureCommandActor(Reference<IDatabase> db,
 		        "backup system\n");
 		ret = false;
 		break;
+	case ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED:
+		fprintf(stderr,
+		        "ERROR: range_backup_worker_enabled configuration is restricted in fdbcli and managed "
+		        "automatically by the backup system\n");
+		ret = false;
+		break;
 	default:
 		ASSERT(false);
 		ret = false;

fdbclient/DatabaseConfiguration.cpp

@@ -52,6 +52,7 @@ void DatabaseConfiguration::resetInternal() {
 	remoteDesiredTLogCount = -1;
 	remoteTLogReplicationFactor = repopulateRegionAntiQuorum = 0;
 	backupWorkerEnabled = false;
+	rangeBackupWorkerEnabled = false;
 	perpetualStorageWiggleSpeed = 0;
 	perpetualStorageWiggleLocality = "0";
 	storageMigrationType = StorageMigrationType::DEFAULT;
@@ -430,6 +431,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
 	}
 
 	result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled;
+	result["range_backup_worker_enabled"] = (int32_t)rangeBackupWorkerEnabled;
 	result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed;
 	result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality;
 	if (perpetualStoreType.storeType() != KeyValueStoreType::END) {
@@ -700,6 +702,9 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	} else if (ck == "backup_worker_enabled"_sr) {
 		parse((&type), value);
 		backupWorkerEnabled = (type != 0);
+	} else if (ck == "range_backup_worker_enabled"_sr) {
+		parse((&type), value);
+		rangeBackupWorkerEnabled = (type != 0);
 	} else if (ck == "usable_regions"_sr) {
 		parse(&usableRegions, value);
 	} else if (ck == "repopulate_anti_quorum"_sr) {

fdbclient/FileBackupAgent.cpp

@@ -4236,16 +4236,13 @@ struct StartFullBackupTaskFunc : BackupTaskFuncBase {
 			co_await tr->onError(err);
 		}
 
-		// Check if backup worker is enabled
-		DatabaseConfiguration dbConfig = co_await getDatabaseConfiguration(cx);
-		bool backupWorkerEnabled = dbConfig.backupWorkerEnabled;
-		if (!backupWorkerEnabled && mutationLogType.get().present() &&
-		    mutationLogType.get().get() == MutationLogType::PARTITIONED_LOG) {
-			// Change configuration only when we set to use partitioned logs and
-			// the flag was not set before.
-			co_await ManagementAPI::changeConfig(cx.getReference(), "backup_worker_enabled:=1", true);
-			backupWorkerEnabled = true;
-			// the user is responsible for manually disabling backup worker after the last backup is done
+		// Enable the appropriate backup worker type if not already enabled.
+		if (mutationLogType.get().present()) {
+			if (mutationLogType.get().get() == MutationLogType::PARTITIONED_LOG) {
+				co_await enableBackupWorker(cx);
+			} else if (mutationLogType.get().get() == MutationLogType::RANGE_PARTITIONED_LOG) {
+				co_await enableRangeBackupWorker(cx);
+			}
 		}
 
 		// Get start version after backup worker are enabled
@@ -7589,6 +7586,15 @@ class FileBackupAgentImpl {
 		co_return;
 	}
 
+	static Future<Void> checkAndDisableRangeBackupWorkers(Database cx) {
+		bool running = co_await runRYWTransaction(
+		    cx, [=](Reference<ReadYourWritesTransaction> tr) { return anyRangePartitionedBackupRunning(tr); });
+		if (!running) {
+			co_await disableRangeBackupWorker(cx);
+		}
+		co_return;
+	}
+
 	static Future<Void> changePause(FileBackupAgent* backupAgent, Database db, bool pause) {
 		Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(db));
 
@@ -8693,6 +8699,10 @@ Future<Void> FileBackupAgent::checkAndDisableBackupWorkers(Database cx) {
 	return FileBackupAgentImpl::checkAndDisableBackupWorkers(cx);
 }
 
+Future<Void> FileBackupAgent::checkAndDisableRangeBackupWorkers(Database cx) {
+	return FileBackupAgentImpl::checkAndDisableRangeBackupWorkers(cx);
+}
+
 Future<std::string> FileBackupAgent::getStatus(Database cx, ShowErrors showErrors, std::string tagName) {
 	return FileBackupAgentImpl::getStatus(this, cx, showErrors, tagName);
 }

fdbclient/ManagementAPI.cpp

@@ -503,6 +503,34 @@ Future<Void> enableBackupWorker(Database cx) {
 	}
 }
 
+Future<Void> enableRangeBackupWorker(Database cx) {
+	DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
+	if (configuration.rangeBackupWorkerEnabled) {
+		TraceEvent("RangeBackupWorkerAlreadyEnabled");
+		co_return;
+	}
+	ConfigurationResult res =
+	    co_await ManagementAPI::changeConfig(cx.getReference(), "range_backup_worker_enabled:=1", true);
+	if (res != ConfigurationResult::SUCCESS) {
+		TraceEvent("RangeBackupWorkerEnableFailed").detail("Result", res);
+		throw operation_failed();
+	}
+}
+
+Future<Void> disableRangeBackupWorker(Database cx) {
+	DatabaseConfiguration configuration = co_await getDatabaseConfiguration(cx);
+	if (!configuration.rangeBackupWorkerEnabled) {
+		TraceEvent("RangeBackupWorkerAlreadyDisabled");
+		co_return;
+	}
+	ConfigurationResult res =
+	    co_await ManagementAPI::changeConfig(cx.getReference(), "range_backup_worker_enabled:=0", true);
+	if (res != ConfigurationResult::SUCCESS) {
+		TraceEvent("RangeBackupWorkerDisableFailed").detail("Result", res);
+		throw operation_failed();
+	}
+}
+
 Future<DatabaseConfiguration> getDatabaseConfiguration(Transaction* tr, bool useSystemPriority) {
 	if (useSystemPriority) {
 		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);

fdbclient/Schemas.cpp

@@ -869,6 +869,7 @@ const KeyRef JSONSchemas::statusSchema = R"statusSchema(
          "grv_proxies":1,
          "proxies":6,
          "backup_worker_enabled":1,
+         "range_backup_worker_enabled":1,
          "perpetual_storage_wiggle":0,
          "perpetual_storage_wiggle_locality":"0",
          "perpetual_storage_wiggle_engine":{

fdbclient/SystemData.cpp

@@ -920,6 +920,7 @@ const KeyRangeRef configKeys("\xff/conf/"_sr, "\xff/conf0"_sr);
 const KeyRef configKeysPrefix = configKeys.begin;
 
 const KeyRef backupWorkerEnabledKey("\xff/conf/backup_worker_enabled"_sr);
+const KeyRef rangeBackupWorkerEnabledKey("\xff/conf/range_backup_worker_enabled"_sr);
 const KeyRef perpetualStorageWiggleKey("\xff/conf/perpetual_storage_wiggle"_sr);
 const KeyRef perpetualStorageWiggleLocalityKey("\xff/conf/perpetual_storage_wiggle_locality"_sr);
 // The below two are there for compatible upgrade and downgrade. After 7.3, the perpetual wiggle related keys should use

fdbclient/include/fdbclient/BackupAgent.h

@@ -317,7 +317,7 @@ class FileBackupAgent : public BackupAgentBase {
 	Future<Void> discontinueBackup(Database cx, Key tagName) {
 		return runRYWTransaction(
 		           cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); }) +
-		       checkAndDisableBackupWorkers(cx);
+		       checkAndDisableBackupWorkers(cx) + checkAndDisableRangeBackupWorkers(cx);
 	}
 
 	// Terminate an ongoing backup, without waiting for the backup to finish.
@@ -332,12 +332,15 @@ class FileBackupAgent : public BackupAgentBase {
 		// First abort the backup, then check and disable backup workers if needed.
 		return runRYWTransaction(cx,
 		                         [=](Reference<ReadYourWritesTransaction> tr) { return abortBackup(tr, tagName); }) +
-		       checkAndDisableBackupWorkers(cx);
+		       checkAndDisableBackupWorkers(cx) + checkAndDisableRangeBackupWorkers(cx);
 	}
 
 	// Disable backup workers if no active partitioned backup is running.
 	Future<Void> checkAndDisableBackupWorkers(Database cx);
 
+	// Disable range backup workers if no active range-partitioned backup is running.
+	Future<Void> checkAndDisableRangeBackupWorkers(Database cx);
+
 	Future<std::string> getStatus(Database cx, ShowErrors, std::string tagName);
 	Future<std::string> getStatusJSON(Database cx, std::string tagName);
 

fdbclient/include/fdbclient/DatabaseConfiguration.h

@@ -209,6 +209,7 @@ struct DatabaseConfiguration {
 
 	// Backup Workers
 	bool backupWorkerEnabled;
+	bool rangeBackupWorkerEnabled;
 
 	// Data centers
 	int32_t usableRegions; // Number of regions which have a replica of the database.

fdbclient/include/fdbclient/GenericManagementAPI.h

@@ -64,7 +64,8 @@ enum class ConfigurationResult {
 	DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL,
 	DATABASE_IS_REGISTERED,
 	INVALID_STORAGE_TYPE,
-	BACKUP_WORKER_ENABLED_RESTRICTED
+	BACKUP_WORKER_ENABLED_RESTRICTED,
+	RANGE_BACKUP_WORKER_ENABLED_RESTRICTED
 };
 
 enum class CoordinatorsResult {
@@ -435,11 +436,17 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
 					}
 				}
 
-				// Clear backup progress when backup workers are disabled
+				// Clear partitioned backup progress when backup workers are disabled
 				if (i->first == backupWorkerEnabledKey && i->second == "0") {
 					tr->clear(backupProgressKeys);
 					TraceEvent("BackupWorkerProgressCleared");
 				}
+
+				// Clear range partitioned backup progress when range partitioned backup workers are disabled
+				if (i->first == rangeBackupWorkerEnabledKey && i->second == "0") {
+					tr->clear(backupProgressKeys);
+					TraceEvent("RangePartitionedBackupWorkerProgressCleared");
+				}
 			}
 
 			if (!creating && resetPPWStats) {

fdbclient/include/fdbclient/ManagementAPI.h

@@ -415,3 +415,5 @@ Future<Void> mgmtSnapCreate(Database cx, Standalone<StringRef> snapCmd, UID snap
 
 Future<Void> disableBackupWorker(Database cx);
 Future<Void> enableBackupWorker(Database cx);
+Future<Void> disableRangeBackupWorker(Database cx);
+Future<Void> enableRangeBackupWorker(Database cx);

fdbclient/include/fdbclient/SystemData.h

@@ -318,6 +318,7 @@ extern const KeyRangeRef configKeys;
 extern const KeyRef configKeysPrefix;
 
 extern const KeyRef backupWorkerEnabledKey;
+extern const KeyRef rangeBackupWorkerEnabledKey;
 extern const KeyRef perpetualStorageWiggleKey;
 extern const KeyRef perpetualStorageWiggleLocalityKey;
 extern const KeyRef perpetualStorageWiggleIDPrefix;

fdbserver/backupworker/BackupWorkerRangePartitioned.cpp

@@ -848,7 +848,7 @@ Future<Void> saveProgress(BackupRangePartitionedData* self, Version backupVersio
 			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 
 			// CHECK: Don't save progress if backup workers are disabled
-			Optional<Value> backupWorkerEnabled = co_await tr.get(backupWorkerEnabledKey);
+			Optional<Value> backupWorkerEnabled = co_await tr.get(rangeBackupWorkerEnabledKey);
 			if (!backupWorkerEnabled.present() || backupWorkerEnabled.get() == "0"_sr) {
 				TraceEvent("BWRangePartitionedProgressSkipped", self->myId).detail("Reason", "BackupWorkersDisabled");
 				co_return;