prototype log version 8.0

CMakeLists.txt

@@ -24,7 +24,7 @@ if(POLICY CMP0135)
 endif()
 
 project(foundationdb
-  VERSION 7.4.0
+  VERSION 7.5.0
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM Swift)

fdbclient/include/fdbclient/FDBTypes.h

@@ -1024,6 +1024,7 @@ struct TLogVersion {
 		// V5 merged reference and value spilling
 		// V6 added span context to list of serialized mutations sent from proxy to tlogs
 		// V7 use xxhash3 for TLog checksum
+		// V8 version vector disk queue changes
 		// V1 = 1,  // 4.6 is dispatched to via 6.0
 		V2 = 2, // 6.0
 		V3 = 3, // 6.1
@@ -1031,10 +1032,11 @@ struct TLogVersion {
 		V5 = 5, // 6.3
 		V6 = 6, // 7.0
 		V7 = 7, // 7.2
+		V8 = 8,
 		MIN_SUPPORTED = V5,
-		MAX_SUPPORTED = V7,
+		MAX_SUPPORTED = V8,
 		MIN_RECRUITABLE = V6,
-		DEFAULT = V7,
+		DEFAULT = V8,
 	} version;
 
 	TLogVersion() : version(UNSET) {}
@@ -1062,6 +1064,8 @@ struct TLogVersion {
 			return V6;
 		if (s == "7"_sr)
 			return V7;
+		if (s == "8"_sr)
+			return V8;
 		return default_error_or();
 	}
 };

fdbserver/include/fdbserver/WorkerInterface.actor.h

@@ -1418,6 +1418,25 @@ Future<T> ioDegradedOrTimeoutError(Future<T> what,
 	}
 }
 
+namespace oldTLog_7_0 {
+ACTOR Future<Void> tLog(IKeyValueStore* persistentData,
+                        IDiskQueue* persistentQueue,
+                        Reference<AsyncVar<ServerDBInfo> const> db,
+                        LocalityData locality,
+                        PromiseStream<InitializeTLogRequest> tlogRequests,
+                        UID tlogId,
+                        UID workerID,
+                        bool restoreFromDisk,
+                        Promise<Void> oldLog,
+                        Promise<Void> recovered,
+                        std::string folder,
+                        Reference<AsyncVar<bool>> degraded,
+                        Reference<AsyncVar<UID>> activeSharedTLog,
+                        Reference<AsyncVar<bool>> enablePrimaryTxnSystemHealthCheck);
+}
+
+typedef decltype(&tLog) TLogFn;
+
 #include "fdbserver/ServerDBInfo.h"
 #include "flow/unactorcompiler.h"
 #endif

fdbserver/worker.actor.cpp

@@ -458,6 +458,7 @@ struct TLogOptions {
 		case TLogVersion::V5:
 		case TLogVersion::V6:
 		case TLogVersion::V7:
+		case TLogVersion::V8:
 			toReturn = "V_" + boost::lexical_cast<std::string>(version);
 			break;
 		}
@@ -486,6 +487,8 @@ TLogFn tLogFnForOptions(TLogOptions options) {
 	case TLogVersion::V5:
 	case TLogVersion::V6:
 	case TLogVersion::V7:
+		return oldTLog_7_0::tLog;
+	case TLogVersion::V8:
 		return tLog;
 	default:
 		ASSERT(false);

fdbserver/workloads/ConfigureDatabase.actor.cpp

@@ -47,7 +47,8 @@ static const char* logTypes[] = { "log_engine:=1",
 	                              "log_version:=5",
 	                              "log_version:=6",
 	                              // downgrade incompatible log version
-	                              "log_version:=7" };
+	                              "log_version:=7",
+	                              "log_version:=8" };
 static const char* redundancies[] = { "single", "double", "triple" };
 static const char* backupTypes[] = { "backup_worker_enabled:=0", "backup_worker_enabled:=1" };
 

tests/CMakeLists.txt

@@ -394,6 +394,12 @@ if(WITH_PYTHON)
   add_fdb_test(
     TEST_FILES restarting/from_7.3.5_until_7.3.29/ClientTransactionProfilingCorrectness-1.toml
     restarting/from_7.3.5_until_7.3.29/ClientTransactionProfilingCorrectness-2.toml)
+  add_fdb_test(
+    TEST_FILES restarting/from_7.4.0/ClientTransactionProfilingCorrectness-1.toml
+    restarting/from_7.4.0/ClientTransactionProfilingCorrectness-2.toml)
+  add_fdb_test(
+    TEST_FILES restarting/from_7.4.0/CycleTestRestart-1.toml
+    restarting/from_7.4.0/CycleTestRestart-2.toml)
   add_fdb_test(
     TEST_FILES restarting/from_7.3.29/ClientTransactionProfilingCorrectness-1.toml
     restarting/from_7.3.29/ClientTransactionProfilingCorrectness-2.toml)

tests/restarting/from_7.4.0/ClientTransactionProfilingCorrectness-1.toml

@@ -0,0 +1,40 @@
+[configuration]
+storageEngineExcludeTypes=[3]
+tenantModes = ['disabled']
+
+[[knobs]]
+dd_physical_shard_move_probability = 0
+
+[[test]]
+testTitle="ClientTransactionProfilingCorrectness"
+clearAfterTest=false
+runSetup=true
+timeout=2100
+
+    [[test.workload]]
+    testName="ApiCorrectness"
+    numKeys=5000
+    onlyLowerCase=true
+    shortKeysRatio=0.5
+    minShortKeyLength=1
+    maxShortKeyLength=3
+    minLongKeyLength=1
+    maxLongKeyLength=128
+    minValueLength=1
+    maxValueLength=1000
+    numGets=1000
+    numGetRanges=100
+    numGetRangeSelectors=100
+    numGetKeys=100
+    numClears=100
+    numClearRanges=10
+    maxTransactionBytes=500000
+    randomTestDuration=30
+
+    [[test.workload]]
+    testName="ClientTransactionProfileCorrectness"
+
+    [[test.workload]]
+    testName="SaveAndKill"
+    restartInfoLocation="simfdb/restartInfo.ini"
+    testDuration=60

tests/restarting/from_7.4.0/ClientTransactionProfilingCorrectness-2.toml

@@ -0,0 +1,29 @@
+[[test]]
+testTitle="ClientTransactionProfilingCorrectness"
+clearAfterTest=true
+timeout=2100
+runSetup=true
+
+    [[test.workload]]
+    testName="ApiCorrectness"
+    numKeys=5000
+    onlyLowerCase=true
+    shortKeysRatio=0.5
+    minShortKeyLength=1
+    maxShortKeyLength=3
+    minLongKeyLength=1
+    maxLongKeyLength=128
+    minValueLength=1
+    maxValueLength=1000
+    numGets=1000
+    numGetRanges=100
+    numGetRangeSelectors=100
+    numGetKeys=100
+    numClears=100
+    numClearRanges=10
+    maxTransactionBytes=500000
+    randomTestDuration=60
+
+    [[test.workload]]
+    testName="ClientTransactionProfileCorrectness"
+

tests/restarting/from_7.4.0/CycleTestRestart-1.toml

@@ -0,0 +1,46 @@
+[configuration]
+maxTLogVersion = 7
+storageEngineExcludeTypes=[3]
+tenantModes = ['disabled']
+
+[[knobs]]
+dd_physical_shard_move_probability = 0
+
+[[test]]
+testTitle="Clogged"
+clearAfterTest=false 
+
+    [[test.workload]]
+    testName="Cycle"
+    transactionsPerSecond=500.0
+    nodeCount=2500
+    testDuration=10.0
+    expectedRate=0
+
+    [[test.workload]]
+    testName="RandomClogging"
+    testDuration=10.0
+
+    [[test.workload]]
+    testName="Rollback"
+    meanDelay=10.0
+    testDuration=10.0
+
+    [[test.workload]]
+    testName="Attrition"
+    machinesToKill=10
+    machinesToLeave=3
+    reboot=true
+    testDuration=10.0
+    
+    [[test.workload]]
+    testName="Attrition"
+    machinesToKill=10
+    machinesToLeave=3
+    reboot=true
+    testDuration=10.0
+    
+    [[test.workload]]
+    testName="SaveAndKill"
+    restartInfoLocation="simfdb/restartInfo.ini"
+    testDuration=10.0

tests/restarting/from_7.4.0/CycleTestRestart-2.toml

@@ -0,0 +1,35 @@
+[configuration]
+maxTLogVersion=8
+[[test]]
+testTitle="Clogged"
+runSetup=false
+
+    [[test.workload]]
+    testName="Cycle"
+    transactionsPerSecond=2500.0
+    nodeCount=2500
+    testDuration=10.0
+    expectedRate=0
+
+    [[test.workload]]
+    testName="RandomClogging"
+    testDuration=10.0
+
+    [[test.workload]]
+    testName="Rollback"
+    meanDelay=10.0
+    testDuration=10.0
+
+    [[test.workload]]
+    testName="Attrition"
+    machinesToKill=10
+    machinesToLeave=3
+    reboot=true
+    testDuration=10.0
+
+    [[test.workload]]
+    testName="Attrition"
+    machinesToKill=10
+    machinesToLeave=3
+    reboot=true
+    testDuration=10.0