Merge pull request #1641 from Expensify/tyler-cluster-test-sync

Update tester to allow forcing ordered commands.
Expensify · Feb 9, 2024 · a08c51a · a08c51a
2 parents 482a4bd + 8c7a966
commit a08c51a
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 4 deletions.
diff --git a/test/clustertest/BedrockClusterTester.h b/test/clustertest/BedrockClusterTester.h
@@ -35,6 +35,8 @@ class ClusterTester {
     // Stops a given node.
     void stopNode(size_t index);
 
+    atomic<uint64_t> groupCommitCount{0};
+
   private:
 
     // The number of nodes in the cluster.
@@ -140,7 +142,7 @@ ClusterTester<T>::ClusterTester(ClusterSize size,
         }
 
         // And add the new entry in the map.
-        _cluster.emplace_back(args, queries, serverPort, nodePort, controlPort, false, processPath);
+        _cluster.emplace_back(args, queries, serverPort, nodePort, controlPort, false, processPath, &groupCommitCount);
     }
 
     // Now start them all.

diff --git a/test/lib/BedrockTester.cpp b/test/lib/BedrockTester.cpp
@@ -40,11 +40,13 @@ BedrockTester::BedrockTester(const map<string, string>& args,
                              uint16_t nodePort,
                              uint16_t controlPort,
                              bool startImmediately,
-                             const string& bedrockBinary) :
+                             const string& bedrockBinary,
+                             atomic<uint64_t>* alternateCounter) :
     _serverPort(serverPort ?: ports.getPort()),
     _nodePort(nodePort ?: ports.getPort()),
     _controlPort(controlPort ?: ports.getPort()),
-    _commandPortPrivate(ports.getPort())
+    _commandPortPrivate(ports.getPort()),
+    _commitCount(alternateCounter ? *alternateCounter : _commitCountBase)
 {
     {
         lock_guard<decltype(_testersMutex)> lock(_testersMutex);
@@ -396,6 +398,9 @@ vector<SData> BedrockTester::executeWaitMultipleData(vector<SData> requests, int
                         break;
                     } else {
                         myRequest = requests[myIndex];
+                        if (_enforceCommandOrder) {
+                            myRequest["commitCount"] = to_string(_commitCount);
+                        }
                     }
 
                     // Reset this for the next request that might need it.
@@ -490,6 +495,12 @@ vector<SData> BedrockTester::executeWaitMultipleData(vector<SData> requests, int
                     } else if (!timedOut) {
                         // Ok, done, let's lock again and insert this in the results.
                         SData responseData;
+                        if (headers.find("commitCount") != headers.end()) {
+                            uint64_t newCommitCount = SToUInt64(headers["commitCount"]);
+                            if (newCommitCount > _commitCount) {
+                                _commitCount = newCommitCount;
+                            }
+                        }
                         responseData.nameValueMap = headers;
                         responseData.methodLine = methodLine;
                         responseData.content = content;
@@ -611,3 +622,7 @@ int BedrockTester::getPID() const
 {
     return _serverPID;
 }
+
+void BedrockTester::setEnforceCommandOrder(bool enforce) {
+    _enforceCommandOrder = enforce;
+}
diff --git a/test/lib/BedrockTester.h b/test/lib/BedrockTester.h
@@ -32,7 +32,8 @@ class BedrockTester {
                   uint16_t nodePort = 0,
                   uint16_t controlPort = 0,
                   bool startImmediately = true,
-                  const string& bedrockBinary = "");
+                  const string& bedrockBinary = "",
+                  atomic<uint64_t>* alternateCounter = nullptr);
 
     // Destructor.
     ~BedrockTester();
@@ -47,6 +48,10 @@ class BedrockTester {
     // Shuts down all bedrock servers associated with any existing testers.
     static void stopAll();
 
+    // If enabled, commands will send the most recent `commitCount` received back from this BedrockTester with each new request,
+    // enforcing that prior commands finish before later commands.
+    void setEnforceCommandOrder(bool enforce);
+
     // Generate a temporary filename with an optional prefix. Used particularly to create new DB files for each server,
     // but can generally be used for any temporary file required.
     static string getTempFileName(string prefix = "");
@@ -120,5 +125,9 @@ class BedrockTester {
     uint16_t _nodePort;
     uint16_t _controlPort;
     uint16_t _commandPortPrivate;
+
+    bool _enforceCommandOrder = false;
+    atomic<uint64_t> _commitCountBase = 0;
+    atomic<uint64_t>& _commitCount;
 };