autest: --thread-limit 1 by default for Proxy Verifier

For production traffic simulation, Proxy Verifier by default runs
through the specified connections in parallel. But for our end to end
tests, that hardly ever makes sense. Rather we want the tests to
deterministically run through each specified connection sequentially.
This patch does so by passing --thread-limit 1 as a default option to
the Proxy Verifier client in the client extension. Many tests were
specifying this already anyway, so this saves some burden on client
configuration for each of the test descriptions.

Author: bneradt

tests/gold_tests/autest-site/verifier_client.test.ext

@@ -70,7 +70,7 @@ def _configure_client(
         # Create a copy of the replay directory in the run directory.
         run_replay_path = os.path.join(client_dir, os.path.basename(replay_path))
         process.Setup.Copy(replay_path, run_replay_path, CopyLogic.SoftFiles)
-        command += " {} ".format(run_replay_path)
+        command += f" {run_replay_path} "
 
     if not http_ports:
         http_ports = [8080]
@@ -103,12 +103,12 @@ def _configure_client(
             ssl_cert = os.path.join(obj.Variables["AtsTestToolsDir"], "proxy-verifier", "ssl", "client.pem")
 
             if not os.path.isfile(ssl_cert):
-                raise ValueError("Tried to use '{}' for --client-cert, but it is not "
-                                 "a valid file.".format(ssl_cert))
+                raise ValueError(f"Tried to use '{ssl_cert}' for --client-cert, but it is not "
+                                 "a valid file.")
         if ssl_cert:
             run_ssl_cert = os.path.join(client_dir, os.path.basename(ssl_cert))
             process.Setup.Copy(ssl_cert, run_ssl_cert, CopyLogic.SoftFiles)
-            command += ' --client-cert "{}" '.format(run_ssl_cert)
+            command += f' --client-cert "{run_ssl_cert}" '
 
             tls_secrets_log_filename = "tls_secrets.txt"
             tls_secrets_log_path = os.path.join(client_dir, tls_secrets_log_filename)
@@ -119,21 +119,26 @@ def _configure_client(
             ca_cert = os.path.join(obj.Variables["AtsTestToolsDir"], "proxy-verifier", "ssl", "ca.pem")
 
             if not os.path.isfile(ca_cert):
-                raise ValueError("Tried to use '{}' for --ca-certs, but it is not "
-                                 "a valid file.".format(ca_cert))
+                raise ValueError(f"Tried to use '{ca_cert}' for --ca-certs, but it is not "
+                                 "a valid file.")
         if ca_cert:
             run_ca_cert = os.path.join(client_dir, os.path.basename(ca_cert))
             process.Setup.Copy(ca_cert, run_ca_cert, CopyLogic.SoftFiles)
-            command += ' --ca-certs "{}" '.format(run_ca_cert)
+            command += f' --ca-certs "{run_ca_cert}" '
 
     if verbose:
         command += ' --verbose diag '
 
     if other_args:
-        command += " {}".format(other_args)
+        command += f" {other_args}"
 
     if keys is not None:
-        command += " --keys {}".format(keys)
+        command += f" --keys {keys}"
+
+    # Generally we prefer the deterministic behavior of a single thread where
+    # each connection is processed sequentially.
+    if 'thread-limit' not in command:
+        command += " --thread-limit 1"
 
     process.Command = command
     process.ReturnCode = 0

tests/gold_tests/cache/background_fill.test.py

@@ -167,8 +167,7 @@ def __testCase3(self):
             "pv_client",
             "replay/bg_fill.yaml",
             http_ports=[self.ts['for_pv'].Variables.port],
-            https_ports=[self.ts['for_pv'].Variables.ssl_port],
-            other_args='--thread-limit 1')
+            https_ports=[self.ts['for_pv'].Variables.ssl_port])
         tr.Processes.Default.ReturnCode = 0
         tr.Processes.Default.Streams.stdout = "gold/background_fill_3_stdout.gold"
         self.__checkProcessAfter(tr)

tests/gold_tests/cache/cache-auth.test.py

@@ -56,8 +56,7 @@ def runTraffic(self):
         tr = Test.AddTestRun(
             "Verify the proper caching behavior for request/response containing auth-related fields when ATS is in default configuration"
         )
-        tr.AddVerifierClientProcess(
-            "auth-default-client", self.authDefaultReplayFile, http_ports=[self.ts.Variables.port], other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("auth-default-client", self.authDefaultReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -98,8 +97,7 @@ def runTraffic(self):
         tr = Test.AddTestRun(
             "Verify the proper caching behavior for request/response containing auth-related fields when ATS is configured to bypass caching for those"
         )
-        tr.AddVerifierClientProcess(
-            "auth-ignored-client", self.authIgnoredReplayFile, http_ports=[self.ts.Variables.port], other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("auth-ignored-client", self.authIgnoredReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server

tests/gold_tests/cache/cache-control.test.py

@@ -203,10 +203,7 @@ def setupTS(self):
     def runTraffic(self):
         tr = Test.AddTestRun("Verify the proper handling of cache-control directives in requests in default configuration")
         tr.AddVerifierClientProcess(
-            "request-cache-control-default-client",
-            self.requestCacheControlReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+            "request-cache-control-default-client", self.requestCacheControlReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -256,10 +253,7 @@ def runTraffic(self):
             "Verify the proper handling of cache-control directives in requests when ATS is configured to honor client's request to bypass the cache"
         )
         tr.AddVerifierClientProcess(
-            "request-cache-control-honor-client-client",
-            self.requestCacheControlReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+            "request-cache-control-honor-client-client", self.requestCacheControlReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -304,10 +298,7 @@ def setupTS(self):
     def runTraffic(self):
         tr = Test.AddTestRun("Verify the proper handling of cache-control directives in responses in default configuration")
         tr.AddVerifierClientProcess(
-            "response-cache-control-client-default",
-            self.responseCacheControlReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+            "response-cache-control-client-default", self.responseCacheControlReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -356,10 +347,7 @@ def runTraffic(self):
             "Verify the proper handling of cache-control directives in responses when ATS is configured to ignore server's request to bypass the cache"
         )
         tr.AddVerifierClientProcess(
-            "response-cache-control-client-ignored",
-            self.responseCacheControlReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+            "response-cache-control-client-ignored", self.responseCacheControlReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server

tests/gold_tests/cache/cache-cookie.test.py

@@ -48,11 +48,7 @@ def setupTS(self):
 
     def runTraffic(self):
         tr = Test.AddTestRun("Verify the correct caching behavior when ATS is in default configuration")
-        tr.AddVerifierClientProcess(
-            "cookie-default-client",
-            self.cookieDefaultReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("cookie-default-client", self.cookieDefaultReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -91,8 +87,7 @@ def setupTS(self):
     def runTraffic(self):
         tr = Test.AddTestRun(
             "Verify the correct caching behavior when ATS is configured to not cache response to cookie for any content type")
-        tr.AddVerifierClientProcess(
-            "cookie-bypass-client", self.cookieBypassReplayFile, http_ports=[self.ts.Variables.port], other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("cookie-bypass-client", self.cookieBypassReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -131,11 +126,7 @@ def setupTS(self):
     def runTraffic(self):
         tr = Test.AddTestRun(
             "Verify the correct caching behavior when ATS is configured to cache response to cookie only for image content type")
-        tr.AddVerifierClientProcess(
-            "cookie-img-only-client",
-            self.cookieImgOnlyReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("cookie-img-only-client", self.cookieImgOnlyReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -175,10 +166,7 @@ def runTraffic(self):
         tr = Test.AddTestRun(
             "Verify the correct caching behavior when ATS is configured to cache response to cookie for all but text types")
         tr.AddVerifierClientProcess(
-            "cookie-all-but-text-client",
-            self.cookieAllButTextReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+            "cookie-all-but-text-client", self.cookieAllButTextReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -222,10 +210,7 @@ def runTraffic(self):
             "Verify the correct caching behavior when ATS is configured to cache all content types but text, but with a few exceptions for text types which would also be cached"
         )
         tr.AddVerifierClientProcess(
-            "cookie-all-but-text-with-excp-client",
-            self.cookieAllButTextReplayFile,
-            http_ports=[self.ts.Variables.port],
-            other_args='--thread-limit 1')
+            "cookie-all-but-text-with-excp-client", self.cookieAllButTextReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server

tests/gold_tests/chunked_encoding/bad_chunked_encoding.test.py

@@ -102,11 +102,7 @@ def setupTS(self):
     def runChunkedTraffic(self):
         tr = Test.AddTestRun()
         tr.AddVerifierClientProcess(
-            "client1",
-            self.chunkedReplayFile,
-            http_ports=[self.ts.Variables.port],
-            https_ports=[self.ts.Variables.ssl_port],
-            other_args='--thread-limit 1')
+            "client1", self.chunkedReplayFile, http_ports=[self.ts.Variables.port], https_ports=[self.ts.Variables.ssl_port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -162,11 +158,7 @@ def setupTS(self):
     def runChunkedTraffic(self):
         tr = Test.AddTestRun()
         tr.AddVerifierClientProcess(
-            "client2",
-            self.chunkedReplayFile,
-            http_ports=[self.ts.Variables.port],
-            https_ports=[self.ts.Variables.ssl_port],
-            other_args='--thread-limit 1')
+            "client2", self.chunkedReplayFile, http_ports=[self.ts.Variables.port], https_ports=[self.ts.Variables.ssl_port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server

tests/gold_tests/connect/connect.test.py

@@ -149,8 +149,7 @@ def setupTS(self):
 
     def runTraffic(self):
         tr = Test.AddTestRun("Verify correct handling of CONNECT request")
-        tr.AddVerifierClientProcess(
-            "connect-client", self.connectReplayFile, http_ports=[self.ts.Variables.port], other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("connect-client", self.connectReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server
@@ -235,8 +234,7 @@ def setupTS(self):
 
     def runTraffic(self):
         tr = Test.AddTestRun("Verify correct handling of CONNECT request on HTTP/2")
-        tr.AddVerifierClientProcess(
-            "connect-client2", self.connectReplayFile, https_ports=[self.ts.Variables.ssl_port], other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("connect-client2", self.connectReplayFile, https_ports=[self.ts.Variables.ssl_port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server

tests/gold_tests/h2/h2get_with_body.test.py

@@ -55,11 +55,7 @@
 tr.Processes.Default.StartBefore(pv_server)
 tr.Processes.Default.StartBefore(ts)
 tr.AddVerifierClientProcess(
-    "pv_client",
-    "h2get_with_body.yaml",
-    http_ports=[ts.Variables.port],
-    https_ports=[ts.Variables.ssl_port],
-    other_args='--thread-limit 1')
+    "pv_client", "h2get_with_body.yaml", http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port])
 tr.Processes.Default.ReturnCode = 0
 
 tr.Processes.Default.Streams.All += Testers.ContainsExpression(

tests/gold_tests/h2/http2_close_connection.test.py

@@ -51,11 +51,7 @@
 tr.Processes.Default.StartBefore(pv_server)
 tr.Processes.Default.StartBefore(ts)
 tr.AddVerifierClientProcess(
-    "pv_client",
-    "http2_close_connection.yaml",
-    http_ports=[ts.Variables.port],
-    https_ports=[ts.Variables.ssl_port],
-    other_args='--thread-limit 1')
+    "pv_client", "http2_close_connection.yaml", http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port])
 tr.Processes.Default.ReturnCode = 0
 
 tr.Processes.Default.Streams.All += Testers.ContainsExpression(

tests/gold_tests/headers/invalid_range_header.test.py

@@ -49,8 +49,7 @@ def setupTS(self):
 
     def runTraffic(self):
         tr = Test.AddTestRun()
-        tr.AddVerifierClientProcess(
-            "client1", self.invalidRangeRequestReplayFile, http_ports=[self.ts.Variables.port], other_args='--thread-limit 1')
+        tr.AddVerifierClientProcess("client1", self.invalidRangeRequestReplayFile, http_ports=[self.ts.Variables.port])
         tr.Processes.Default.StartBefore(self.server)
         tr.Processes.Default.StartBefore(self.ts)
         tr.StillRunningAfter = self.server