From 704bef722fe317ff4fbfa6cbb9b495fb1601e6f4 Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Thu, 12 Mar 2026 11:22:08 -0400 Subject: [PATCH 1/4] Add envSampleCount and framesPerMaterial test suite options New options in _options.mtlx for controlling render test performance data collection: - envSampleCount: Configurable IBL sample count (default 1024). Lower values (1-16) make shader complexity a larger fraction of GPU time, enabling game-representative performance comparisons. - framesPerMaterial: Number of render iterations per material for GPU timing stability (default 1). Higher values yield statistically meaningful averages when combined with warm-up frame analysis. --- resources/Materials/TestSuite/_options.mtlx | 14 ++++++++++++++ .../MaterialXGenShader/GenShaderUtil.cpp | 14 ++++++++++++++ .../MaterialXGenShader/GenShaderUtil.h | 9 +++++++++ .../MaterialXRenderGlsl/RenderGlsl.cpp | 2 +- .../MaterialXTest/MaterialXRenderMsl/RenderMsl.mm | 2 +- .../MaterialXRenderSlang/RenderSlang.cpp | 2 +- 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/resources/Materials/TestSuite/_options.mtlx b/resources/Materials/TestSuite/_options.mtlx index 49b0b7cc39..10d3a043b4 100644 --- a/resources/Materials/TestSuite/_options.mtlx +++ b/resources/Materials/TestSuite/_options.mtlx @@ -84,5 +84,19 @@ Default is false to avoid overhead when not profiling. --> + + + + + + diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp index e2a197e1cc..0960685c24 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp @@ -1006,6 +1006,8 @@ void TestSuiteOptions::print(std::ostream& output) const output << "\tEnable Reference Quality: " << enableReferenceQuality << std::endl; output << "\tOutput Directory: " << (outputDirectory.isEmpty() ? "(default)" : outputDirectory.asString()) << std::endl; output << "\tEnable Tracing: " << enableTracing << std::endl; + output << "\tFrames Per Material: " << framesPerMaterial << std::endl; + output << "\tEnv Sample Count: " << envSampleCount << std::endl; } bool TestSuiteOptions::readOptions(const std::string& optionFile) @@ -1033,6 +1035,8 @@ bool TestSuiteOptions::readOptions(const std::string& optionFile) const std::string ENABLE_REFERENCE_QUALITY("enableReferenceQuality"); const std::string OUTPUT_DIRECTORY_STRING("outputDirectory"); const std::string ENABLE_TRACING_STRING("enableTracing"); + const std::string FRAMES_PER_MATERIAL_STRING("framesPerMaterial"); + const std::string ENV_SAMPLE_COUNT_STRING("envSampleCount"); overrideFiles.clear(); dumpGeneratedCode = false; @@ -1148,6 +1152,16 @@ bool TestSuiteOptions::readOptions(const std::string& optionFile) { enableTracing = val->asA(); } + else if (name == FRAMES_PER_MATERIAL_STRING) + { + int frames = val->asA(); + framesPerMaterial = (frames >= 1) ? static_cast(frames) : 1u; + } + else if (name == ENV_SAMPLE_COUNT_STRING) + { + int count = val->asA(); + envSampleCount = (count >= 1) ? count : 1024; + } } } } diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h index e6ae23f19f..d8e58c4e6a 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h @@ -126,6 +126,15 @@ class TestSuiteOptions // Default is false to avoid overhead when not profiling. bool enableTracing = false; + // Number of frames to render per material for GPU timing. + // Default is 1. Set higher (e.g., 5-10) for statistical validity. + // First frame often includes driver shader compilation overhead. + unsigned int framesPerMaterial = 1; + + // Number of environment radiance samples for IBL lighting. + // Default is 1024. Lower values (1-16) are more representative of real-time rendering. + int envSampleCount = 1024; + // Helper to resolve output path for an artifact. // If outputDirectory is set, returns outputDirectory/filename. // Otherwise returns the original path unchanged. diff --git a/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp b/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp index 197c43c93e..cdaaabc1fb 100644 --- a/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp +++ b/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp @@ -99,7 +99,7 @@ void GlslShaderRenderTester::registerLights(mx::DocumentPtr document, // Apply light settings for render tests. _lightHandler->setEnvRadianceMap(envRadiance); _lightHandler->setEnvIrradianceMap(envIrradiance); - _lightHandler->setEnvSampleCount(options.enableReferenceQuality ? 4096 : 1024); + _lightHandler->setEnvSampleCount(options.envSampleCount); _lightHandler->setRefractionTwoSided(true); } diff --git a/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm b/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm index c6340101da..8757393c08 100644 --- a/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm +++ b/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm @@ -103,7 +103,7 @@ bool runRenderer(const std::string& shaderName, // Apply light settings for render tests. _lightHandler->setEnvRadianceMap(envRadiance); _lightHandler->setEnvIrradianceMap(envIrradiance); - _lightHandler->setEnvSampleCount(options.enableReferenceQuality ? 4096 : 1024); + _lightHandler->setEnvSampleCount(options.envSampleCount); _lightHandler->setRefractionTwoSided(true); } diff --git a/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp b/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp index 7a96b7af78..2bd4961e24 100644 --- a/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp +++ b/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp @@ -98,7 +98,7 @@ void SlangShaderRenderTester::registerLights(mx::DocumentPtr document, // Apply light settings for render tests. _lightHandler->setEnvRadianceMap(envRadiance); _lightHandler->setEnvIrradianceMap(envIrradiance); - _lightHandler->setEnvSampleCount(options.enableReferenceQuality ? 4096 : 1024); + _lightHandler->setEnvSampleCount(options.envSampleCount); _lightHandler->setRefractionTwoSided(true); } From f997c2f19b4044871e528df51a5ac5d905c5dbea Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Thu, 12 Mar 2026 11:33:06 -0400 Subject: [PATCH 2/4] Preserve enableReferenceQuality fallback for envSampleCount When envSampleCount is not explicitly set in _options.mtlx, derive it from enableReferenceQuality (4096 if true, 1024 default). An explicit envSampleCount always takes priority. --- .../MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp index 0960685c24..761fd530f4 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp @@ -1045,6 +1045,8 @@ bool TestSuiteOptions::readOptions(const std::string& optionFile) enableIndirectLighting = true; enableReferenceQuality = false; + bool envSampleCountSet = false; + mx::DocumentPtr doc = mx::createDocument(); try { @@ -1161,11 +1163,19 @@ bool TestSuiteOptions::readOptions(const std::string& optionFile) { int count = val->asA(); envSampleCount = (count >= 1) ? count : 1024; + envSampleCountSet = true; } } } } + // If reference quality is enabled and envSampleCount wasn't explicitly + // overridden, use the higher sample count for reference-quality rendering. + if (enableReferenceQuality && !envSampleCountSet) + { + envSampleCount = 4096; + } + // Handle direct and indirect lighting toggles. if (!enableDirectLighting) { From 9a04923ac66cf11d1dbc0c9177733dd3440a848e Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Thu, 19 Mar 2026 16:35:47 -0400 Subject: [PATCH 3/4] Use framesPerMaterial option in render test loops Wrap the per-material render call in all three backends (GLSL, MSL, Slang) with a loop controlled by testOptions.framesPerMaterial. --- source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp | 5 ++++- source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm | 5 ++++- source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp b/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp index cdaaabc1fb..81b5f82ffc 100644 --- a/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp +++ b/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp @@ -356,7 +356,10 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName, unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor; unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor; _renderer->setSize(width, height); - _renderer->render(); + for (unsigned int frame = 0; frame < testOptions.framesPerMaterial; frame++) + { + _renderer->render(); + } } { diff --git a/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm b/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm index 8757393c08..09bd40185b 100644 --- a/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm +++ b/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm @@ -357,7 +357,10 @@ bool runRenderer(const std::string& shaderName, unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor; unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor; _renderer->setSize(width, height); - _renderer->render(); + for (unsigned int frame = 0; frame < testOptions.framesPerMaterial; frame++) + { + _renderer->render(); + } } { diff --git a/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp b/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp index 2bd4961e24..f67b370e01 100644 --- a/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp +++ b/source/MaterialXTest/MaterialXRenderSlang/RenderSlang.cpp @@ -350,7 +350,10 @@ bool SlangShaderRenderTester::runRenderer(const std::string& shaderName, unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor; unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor; _renderer->setSize(width, height); - _renderer->render(); + for (unsigned int frame = 0; frame < testOptions.framesPerMaterial; frame++) + { + _renderer->render(); + } } { From 7ccb55ba65263e6b2e1c2cbc2827fea1b7e50a29 Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Thu, 19 Mar 2026 17:12:27 -0400 Subject: [PATCH 4/4] Comment out envSampleCount from default options Keep envSampleCount documented but inactive so the enableReferenceQuality fallback (1024 -> 4096) works when envSampleCount is not explicitly overridden. --- resources/Materials/TestSuite/_options.mtlx | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/resources/Materials/TestSuite/_options.mtlx b/resources/Materials/TestSuite/_options.mtlx index 10d3a043b4..13254f79eb 100644 --- a/resources/Materials/TestSuite/_options.mtlx +++ b/resources/Materials/TestSuite/_options.mtlx @@ -92,11 +92,14 @@ --> - +