Skip to content

Commit 1767176

Browse files
test(profiling): revert temporarily mark flaky tests as xfail (#15439)
## Description https://datadoghq.atlassian.net/browse/PROF-13152 This reverts commit 4753e14 (from #15433) and unmarks new Profiling tests as `xfail`. Why is it OK to do that? * We actually only had two flaky tests, one that was from Echion (`test_asyncio_coroutines.py`) and one (or some) that were previously added, e.g. `test_asyncio_import_profiler_from_process_after_starting_loop` * I fixed `test_asyncio_coroutines` by asserting not on a specific line, but any line within the function we wanted to see. It's not perfect, but the line detection is somewhat flaky for on-CPU Tasks (for the time being, this will soon be fixed hopefully!) * I fixed the `import_profiler...` tests by doing the same thing, because the line we were getting Samples for was not consistent across runs and in that case, we aren't really interested in the exact lines – we just want to make sure that we're actually getting Samples even when starting the Profiler later. [Proof of no flakes](https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40git.branch%3Akowalski%2Ftest-profiling-revert-temporarily-mark-flaky-tests-as-xfail-15433%20%40test.service%3Add-trace-py%20%40test.status%3Afail%20%40git.commit.sha%3A33f587f128%2A&agg_m=count&agg_m_source=base&agg_t=count&currentTab=overview&eventStack=&fromUser=false&index=citest&start=1764235048278&end=1764238648278&paused=false) on those tests on the latest version (`test_uwsgi_threads_processes_no_primary_lazy_apps[py3.12]` is unrelated).
1 parent 4753e14 commit 1767176

10 files changed

+61
-40
lines changed

tests/profiling/collector/pprof_utils.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,7 @@ def assert_profile_has_sample(
358358
profile: pprof_pb2.Profile,
359359
samples: List[pprof_pb2.Sample],
360360
expected_sample: StackEvent,
361+
print_samples_on_failure: bool = False,
361362
) -> None:
362363
found = False
363364
for sample in samples:
@@ -370,6 +371,9 @@ def assert_profile_has_sample(
370371
if DEBUG_TEST:
371372
print(e)
372373

374+
if not found and print_samples_on_failure:
375+
print_all_samples(profile)
376+
373377
assert found, "Expected samples not found in profile " + str(expected_sample.locations)
374378

375379

tests/profiling/collector/test_asyncio_context_manager.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
1+
from sys import version_info as PYVERSION
2+
13
import pytest
24

35

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
6+
@pytest.mark.xfail(
7+
condition=PYVERSION >= (3, 13), reason="Sampling async context manager stacks does not work on >=3.13"
8+
)
59
@pytest.mark.subprocess(
610
env=dict(
711
DD_PROFILING_OUTPUT_PPROF="/tmp/test_asyncio_context_manager",

tests/profiling/collector/test_asyncio_coroutines.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
22

33

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
54
@pytest.mark.subprocess(
65
env=dict(
76
DD_PROFILING_OUTPUT_PPROF="/tmp/test_asyncio_coroutines",
@@ -100,6 +99,7 @@ async def outer_function() -> None:
10099
),
101100
],
102101
),
102+
print_samples_on_failure=True,
103103
)
104104

105105
# Test that we see the background_task_func task
@@ -126,6 +126,7 @@ async def outer_function() -> None:
126126
),
127127
],
128128
),
129+
print_samples_on_failure=True,
129130
)
130131

131132
# Test that we see the background_math_function task
@@ -138,7 +139,7 @@ async def outer_function() -> None:
138139
pprof_utils.StackLocation(
139140
function_name="background_math_function",
140141
filename="test_asyncio_coroutines.py",
141-
line_no=background_math_function.__code__.co_firstlineno + 2,
142+
line_no=-1, # any line
142143
),
143144
# TODO: We should see outer_function, but for some reason we simply do not...
144145
# pprof_utils.StackLocation(
@@ -148,4 +149,5 @@ async def outer_function() -> None:
148149
# ),
149150
],
150151
),
152+
print_samples_on_failure=True,
151153
)

tests/profiling/collector/test_asyncio_deadlock.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
22

33

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
54
@pytest.mark.subprocess(
65
env=dict(
76
DD_PROFILING_OUTPUT_PPROF="/tmp/test_asyncio_deadlock",

tests/profiling/collector/test_asyncio_executor.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
22

33

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
54
@pytest.mark.subprocess(
65
env=dict(
76
DD_PROFILING_OUTPUT_PPROF="/tmp/test_asyncio_executor",

tests/profiling/collector/test_asyncio_gather_coroutines.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
22

33

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
54
@pytest.mark.subprocess(
65
env=dict(
76
DD_PROFILING_OUTPUT_PPROF="/tmp/test_asyncio_gather_coroutines",

tests/profiling/collector/test_asyncio_gather_tasks.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
22

33

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
54
@pytest.mark.subprocess(
65
env=dict(
76
DD_PROFILING_OUTPUT_PPROF="/tmp/test_asyncio_gather_tasks",

tests/profiling/collector/test_asyncio_wait.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
22

33

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
54
@pytest.mark.subprocess(
65
env=dict(
76
DD_PROFILING_OUTPUT_PPROF="/tmp/test_asyncio_wait",

tests/profiling/collector/test_generators.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import pytest
22

33

4-
@pytest.mark.xfail(reason="Temporarily marking new tests as xfail until we make them non-flaky")
54
@pytest.mark.subprocess(
65
env=dict(
76
DD_PROFILING_OUTPUT_PPROF="/tmp/test_generators",

0 commit comments

Comments
 (0)