fix(er): chained exception frame ordering (#15397)

## Description

We fix the order in which frames are collected from an exception chain.
We make sure that the frames closer to the point where the initial
exception is thrown are captured first. This ensures that we propagate
the tags on the relevant spans as the exception chain bubbles up and
unwinds.

Author: P403n1x87

ddtrace/debugging/_exception/replay.py

@@ -113,7 +113,11 @@ def limit_exception(exc_ident: int) -> bool:
 def unwind_exception_chain(
     exc: t.Optional[BaseException], tb: t.Optional[TracebackType]
 ) -> t.Tuple[ExceptionChain, t.Optional[uuid.UUID]]:
-    """Unwind the exception chain and assign it an ID."""
+    """Unwind the exception chain and assign it an ID.
+
+    The chain goes from "cause to effect", meaning that every cause for an
+    exception is put first.
+    """
     chain: ExceptionChain = deque()
 
     while exc is not None:
@@ -141,9 +145,13 @@ def unwind_exception_chain(
     return chain, exc_id
 
 
-def get_tb_frames_from_exception_chain(chain: ExceptionChain) -> t.List[TracebackType]:
-    """Get the frames from the exception chain."""
-    frames: t.List[TracebackType] = []
+def get_tb_frames_from_exception_chain(chain: ExceptionChain) -> t.Generator[t.Tuple[int, TracebackType], None, None]:
+    """Get the frames from the exception chain.
+
+    For each exception in the chain we collect the maximum number of frames
+    configured, starting from the point where the exception was thrown.
+    """
+    frame_count = 0
 
     for _, tb in chain:
         local_frames = []
@@ -157,9 +165,21 @@ def get_tb_frames_from_exception_chain(chain: ExceptionChain) -> t.List[Tracebac
 
         # Get only the last N frames from the traceback, where N is the
         # configured max size for span tracebacks.
-        frames.extend(local_frames[-global_config._span_traceback_max_size :])
+        # local_frames = local_frames[-global_config._span_traceback_max_size :]
+        local_frames[: -global_config._span_traceback_max_size] = []
+
+        # Update the frame count to allow computing the sequence number
+        frame_count += len(local_frames)
+
+        # Set the initial sequence number. This will decrease as we yield the
+        # new frames
+        frame_index = frame_count
 
-    return frames
+        # Pop and yield the frames from this traceback in reverse order
+        while local_frames:
+            frame = local_frames.pop()
+            yield frame_index, frame
+            frame_index -= 1
 
 
 class SpanExceptionProbe(LogLineProbe):
@@ -262,8 +282,7 @@ def _attach_tb_frame_snapshot_to_span(
                 return False
 
             snapshot = None
-            snapshot_id = frame.f_locals.get(SNAPSHOT_KEY, None)
-            if snapshot_id is None:
+            if (snapshot_id := frame.f_locals.get(SNAPSHOT_KEY, None)) is None:
                 # We don't have a snapshot for the frame so we create one
                 if cached_only:
                     # If we only want a cached snapshot we return True as a signal
@@ -328,7 +347,7 @@ def on_span_exception(
 
         # Capture more frames if we have budget left, otherwise set just the
         # tags on the span for those frames that we have already captured.
-        for seq_nr, _tb in reversed(list(enumerate(get_tb_frames_from_exception_chain(chain), 1))):
+        for seq_nr, _tb in get_tb_frames_from_exception_chain(chain):
             has_snapshot_budget = frames_captured < config.max_frames
             has_captured = self._attach_tb_frame_snapshot_to_span(
                 span, _tb, exc_id, seq_nr, cached_only=not has_snapshot_budget

releasenotes/notes/fix-er-chained-exception-frame-ordering-d457f89ae7b4915f.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    exception replay: fixed the order in which frames are captured to ensure
+    that the values of frames close to the point where the initial exception
+    was thrown are always attached to the relevant spans.

tests/debugging/exception/test_replay.py

@@ -34,19 +34,19 @@ def c(foo=42):
         c()
     except Exception as e:
         chain, _ = replay.unwind_exception_chain(e, e.__traceback__)
-        frames = replay.get_tb_frames_from_exception_chain(chain)
+        frames = list(replay.get_tb_frames_from_exception_chain(chain))
         # There are two tracebacks in the chain: one for KeyError and one for
         # ValueError. The innermost goes from the call to a in b up to the point
         # where the exception is raised in a. The outermost goes from the call
         # in this test function up to the point in b where the exception from a
         # is caught and the the KeyError is raised.
         assert len(frames) == 2 + 3
-        assert [f.tb_frame.f_code.co_name for f in frames] == [
-            "b",
-            "a",
-            "test_tb_frames_from_exception_chain",
-            "c",
-            "b",
+        assert [(n, f.tb_frame.f_code.co_name) for n, f in frames] == [
+            (2, "a"),
+            (1, "b"),
+            (5, "b"),
+            (4, "c"),
+            (3, "test_tb_frames_from_exception_chain"),
         ]