Fix data loss when POLLHUP races with unread kernel data

[SoloJacobs]

Some of our customers reported that responses of our UNIX socket based back-ends were truncated. After some investigation, I found that this issue could be fairly easily reproduced with the below AI generated python script.

I found some related bug reports:

• https://stunnel.org/mailman3/hyperkitty/list/stunnel-users@lists.stunnel.org/thread/3ABSP2BIBRQT2ZB675CINBCAY5NX7QFT/#WNKUW2TTANI2JBCIYY24DVE7UG56WTVA
• https://bugzilla.redhat.com/show_bug.cgi?id=1170722

However, the issue can still be reproduced in stunnel 5.78.

For the implementation, I simply copied this implementation from src/client.c:

        if(sock_open_rd && s_poll_rdhup(c->fds, c->sock_rfd->fd) &&
                (ioctlsocket(c->sock_rfd->fd, FIONREAD, &bytes) || !bytes)) {
            s_log(LOG_INFO, "Read socket closed (read hangup)");
            sock_open_rd=0;
        }

Now my knowledge of c is somewhat limited, I only have a surface level understanding of read loop in src/client.c . The patch should thus be reviewed with caution.

"""
Regression test: stunnel must not truncate backend responses when
POLLHUP races with POLLIN on the backend unix socket.
"""

import contextlib
import os
import socket
import ssl
import subprocess
import threading
import time
from collections.abc import Generator

import pytest

_RUNFILES = os.environ.get("RUNFILES_DIR", "")
_WS = "_main"

STUNNEL_BIN = os.path.join(_RUNFILES, _WS, "omd/packages/stunnel/bin_runpath/stunnel")
OPENSSL_LIBS = os.path.join(_RUNFILES, _WS, "omd/packages/openssl/lib")

_NUM_LINES = 10_000
_LINE = "host_{n:05d};plugin_output_padding_to_stress_the_pollhup_race_xxxx\n"
_PAYLOAD = b"".join(_LINE.format(n=i).encode() for i in range(_NUM_LINES))


def _backend(sock_path: str, ready: threading.Event, stop: threading.Event) -> None:
    """Mock CMC: for each connection, discard the query then send _PAYLOAD and close."""
    srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    srv.bind(sock_path)
    srv.listen(64)
    srv.settimeout(0.2)
    ready.set()
    while not stop.is_set():
        try:
            conn, _ = srv.accept()
        except socket.timeout:
            continue
        with conn:
            conn.settimeout(0.2)
            try:
                conn.recv(4096)
            except (socket.timeout, OSError):
                pass
            conn.sendall(_PAYLOAD)
            # close immediately — this triggers the POLLHUP/POLLIN race in stunnel
    srv.close()


def _make_cert(path: str) -> None:
    subprocess.check_call(
        [
            "openssl",
            "req",
            "-x509",
            "-newkey",
            "rsa:2048",
            "-keyout",
            path,
            "-out",
            path,
            "-days",
            "1",
            "-nodes",
            "-subj",
            "/CN=test",
        ],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
    )


@contextlib.contextmanager
def _stunnel(tmp: str, cert: str, tls_sock: str, backend_sock: str) -> Generator[None, None, None]:
    pid_path = os.path.join(tmp, "stunnel.pid")
    conf_path = os.path.join(tmp, "stunnel.conf")
    with open(conf_path, "w") as f:
        f.write(
            f"cert           = {cert}\n"
            f"pid            = {pid_path}\n"
            f"output         = {tmp}/stunnel.log\n"
            "syslog         = no\n"
            "debug          = 5\n"
            "foreground     = no\n"
            "sslVersionMin  = TLSv1.2\n"
            "socket         = l:TCP_NODELAY=1\n"
            "socket         = r:TCP_NODELAY=1\n"
            "client         = no\n"
            "[live-tls]\n"
            f"accept  = {tls_sock}\n"
            f"connect = {backend_sock}\n"
        )
    env = {**os.environ, "LD_LIBRARY_PATH": OPENSSL_LIBS}
    subprocess.check_call([STUNNEL_BIN, conf_path], env=env)
    for _ in range(50):
        if os.path.exists(tls_sock):
            break
        time.sleep(0.05)
    else:
        raise RuntimeError("stunnel TLS socket never appeared")
    try:
        yield
    finally:
        try:
            with open(pid_path) as f:
                os.kill(int(f.read().strip()), 15)
        except (OSError, ValueError):
            pass


def _query(tls_sock: str) -> bytes:
    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    raw = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    raw.connect(tls_sock)
    with ctx.wrap_socket(raw) as s:
        s.sendall(b"GET services\nColumns: host_name plugin_output\n\n")
        buf = bytearray()
        while chunk := s.read(65536):
            buf += chunk
    return bytes(buf)


def test_no_truncation(tmp_path: pytest.TempPathFactory) -> None:
    tls_sock = str(tmp_path / "live-tls.sock")
    backend_sock = str(tmp_path / "live.sock")
    cert = str(tmp_path / "cert.pem")

    _make_cert(cert)

    ready, stop = threading.Event(), threading.Event()
    t = threading.Thread(target=_backend, args=(backend_sock, ready, stop), daemon=True)
    t.start()
    assert ready.wait(timeout=5), "backend socket never appeared"

    failures: list[tuple[int, int]] = []
    try:
        with _stunnel(str(tmp_path), cert, tls_sock, backend_sock):
            for run in range(1, 31):
                data = _query(tls_sock)
                if data != _PAYLOAD:
                    failures.append((run, len(data)))
    finally:
        stop.set()
        t.join(timeout=2)

    assert not failures, "{}/{} queries were truncated (want {} bytes each):\n{}".format(
        len(failures),
        30,
        len(_PAYLOAD),
        "\n".join(f"  run {r}: got {n} bytes" for r, n in failures),
    )
    ```

[SoloJacobs]

So, I found the previous patch did not fix all the instances of truncation. Thus, I have added another patch. With this one our live instance no longer looses any data. Of course, there might be more issues, which are just not able to reproduce.

Kind regards