fix: move filters and event sources as builtins

fix: bring webhook as builtin

fix: re-raise asyncio.CancelledError

fix: handle asyncio.CancelledError properly

feat: map deprecated plugins to builtin names

docs: update examples using builtins

fix: remove mock runtime.yml from test data

Author: andresberejnoi

ansible_rulebook/engine.py

@@ -43,7 +43,9 @@
 from ansible_rulebook.util import (
     collect_ansible_facts,
     find_builtin_filter,
+    find_builtin_source,
     has_builtin_filter,
+    has_builtin_source,
     send_session_stats,
     substitute_variables,
 )
@@ -130,6 +132,8 @@ async def start_source(
             module = runpy.run_path(
                 os.path.join(source_dirs[0], source.source_name + ".py")
             )
+        elif has_builtin_source(source.source_name):
+            module = runpy.run_path(find_builtin_source(source.source_name))
         elif has_source(*split_collection_name(source.source_name)):
             module = runpy.run_path(
                 find_source(*split_collection_name(source.source_name))

ansible_rulebook/event_filter/dashes_to_underscores.py

@@ -0,0 +1,62 @@
+import multiprocessing as mp
+from typing import Any
+
+DOCUMENTATION = r"""
+---
+short_description: Change dashes to underscores.
+description:
+  - An event filter that changes dashes in keys to underscores.
+    For instance, the key X-Y becomes the new key X_Y.
+options:
+  overwrite:
+    description:
+      - Overwrite the values if there is a collision with a new key.
+    type: bool
+    default: true
+"""
+
+EXAMPLES = r"""
+- ansible.eda.alertmanager:
+    host: 0.0.0.0
+    port: 5050
+  filters:
+    - eda.builtin.dashes_to_underscores:
+        overwrite: false
+"""
+
+
+def _should_replace_key(obj: dict, new_key: str, overwrite: bool) -> bool:
+    """Check if the new key should replace the old one."""
+    return (new_key in obj and overwrite) or (new_key not in obj)
+
+
+def _process_dict(obj: dict, queue: list, logger, overwrite: bool) -> None:
+    """Process dictionary keys, replacing dashes with underscores."""
+    # list() required: dict modified during iteration (line 40: del obj[key])
+    for key in list(obj.keys()):  # NOSONAR(S7504)
+        value = obj[key]
+        queue.append(value)
+        if "-" in key:
+            new_key = key.replace("-", "_")
+            del obj[key]
+            if _should_replace_key(obj, new_key, overwrite):
+                obj[new_key] = value
+                logger.info("Replacing %s with %s", key, new_key)
+
+
+def main(
+    event: dict[str, Any],
+    overwrite: bool = True,  # noqa: FBT001, FBT002
+) -> dict[str, Any]:
+    """Change dashes in keys to underscores."""
+    logger = mp.get_logger()
+    logger.info("dashes_to_underscores")
+    queue = [event]
+    while queue:
+        obj = queue.pop()
+        if isinstance(obj, dict):
+            _process_dict(obj, queue, logger, overwrite)
+        elif isinstance(obj, list):
+            queue.extend(obj)
+
+    return event

ansible_rulebook/event_filter/insert_hosts_to_meta.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import dpath
+
+DOCUMENTATION = r"""
+---
+short_description: Extract hosts from the event data and insert them to
+                   the meta dict.
+description:
+  - An ansible-rulebook event filter that extracts hosts from the event
+    data and inserts them to the meta dict. In ansible-rulebook, this
+    will limit an action running on hosts in the meta dict.
+options:
+  host_path:
+    description:
+      - The json path inside the event data to find hosts.
+      - Do nothing if the key is not present or does exist in event.
+    type: str
+    default: null
+  path_separator:
+    description:
+      - The separator to interpret host_path.
+    type: str
+    default: "."
+  host_separator:
+    description:
+      - The separator to interpret host string.
+      - host_path can point to a string or a list. If it is a single
+        string but contains multiple hosts, use this parameter to
+        delimits the hosts. Treat the value as a single host if the
+        parameter is not present.
+    type: str
+    default: null
+  raise_error:
+    description:
+      - Whether raise PathNotExistError if host_path does not
+        exist in the event.
+      - It is recommended to turn it on during the rulebook
+        development time. You can then turn it off for production.
+    type: bool
+    default: false
+  log_error:
+    description:
+      - Whether log an error message if host_path does not
+        exist in the event.
+      - You can turn if off if it is expected to have events not
+        having the host_path to avoid noises in the log.
+    type: bool
+    default: true
+"""
+
+EXAMPLES = r"""
+- ansible.eda.alertmanager:
+    host: 0.0.0.0
+    port: 5050
+  filters:
+    - eda.builtin.insert_hosts_to_meta:
+        host_path: "app.target"
+        path_separator: "."
+        host_separator: ";"
+        raise_error: true
+        log_error: true
+"""
+
+LOGGER = logging.getLogger(__name__)
+
+
+class PathNotExistError(Exception):
+    """Cannot find the path in the event."""
+
+
+def _handle_path_error(
+    event: dict[str, Any],
+    host_path: str,
+    error: KeyError,
+    *,
+    raise_error: bool,
+    log_error: bool,
+) -> dict[str, Any]:
+    """Handle error when host path doesn't exist in event."""
+    msg = f"Event {event} does not contain {host_path}"
+    if log_error:
+        LOGGER.error(msg)  # noqa: TRY400
+    if raise_error:
+        raise PathNotExistError(msg) from error
+    return event
+
+
+def _validate_host_list(hosts: list | tuple) -> None:
+    """Validate that all items in the host list are strings."""
+    for host in hosts:
+        if not isinstance(host, str):
+            msg = f"{host} is not a valid hostname"
+            raise TypeError(msg)
+
+
+def _normalize_hosts(
+    hosts: str | list | tuple, host_separator: str | None
+) -> list[str]:
+    """Normalize hosts to a list of strings."""
+    if isinstance(hosts, str):
+        return hosts.split(host_separator) if host_separator else [hosts]
+    if isinstance(hosts, (list, tuple)):
+        _validate_host_list(hosts)
+        return list(hosts)
+    msg = f"{hosts} is not a valid hostname"
+    raise TypeError(msg)
+
+
+# pylint: disable=too-many-arguments
+def main(
+    event: dict[str, Any],
+    host_path: str | None = None,
+    host_separator: str | None = None,
+    path_separator: str = ".",
+    *,
+    raise_error: bool = False,
+    log_error: bool = True,
+) -> dict[str, Any]:
+    """Extract hosts from event data and insert into meta dict."""
+    if not host_path:
+        return event
+
+    try:
+        hosts = dpath.get(event, host_path, path_separator)
+    except KeyError as error:
+        return _handle_path_error(
+            event,
+            host_path,
+            error,
+            raise_error=raise_error,
+            log_error=log_error,
+        )
+
+    normalized_hosts = _normalize_hosts(hosts, host_separator)
+
+    if "meta" not in event:
+        event["meta"] = {}
+    event["meta"]["hosts"] = normalized_hosts
+    return event

ansible_rulebook/event_filter/json_filter.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import fnmatch
+from typing import Any, Optional
+
+DOCUMENTATION = r"""
+---
+short_description: Filter keys out of events.
+description:
+  - An event filter that filters keys out of events.
+    Includes override excludes.
+    This is useful to exclude information from events that is not
+    needed by the rule engine.
+options:
+  exclude_keys:
+    description:
+      - A list of strings or patterns to remove.
+    type: list
+    elements: str
+    default: null
+  include_keys:
+    description:
+      - A list of strings or patterns to keep even if it matches
+        exclude_keys patterns.
+    type: list
+    elements: str
+    default: null
+notes:
+  - The values in both parameters - include_keys and exclude_keys,
+    must be a full path in top-to-bottom order to the keys to be
+    filtered (or left-to-right order if it is given as a list),
+    as shown in the examples below.
+"""
+
+EXAMPLES = r"""
+- eda.builtin.generic:
+    payload:
+      key1:
+        key2:
+          f_ignore_1: 1
+          f_ignore_2: 2
+      key3:
+        key4:
+          f_use_1: 42
+          f_use_2: 45
+  filters:
+    - eda.builtin.json_filter:
+        include_keys:
+          - key3
+          - key4
+          - f_use*
+        exclude_keys: ['key1', 'key2', 'f_ignore_1']
+"""
+
+
+def _matches_include_keys(include_keys: list[str], string: str) -> bool:
+    return any(fnmatch.fnmatch(string, pattern) for pattern in include_keys)
+
+
+def _matches_exclude_keys(exclude_keys: list[str], string: str) -> bool:
+    return any(fnmatch.fnmatch(string, pattern) for pattern in exclude_keys)
+
+
+def _should_include(item: str, include_keys: list[str]) -> bool:
+    """Check if item should be included based on include_keys."""
+    return (item in include_keys) or _matches_include_keys(include_keys, item)
+
+
+def _should_exclude(item: str, exclude_keys: list[str]) -> bool:
+    """Check if item should be excluded based on exclude_keys."""
+    return (item in exclude_keys) or _matches_exclude_keys(exclude_keys, item)
+
+
+def _process_dict_keys(
+    obj: dict[str, Any],
+    queue: list,
+    exclude_keys: list[str],
+    include_keys: list[str],
+) -> None:
+    """Process dictionary keys for filtering."""
+    # list() required: dict modified during iteration (line 85: del obj[item])
+    for item in list(obj.keys()):  # NOSONAR(S7504)
+        if _should_include(item, include_keys):
+            queue.append(obj[item])
+        elif _should_exclude(item, exclude_keys):
+            del obj[item]
+        else:
+            queue.append(obj[item])
+
+
+def main(
+    event: dict[str, Any],
+    exclude_keys: Optional[list[str]] = None,  # noqa: UP045
+    include_keys: Optional[list[str]] = None,  # noqa: UP045
+) -> dict[str, Any]:
+    """Filter keys out of events."""
+    exclude_keys = exclude_keys or []
+    include_keys = include_keys or []
+
+    queue = [event]
+    while queue:
+        obj = queue.pop()
+        if isinstance(obj, dict):
+            _process_dict_keys(obj, queue, exclude_keys, include_keys)
+
+    return event

ansible_rulebook/event_filter/noop.py

@@ -0,0 +1,13 @@
+from typing import Any
+
+DOCUMENTATION = r"""
+---
+short_description: Do nothing.
+description:
+  - An event filter that does nothing to the input.
+"""
+
+
+def main(event: dict[str, Any]) -> dict[str, Any]:
+    """Return the input."""
+    return event