diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe40dd0..7b8dd2d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: check-hooks-apply - id: check-useless-excludes - repo: https://github.com/tox-dev/pyproject-fmt - rev: v2.16.2 + rev: v2.18.1 hooks: - id: pyproject-fmt - repo: https://github.com/lyz-code/yamlfix @@ -46,8 +46,12 @@ repos: rev: v1.38.0 hooks: - id: yamllint + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.37.0 + hooks: + - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.5 + rev: v0.15.6 hooks: - id: ruff-check args: @@ -79,6 +83,14 @@ repos: args: - --wrap - '88' - files: (CLAUDE\.md|README\.md) + files: (AGENTS\.md|CLAUDE\.md|README\.md) + - id: mdformat + additional_dependencies: + - mdformat-myst + - mdformat-ruff + args: + - --wrap + - '88' + files: docs/. ci: autoupdate_schedule: monthly diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..494f504 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,73 @@ +@https://raw.githubusercontent.com/OpenSourceEconomics/ai-instructions/make-submodule/profiles/tier-a.md + +# dags + +## Overview + +**dags** is a Python library for creating executable DAGs (Directed Acyclic Graphs) from +interdependent functions. It provides tools to concatenate functions, manage type +annotations, and execute function graphs. + +## Build & Test + +This project uses [pixi](https://pixi.sh) for environment management. + +```bash +pixi run -e py313 tests # Run tests with Python 3.13 +pixi run -e py314 tests # Run tests with Python 3.14 +pixi run -e py313 tests-with-cov # Run tests with coverage +pixi run ty # Type checking +prek run --all-files # Linting & formatting +pixi run -e docs build-docs # Build HTML docs with Jupyter Book +pixi run -e docs view-docs # Live preview of docs +``` + +Available Python environments: `py311`, `py312`, `py313`, `py314` + +Documentation uses **Jupyter Book 2.0** with **MyST** markdown. Config is in +`docs/myst.yml`. Docs include executable Jupyter notebooks. + +## Architecture + +``` +src/dags/ +├── __init__.py # Main exports +├── annotations.py # Type annotation handling +├── dag.py # Core DAG functionality (concatenate_functions) +├── exceptions.py # Custom exceptions +├── output.py # Output processing utilities +├── signature.py # Function signature utilities +├── typing.py # Type definitions +├── utils.py # General utilities +└── tree/ # Tree-related utilities + +docs/ +├── myst.yml # Jupyter Book config +├── index.ipynb # Homepage +├── getting_started.ipynb # Getting started guide +├── usage_patterns.ipynb # Interactive examples notebook +├── tree.ipynb # Tree utilities docs +└── api.md # API reference + +tests/ +├── test_annotations.py # Annotation tests +├── test_dag.py # DAG concatenation tests +├── test_signature.py # Signature tests +└── ... +``` + +### Key Modules + +- **annotations.py**: Handles function type annotations, including a workaround for + Python 3.14's `functools.wraps` annotation mismatch bug +- **dag.py**: Core `concatenate_functions()` for combining interdependent functions into + a single callable +- **exceptions.py**: `AnnotationMismatchError`, `NonStringAnnotationError`, etc. + +## Code Style + +- Does **not** use `from __future__ import annotations` or `TYPE_CHECKING` blocks +- Ruff for linting (target: Python 3.11) +- ty for type checking (all rules set to error) +- Google docstring convention +- User-facing APIs accept `Sequence` (not `list`) for input parameters diff --git a/CHANGES.md b/CHANGES.md index 712d156..d00951b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,13 @@ This is a record of all past dags releases and what went into them in reverse chronological order. We follow [semantic versioning](https://semver.org/) and all releases are available on [conda-forge](https://anaconda.org/conda-forge/dags). +## 0.5.1 + +- :gh:`79` Use AGENTS.md, update hooks and rules (:ghuser:`hmgaudecker`). + +- :gh:`78` Fix `decorator_rename_arguments` by calling `get_free_arguments` inside the + decorator (:ghuser:`hmgaudecker`). + ## 0.5.0 - :gh:`77` Fix `decorator_rename_arguments` by calling `get_free_arguments` inside the diff --git a/CLAUDE.md b/CLAUDE.md index 57b0539..43c994c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,88 +1 @@ -# CLAUDE.md - -## Project Overview - -**dags** is a Python library for creating executable DAGs (Directed Acyclic Graphs) from -interdependent functions. It provides tools to concatenate functions, manage type -annotations, and execute function graphs. - -## Development Setup - -This project uses **pixi** for environment management. - -### Running Tests - -```bash -pixi run -e py313 tests # Run tests with Python 3.13 -pixi run -e py314 tests # Run tests with Python 3.14 -pixi run -e py313 tests-with-cov # Run tests with coverage -``` - -Available Python environments: `py311`, `py312`, `py313`, `py314` - -### Type Checking - -```bash -pixi run ty -``` - -### Linting & Formatting - -```bash -pixi run prek run --all-files -``` - -### Building Docs - -```bash -pixi run -e docs build-docs # Build HTML docs with Jupyter Book -pixi run -e docs view-docs # Live preview of docs -``` - -Documentation uses **Jupyter Book 2.0** with **MyST** markdown. Config is in -`docs/myst.yml`. Docs include executable Jupyter notebooks. - -## Project Structure - -``` -src/dags/ -├── __init__.py # Main exports -├── annotations.py # Type annotation handling -├── dag.py # Core DAG functionality (concatenate_functions) -├── exceptions.py # Custom exceptions -├── output.py # Output processing utilities -├── signature.py # Function signature utilities -├── typing.py # Type definitions -├── utils.py # General utilities -└── tree/ # Tree-related utilities - -docs/ -├── myst.yml # Jupyter Book config -├── index.ipynb # Homepage -├── getting_started.ipynb # Getting started guide -├── usage_patterns.ipynb # Interactive examples notebook -├── tree.ipynb # Tree utilities docs -└── api.md # API reference - -tests/ -├── test_annotations.py # Annotation tests -├── test_dag.py # DAG concatenation tests -├── test_signature.py # Signature tests -└── ... -``` - -## Key Modules - -- **annotations.py**: Handles function type annotations, including a workaround for - Python 3.14's `functools.wraps` annotation mismatch bug -- **dag.py**: Core `concatenate_functions()` for combining interdependent functions into - a single callable -- **exceptions.py**: `AnnotationMismatchError`, `NonStringAnnotationError`, etc. - -## Code Style - -- Does **not** use `from __future__ import annotations` or `TYPE_CHECKING` blocks -- Ruff for linting (target: Python 3.11) -- ty for type checking (all rules set to error) -- Google docstring convention -- User-facing APIs accept `Sequence` (not `list`) for input parameters +@AGENTS.md diff --git a/docs/api.md b/docs/api.md index 8f4c177..cbbd281 100644 --- a/docs/api.md +++ b/docs/api.md @@ -19,11 +19,12 @@ concatenate_functions( ) -> Callable[..., Any] ``` -Combine multiple interdependent functions into a single callable. Functions are executed in topological order based on name-matching dependencies. +Combine multiple interdependent functions into a single callable. Functions are executed +in topological order based on name-matching dependencies. `dags.dag` ---- +______________________________________________________________________ ### `dags.create_dag` @@ -38,7 +39,7 @@ Build a directed acyclic graph from functions without creating a combined functi `dags.dag` ---- +______________________________________________________________________ ### `dags.get_ancestors` @@ -55,7 +56,7 @@ Find all functions that a set of targets depends on. `dags.dag` ---- +______________________________________________________________________ ## Annotations & Signatures @@ -70,11 +71,12 @@ get_annotations( ) -> dict[str, str] | dict[str, type] ``` -Get type annotations from a function, handling `functools.partial` and Python 3.14 edge cases. Returns a dict keyed by argument names plus `"return"`. +Get type annotations from a function, handling `functools.partial` and Python 3.14 edge +cases. Returns a dict keyed by argument names plus `"return"`. `dags.annotations` ---- +______________________________________________________________________ ### `dags.get_free_arguments` @@ -86,7 +88,7 @@ Get parameter names of a function, excluding arguments bound via `functools.part `dags.annotations` ---- +______________________________________________________________________ ### `dags.rename_arguments` @@ -98,11 +100,12 @@ rename_arguments( ) -> Callable ``` -Rename function parameters using a mapping dict. Can be used as a decorator or called directly. +Rename function parameters using a mapping dict. Can be used as a decorator or called +directly. `dags.signature` ---- +______________________________________________________________________ ### `dags.with_signature` @@ -117,11 +120,12 @@ with_signature( ) -> Callable ``` -Add a signature to a function of type `f(*args, **kwargs)`. Can be used as a decorator with or without arguments. +Add a signature to a function of type `f(*args, **kwargs)`. Can be used as a decorator +with or without arguments. `dags.signature` ---- +______________________________________________________________________ ## Tree Utilities @@ -133,7 +137,7 @@ The delimiter used for qualified names: `"__"`. `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.qname_from_tree_path` @@ -145,7 +149,7 @@ Convert a tree path tuple to a qualified name string (e.g., `("a", "b")` → `"a `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.tree_path_from_qname` @@ -157,7 +161,7 @@ Convert a qualified name string to a tree path tuple. `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.flatten_to_qnames` @@ -169,7 +173,7 @@ Flatten a nested dict to a flat dict with qualified name keys. `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.unflatten_from_qnames` @@ -181,7 +185,7 @@ Restore a nested dict from a flat dict with qualified name keys. `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.flatten_to_tree_paths` @@ -193,7 +197,7 @@ Flatten a nested dict to a flat dict with tuple keys. `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.unflatten_from_tree_paths` @@ -205,7 +209,7 @@ Restore a nested dict from a flat dict with tuple keys. `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.qnames` @@ -217,7 +221,7 @@ Return a list of all qualified name keys from a nested dict. `dags.tree.tree_utils` ---- +______________________________________________________________________ ### `dags.tree.tree_paths` @@ -229,7 +233,7 @@ Return a list of all tree path keys from a nested dict. `dags.tree.tree_utils` ---- +______________________________________________________________________ ## Tree DAG Functions @@ -245,11 +249,12 @@ concatenate_functions_tree( ) -> Callable[[NestedInputDict], NestedOutputDict] ``` -Combine a nested dictionary of functions into a single callable that takes nested inputs and returns nested outputs. +Combine a nested dictionary of functions into a single callable that takes nested inputs +and returns nested outputs. `dags.tree.dag_tree` ---- +______________________________________________________________________ ### `dags.tree.create_dag_tree` @@ -265,7 +270,7 @@ Build a DAG from nested function dictionaries and input structure. `dags.tree.dag_tree` ---- +______________________________________________________________________ ### `dags.tree.create_tree_with_input_types` @@ -277,11 +282,13 @@ create_tree_with_input_types( ) -> NestedInputStructureDict ``` -Create a nested input structure template with type annotations based on functions and targets. Useful for discovering the expected input shape before calling `concatenate_functions_tree`. +Create a nested input structure template with type annotations based on functions and +targets. Useful for discovering the expected input shape before calling +`concatenate_functions_tree`. `dags.tree.dag_tree` ---- +______________________________________________________________________ ### `dags.tree.get_functions_without_tree_logic` @@ -292,11 +299,12 @@ get_functions_without_tree_logic( ) -> QNameFunctionDict ``` -Flatten a nested function dict and rename all parameters to qualified absolute names, producing a flat dict suitable for `concatenate_functions`. +Flatten a nested function dict and rename all parameters to qualified absolute names, +producing a flat dict suitable for `concatenate_functions`. `dags.tree.dag_tree` ---- +______________________________________________________________________ ### `dags.tree.get_one_function_without_tree_logic` @@ -308,11 +316,12 @@ get_one_function_without_tree_logic( ) -> Callable ``` -Convert a single function from relative parameter names to qualified absolute names given its tree path and top-level namespace. +Convert a single function from relative parameter names to qualified absolute names +given its tree path and top-level namespace. `dags.tree.dag_tree` ---- +______________________________________________________________________ ## Tree Validation @@ -331,11 +340,12 @@ fail_if_paths_are_invalid( ) -> None ``` -Validate all tree paths for trailing underscores and repeated top-level elements. Raises `TrailingUnderscoreError` or `RepeatedTopLevelElementError`. +Validate all tree paths for trailing underscores and repeated top-level elements. Raises +`TrailingUnderscoreError` or `RepeatedTopLevelElementError`. `dags.tree.validation` ---- +______________________________________________________________________ ### `dags.tree.validation.fail_if_path_elements_have_trailing_underscores` @@ -349,7 +359,7 @@ Check that no non-leaf path element ends with an underscore. `dags.tree.validation` ---- +______________________________________________________________________ ### `dags.tree.validation.fail_if_top_level_elements_repeated_in_paths` @@ -364,7 +374,7 @@ Fail if any top-level namespace element appears deeper in the hierarchy. `dags.tree.validation` ---- +______________________________________________________________________ ### `dags.tree.validation.fail_if_top_level_elements_repeated_in_single_path` @@ -379,18 +389,18 @@ Same check as above, for a single path. `dags.tree.validation` ---- +______________________________________________________________________ ## Exceptions -| Exception | Module | Description | -|-----------|--------|-------------| -| `DagsError` | `dags.exceptions` | Base exception for all dags errors | -| `CyclicDependencyError` | `dags.exceptions` | DAG contains a cycle | -| `MissingFunctionsError` | `dags.exceptions` | Target functions not provided | -| `AnnotationMismatchError` | `dags.exceptions` | Type annotations conflict between functions | -| `NonStringAnnotationError` | `dags.exceptions` | Non-string annotation with `set_annotations=True` | -| `InvalidFunctionArgumentsError` | `dags.exceptions` | Invalid function arguments (too many positional, duplicated, unexpected keyword) | -| `ValidationError` | `dags.exceptions` | Base for validation errors | -| `RepeatedTopLevelElementError` | `dags.tree.exceptions` | Top-level element repeated in tree path | -| `TrailingUnderscoreError` | `dags.tree.exceptions` | Non-leaf path element ends with underscore | +| Exception | Module | Description | +| ------------------------------- | ---------------------- | -------------------------------------------------------------------------------- | +| `DagsError` | `dags.exceptions` | Base exception for all dags errors | +| `CyclicDependencyError` | `dags.exceptions` | DAG contains a cycle | +| `MissingFunctionsError` | `dags.exceptions` | Target functions not provided | +| `AnnotationMismatchError` | `dags.exceptions` | Type annotations conflict between functions | +| `NonStringAnnotationError` | `dags.exceptions` | Non-string annotation with `set_annotations=True` | +| `InvalidFunctionArgumentsError` | `dags.exceptions` | Invalid function arguments (too many positional, duplicated, unexpected keyword) | +| `ValidationError` | `dags.exceptions` | Base for validation errors | +| `RepeatedTopLevelElementError` | `dags.tree.exceptions` | Top-level element repeated in tree path | +| `TrailingUnderscoreError` | `dags.tree.exceptions` | Non-leaf path element ends with underscore | diff --git a/pixi.lock b/pixi.lock index 5545cb6..fd229db 100644 --- a/pixi.lock +++ b/pixi.lock @@ -5406,8 +5406,8 @@ packages: timestamp: 1770674447292 - pypi: ./ name: dags - version: 0.4.4.dev8+g2550a1f15.d20260309 - sha256: dd3a91e4b5d5e324ed387d033e9014e33f39c0bab1090e444ae4501e207f1be3 + version: 0.5.1.dev2+g7139b172b.d20260313 + sha256: 52b55361ac76033cc242b862f833794c809774101807db5b0117b5df4b8d0a36 requires_dist: - flatten-dict - networkx>=3.6 diff --git a/pyproject.toml b/pyproject.toml index 0ec61c3..4798736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,40 +75,39 @@ extend-ignore = [ "TC002", # Move third-party import into a type-checking block "TC003", # Move standard library import into a type-checking block "TRY003", # Long messages outside exception class - "ANN401", - "FA100", - "FA102", + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed + "FA100", # Missing from __future__ import annotations + "FA102", # Missing from __future__ import annotations ] per-file-ignores."docs/**/*" = [ - "ANN001", - "ANN002", - "ANN003", - "ANN201", - "ANN202", - "ARG001", - "D100", - "D103", - "D104", - "T201", + "ANN001", # Missing type annotation for function argument + "ANN002", # Missing type annotation for *args + "ANN003", # Missing type annotation for **kwargs + "ANN201", # Missing return type annotation for public function + "ANN202", # Missing return type annotation for private function + "ARG001", # Unused function argument + "D100", # Missing docstring in public module + "D103", # Missing docstring in public function + "D104", # Missing docstring in public package + "T201", # print found ] -per-file-ignores."src/dags/tree/__init__.py" = [ "RUF022" ] per-file-ignores."tests/*" = [ - "ANN001", - "ANN002", - "ANN003", - "ANN201", - "ANN202", - "D100", - "D103", - "D104", - "D401", - "FBT001", - "INP001", - "PLC2401", - "PLR2004", - "S101", + "ANN001", # Missing type annotation for function argument + "ANN002", # Missing type annotation for *args + "ANN003", # Missing type annotation for **kwargs + "ANN201", # Missing return type annotation for public function + "ANN202", # Missing return type annotation for private function + "D100", # Missing docstring in public module + "D103", # Missing docstring in public function + "D104", # Missing docstring in public package + "D401", # First line of docstring should be in imperative mood + "FBT001", # Boolean-typed positional argument in function definition + "INP001", # Implicit namespace packages + "PLC2401", # Variable name contains a non-ASCII character + "PLR2004", # Magic value used in comparison + "S101", # Use of assert ] -per-file-ignores."tests/test_dag.py" = [ "ARG001" ] +per-file-ignores."tests/test_dag.py" = [ "ARG001" ] # Unused function argument pydocstyle.convention = "google" [tool.pyproject-fmt] @@ -173,7 +172,7 @@ docs = [ "docs", "py314" ] py311 = [ "py311", "tests" ] py312 = [ "py312", "tests" ] py313 = [ "py313", "tests" ] -py314 = [ "py314", "tests", "ty-task" ] +py314 = [ "py314", "tests", "type-checking" ] [tool.pixi.feature.docs.dependencies] jupyter-book = ">=2.0" mystmd = "*" @@ -198,10 +197,10 @@ pytest-xdist = "*" [tool.pixi.feature.tests.tasks] tests = "pytest" tests-with-cov = "pytest --cov-report=xml --cov=./" -[tool.pixi.feature.ty-task.pypi-dependencies] +[tool.pixi.feature.type-checking.pypi-dependencies] ty = "*" types-networkx = "*" -[tool.pixi.feature.ty-task.tasks] +[tool.pixi.feature.type-checking.tasks] ty = "ty check" [tool.pixi.pypi-dependencies] dags = { path = ".", editable = true } diff --git a/src/dags/tree/__init__.py b/src/dags/tree/__init__.py index d1a5519..786881c 100644 --- a/src/dags/tree/__init__.py +++ b/src/dags/tree/__init__.py @@ -82,20 +82,17 @@ def fail_if_paths_are_invalid( __all__ = [ - # Primary functions - "create_tree_with_input_types", - "create_dag_tree", + "QNAME_DELIMITER", "concatenate_functions_tree", - "get_functions_without_tree_logic", - "get_one_function_without_tree_logic", - # Deprecated + "create_dag_tree", + "create_tree_with_input_types", "fail_if_paths_are_invalid", - "functions_without_tree_logic", - "one_function_without_tree_logic", - # Qualified name utilities - "QNAME_DELIMITER", "flatten_to_qnames", "flatten_to_tree_paths", + "functions_without_tree_logic", + "get_functions_without_tree_logic", + "get_one_function_without_tree_logic", + "one_function_without_tree_logic", "qname_from_tree_path", "qnames", "tree_path_from_qname",