Feature/output yaml values

exasol/toolbox/tools/template.py

@@ -2,31 +2,25 @@
 import io
 from collections.abc import Mapping
 from contextlib import ExitStack
-from inspect import cleandoc
 from pathlib import Path
 from typing import (
     Any,
 )
 
 import importlib_resources as resources
 import typer
-import yaml
-from jinja2 import Environment
 from rich.columns import Columns
 from rich.console import Console
 from rich.syntax import Syntax
 
+from exasol.toolbox.util.workflows.workflow import Workflow
 from noxconfig import PROJECT_CONFIG
 
 stdout = Console()
 stderr = Console(stderr=True)
 
 CLI = typer.Typer()
 
-jinja_env = Environment(
-    variable_start_string="((", variable_end_string="))", autoescape=True
-)
-
 
 def _templates(pkg: str) -> Mapping[str, Any]:
     def _normalize(name: str) -> str:
@@ -72,18 +66,13 @@ def show_templates(
 
 def _render_template(
     src: str | Path,
-    stack: ExitStack,
 ) -> str:
-    input_file = stack.enter_context(open(src, encoding="utf-8"))
-
-    # dynamically render the template with Jinja2
-    template = jinja_env.from_string(input_file.read())
-    rendered_string = template.render(PROJECT_CONFIG.github_template_dict)
-
-    # validate that the rendered content is a valid YAML. This is not
-    # written out as by default it does not give GitHub-safe output.
-    yaml.safe_load(rendered_string)
-    return cleandoc(rendered_string) + "\n"
+    src_path = Path(src)
+    github_template_dict = PROJECT_CONFIG.github_template_dict
+    workflow = Workflow.load_from_template(
+        file_path=src_path, github_template_dict=github_template_dict
+    )
+    return workflow.content + "\n"
 
 
 def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> None:
@@ -107,7 +96,7 @@ def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> No
                     old = old.read().split("\n")
                     new = new.read().split("\n")
                 elif template_type == "workflow":
-                    new = _render_template(src=new, stack=stack)
+                    new = _render_template(src=new)
                     old = old.read().split("\n")
                     new = new.split("\n")
 
@@ -134,7 +123,7 @@ def _install_template(
             return
 
         output_file = stack.enter_context(open(dest, "wb"))
-        rendered_string = _render_template(src=src, stack=stack)
+        rendered_string = _render_template(src=src)
         output_file.write(rendered_string.encode("utf-8"))
 
 

exasol/toolbox/util/dependencies/poetry_dependencies.py

@@ -38,7 +38,7 @@ class PoetryToml(BaseModel):
     def load_from_toml(cls, working_directory: Path) -> PoetryToml:
         file_path = working_directory / PoetryFiles.pyproject_toml
         if not file_path.exists():
-            raise ValueError(f"File not found: {file_path}")
+            raise FileExistsError(f"File not found: {file_path}")
 
         try:
             text = file_path.read_text()

exasol/toolbox/util/workflows/format_yaml.py

@@ -0,0 +1,60 @@
+import re
+
+from yaml import SafeDumper
+from yaml.resolver import Resolver
+
+# Regex for common strings in YAML that lose quotes:
+#   1. Version numbers (e.g., 2.3.0, 3.10)
+#   2. OS/image names (e.g., ubuntu-24.04)
+#   3. Numeric strings that look like octals or floats (e.g., 045, 1.2)
+QUOTE_REGEX = re.compile(r"^(\d+\.\d+(\.\d+)?|[a-zA-Z]+-\d+\.\d+|0\d+)$")
+
+# yaml uses a shorthand to identify "on" and "off" tags.
+# for GitHub workflows, we do NOT want "on" replaced with "True".
+for character in ["O", "o"]:
+    Resolver.yaml_implicit_resolvers[character] = [
+        x
+        for x in Resolver.yaml_implicit_resolvers[character]
+        if x[0] != "tag:yaml.org,2002:bool"
+    ]
+
+
+class GitHubDumper(SafeDumper):
+    pass
+
+
+def empty_representer(dumper: SafeDumper, data):
+    """
+    Leave empty fields like empty, instead of adding "null"
+
+    Without using `empty_representer`
+        on:
+          workflow_call: null
+
+    Using `empty_representer`
+        on:
+          workflow_call:
+    """
+    return dumper.represent_scalar("tag:yaml.org,2002:null", "")
+
+
+def str_presenter(dumper: SafeDumper, data):
+    """
+    Present str in a custom format compatible with GitHub
+    """
+    # For line breaks in a multiline step, use pipe "|" instead of quotes "'"
+    if "\n" in data:
+        # Ensure it ends with \n so PyYAML doesn't add the '-' strip indicator
+        if not data.endswith("\n"):
+            data += "\n"
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+
+    # For strings with versions, ensure that they are quoted '"' so that they
+    # are not incorrectly parsed in the workflow, e.g. to an integer instead of a float.
+    if QUOTE_REGEX.match(data):
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+
+GitHubDumper.add_representer(str, str_presenter)
+GitHubDumper.add_representer(type(None), empty_representer)

exasol/toolbox/util/workflows/workflow.py

@@ -0,0 +1,59 @@
+from inspect import cleandoc
+from pathlib import Path
+from typing import Any
+
+from jinja2 import Environment
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+)
+from yaml import (
+    dump,
+    safe_load,
+)
+
+from exasol.toolbox.util.workflows.format_yaml import GitHubDumper
+
+jinja_env = Environment(
+    variable_start_string="((", variable_end_string="))", autoescape=True
+)
+
+
+def _render_template(template: str, github_template_dict: dict[str, Any]) -> str:
+    """
+    Render the template with Jinja2 & dump as a str
+    """
+    # Dynamically render the template with Jinja2
+    jinja_template = jinja_env.from_string(template)
+    rendered_string = jinja_template.render(github_template_dict)
+
+    # Also checks that the rendered template is a valid YAML.
+    data = safe_load(rendered_string)
+
+    return cleandoc(
+        dump(
+            data,
+            Dumper=GitHubDumper,
+            sort_keys=False,  # if True, then re-orders the jobs alphabetically
+        )
+    )
+
+
+class Workflow(BaseModel):
+    model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
+
+    content: str
+
+    @classmethod
+    def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any]):
+        if not file_path.exists():
+            raise FileNotFoundError(file_path)
+
+        try:
+            raw_content = file_path.read_text()
+            rendered_content = _render_template(
+                template=raw_content, github_template_dict=github_template_dict
+            )
+            return cls(content=rendered_content)
+        except Exception as e:
+            raise ValueError(f"Error rendering file: {str(e)}")

test/unit/util/workflows/format_yaml_test.py

@@ -0,0 +1,100 @@
+from inspect import cleandoc
+
+from yaml import (
+    dump,
+    safe_load,
+)
+
+from exasol.toolbox.util.workflows.format_yaml import GitHubDumper
+
+
+class TestEmptyRepresenter:
+    documentation = """
+    name: Merge-Gate
+    on:
+      workflow_call:
+    """
+
+    def test_works_as_expected(self):
+        data = safe_load(cleandoc(self.documentation))
+        output = dump(
+            data,
+            Dumper=GitHubDumper,
+        )
+        assert output == cleandoc(self.documentation) + "\n"
+
+    def test_default_behavior_differs(self):
+        expected = cleandoc(
+            """
+            name: Merge-Gate
+            on:
+              workflow_call: null
+            """
+        )
+
+        data = safe_load(cleandoc(self.documentation))
+
+        output = dump(data)
+        assert output == expected + "\n"
+
+
+class TestStrPresenter:
+    doc_with_line_break = """
+    steps:
+    - name: Generate GitHub Summary
+      run: |
+        echo -e "# Summary" >> $GITHUB_STEP_SUMMARY
+        poetry run -- nox -s project:report -- --format markdown >> $GITHUB_STEP_SUMMARY
+    """
+    doc_with_version = """
+    steps:
+    - name: Setup Python & Poetry Environment
+      uses: exasol/python-toolbox/.github/actions/python-environment@v5
+      with:
+        python-version: "3.10"
+        poetry-version: "2.3.0"
+    """
+
+    def test_line_break_works_as_expected(self):
+        data = safe_load(cleandoc(self.doc_with_line_break))
+        output = dump(
+            data,
+            Dumper=GitHubDumper,
+        )
+        assert output == cleandoc(self.doc_with_line_break) + "\n"
+
+    def test_line_break_with_default_differs(self):
+        data = safe_load(cleandoc(self.doc_with_line_break))
+        output = dump(data)
+        assert output == (
+            "steps:\n"
+            "- name: Generate GitHub Summary\n"
+            '  run: \'echo -e "# Summary" >> $GITHUB_STEP_SUMMARY\n'
+            "\n"
+            "    poetry run -- nox -s project:report -- --format markdown >> "
+            "$GITHUB_STEP_SUMMARY'\n"
+        )
+
+    def test_quote_regex_works_as_expected(self):
+        data = safe_load(cleandoc(self.doc_with_version))
+        output = dump(
+            data,
+            Dumper=GitHubDumper,
+            sort_keys=False,  # if True, then re-orders the jobs alphabetically
+        )
+        assert output == cleandoc(self.doc_with_version) + "\n"
+
+    def test_quote_regex_with_default_differs(self):
+        data = safe_load(cleandoc(self.doc_with_version))
+        output = dump(
+            data,
+            sort_keys=False,  # if True, then re-orders the jobs alphabetically
+        )
+        assert output == (
+            "steps:\n"
+            "- name: Setup Python & Poetry Environment\n"
+            "  uses: exasol/python-toolbox/.github/actions/python-environment@v5\n"
+            "  with:\n"
+            "    python-version: '3.10'\n"
+            "    poetry-version: 2.3.0\n"
+        )