feat: add section for still failing packages to report

Author: Defelo

.github/workflows/review.yml

@@ -78,6 +78,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       pr: ${{ steps.pr.outputs.pr }}
+      base: ${{ steps.pr.outputs.base }}
+      base_ref: ${{ steps.pr.outputs.base_ref }}
       head: ${{ steps.pr.outputs.head }}
       merge: ${{ steps.pr.outputs.merge }}
       author_id: ${{ steps.pr.outputs.author_id }}
@@ -98,6 +100,8 @@ jobs:
             exit 1
           fi
           echo "pr=$pr" >> "$GITHUB_OUTPUT"
+          echo "base=$(jq -r '.base.sha' <<< "$pr")" >> "$GITHUB_OUTPUT"
+          echo "base_ref=$(jq -r '.base.ref' <<< "$pr")" >> "$GITHUB_OUTPUT"
           echo "head=$(jq -r '.head.sha' <<< "$pr")" >> "$GITHUB_OUTPUT"
           echo "merge=$(jq -r '.merge_commit_sha' <<< "$pr")" >> "$GITHUB_OUTPUT"
           echo "author_id=$(jq -r '.user.id' <<< "$pr")" >> "$GITHUB_OUTPUT"
@@ -132,6 +136,7 @@ jobs:
             .github/actions
             flake.lock
             flake.nix
+            generate_markdown_report.py
 
       - name: setup nix
         uses: ./.github/actions/setup-nix
@@ -147,7 +152,7 @@ jobs:
 
       - name: install packages
         run: |
-          pkgs=(coreutils nixpkgs-review jq gnused)
+          pkgs=(coreutils moreutils nixpkgs-review jq gnused generate-markdown-report)
           if [[ ${{ inputs.push-to-cache && vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '' }} = true ]]; then
             pkgs+=(attic-client)
           elif [[ ${{ inputs.push-to-cache && vars.CACHIX_CACHE != '' }} = true ]]; then
@@ -173,11 +178,38 @@ jobs:
             --build-args="-L" \
             --pr-json="$PR_JSON" \
             $EXTRA_ARGS
+
+          report_json=~/.cache/nixpkgs-review/pr-${PR_NUMBER}/report.json
+          report_md=~/.cache/nixpkgs-review/pr-${PR_NUMBER}/report.md
+          if failed=$(jq -er '.result[].failed[]' "$report_json"); then
+            git fetch origin "$BASE" && git switch -d "$BASE"
+            build=()
+            paths=()
+            for pkg in $failed; do
+              [[ "$(nix eval --impure --expr "import ./. {} ? ${pkg}")" = "true" ]] || continue
+              path=$(nix eval -f. "${pkg}.outPath" --raw) || continue
+              build+=("$pkg")
+              paths+=("$path")
+            done
+
+            if [[ ${#build[@]} -ne 0 ]]; then
+              nix build --keep-going -L -f. ${build[@]} || true
+            fi
+
+            for i in ${!build[@]}; do
+              ! nix store verify --no-contents --no-trust "${paths[i]}" || continue
+              jq '.result[].failed -= [$pkg] | .result[].still_failing += [$pkg]' --arg pkg "${build[i]}" "$report_json" | sponge "$report_json"
+            done
+          fi
+
+          generate-markdown-report "$report_json" "$report_md" "$BASE_REF" | sponge "$report_md"
         working-directory: nixpkgs
         env:
           GITHUB_TOKEN: ${{ github.token }}
           EXTRA_ARGS: ${{ inputs.extra-args }}
           PR_JSON: ${{ needs.prepare.outputs.pr }}
+          BASE: ${{ needs.prepare.outputs.base }}
+          BASE_REF: ${{ needs.prepare.outputs.base_ref }}
 
       - name: push results to cache
         if: ${{ inputs.push-to-cache && ((vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '') || vars.CACHIX_CACHE != '') }}
@@ -309,7 +341,7 @@ jobs:
           done
 
           cat report.md
-          echo success=$(jq -s 'all(.[].result[]; .failed==[])' report_*.json) >> "$GITHUB_OUTPUT"
+          echo success=$(jq -s 'all(.[].result[]; (.failed+.still_failing)==[])' report_*.json) >> "$GITHUB_OUTPUT"
           sed '1s|$| for [#'"$PR_NUMBER"'](https://github.com/NixOS/nixpkgs/pull/'"$PR_NUMBER"')|' report.md >> $GITHUB_STEP_SUMMARY
         env:
           HEAD: ${{ needs.prepare.outputs.head }}

flake.nix

@@ -26,6 +26,14 @@
     in
 
     {
+      packages = eachSystem (pkgs: {
+        generate-markdown-report = pkgs.writers.writePython3Bin "generate-markdown-report" {
+          flakeIgnore = [
+            "E501" # line too long
+          ];
+        } (builtins.readFile ./generate_markdown_report.py);
+      });
+
       overlays.nixpkgs-review = final: prev: {
         inherit (nixpkgs-review.packages.${final.stdenv.hostPlatform.system}) nixpkgs-review;
       };
@@ -44,6 +52,7 @@
 
       checks = eachSystem (pkgs: {
         inherit (pkgs) nixpkgs-review;
+        packages = pkgs.linkFarm "packages" self.packages.${pkgs.stdenv.hostPlatform.system};
         fmt = pkgs.runCommand "fmt-check" { } ''
           cp -r --no-preserve=mode ${self} repo
           ${lib.getExe self.formatter.${pkgs.stdenv.hostPlatform.system}} -C repo --ci

generate_markdown_report.py

@@ -0,0 +1,42 @@
+import json
+import re
+import sys
+
+with open(sys.argv[1]) as f:
+    report_json = json.load(f)
+with open(sys.argv[2]) as f:
+    report_md = f.read()
+base = sys.argv[3]
+
+aliases = {m[1]: m[2].split(", ") for m in re.finditer(r"<li>([^ ]*) \((.*)\)</li>", report_md)}
+
+
+def html_pkgs_section(emoji: str, packages: list[str], msg: str, what: str = "package") -> str:
+    if len(packages) == 0:
+        return ""
+    plural = "s" if len(packages) > 1 else ""
+    res = "<details>\n"
+    res += f"  <summary>{emoji} {len(packages)} {what}{plural} {msg}:</summary>\n  <ul>\n"
+    for pkg in packages:
+        if pkg in aliases:
+            pkg += f" ({", ".join(aliases[pkg])})"
+        res += f"    <li>{pkg}</li>\n"
+    res += "  </ul>\n</details>\n"
+    return res
+
+
+msg = ""
+for system, report in report_json["result"].items():
+    msg += "\n---\n"
+    msg += f"### `{system}`\n"
+    msg += html_pkgs_section(":fast_forward:", report["broken"], "marked as broken and skipped")
+    msg += html_pkgs_section(
+        ":fast_forward:", report["non-existent"], "present in ofBorgs evaluation, but not found in the checkout"
+    )
+    msg += html_pkgs_section(":fast_forward:", report["blacklisted"], "blacklisted")
+    msg += html_pkgs_section(":x:", report["failed"], "failed to build")
+    msg += html_pkgs_section(":x:", report.get("still_failing", []), f"still failing to build (also failed on {base})")
+    msg += html_pkgs_section(":white_check_mark:", report["tests"], "built", what="test")
+    msg += html_pkgs_section(":white_check_mark:", report["built"], "built")
+
+print(msg, end="")

treefmt.nix

@@ -11,6 +11,15 @@
     "LICENSE"
   ];
 
+  formatter.black = {
+    command = lib.getExe pkgs.black;
+    includes = [ "*.py" ];
+    options = [
+      "--line-length=120"
+      "--skip-magic-trailing-comma"
+    ];
+  };
+
   formatter.nixfmt = {
     command = lib.getExe pkgs.nixfmt;
     includes = [ "*.nix" ];