Add removed dependencies tracking and PR reporting

- Detect dependencies removed since last extraction
- Show removed deps in console output (first 10)
- Generate JSON report of removed dependencies
- Include removed deps list in nightly PR body with details
- Flag critical removed dependencies
- Update baseline count dynamically

Signed-off-by: Dan Gil <[email protected]>

Author: dagil-nvidia

.github/workflows/dependency-extraction-nightly.yml

@@ -50,7 +50,8 @@ jobs:
           # Generate timestamped version (for artifacts)
           python3 .github/workflows/extract_dependency_versions.py \
             --output .github/reports/dependency_versions_${TIMESTAMP}.csv \
-            --report-unversioned
+            --report-unversioned \
+            --report-removed .github/reports/removed_dependencies.json
           
           # Copy to latest version (for repo tracking)
           mkdir -p .github/reports
@@ -74,9 +75,41 @@ jobs:
             changed_count=$(grep -c ",Changed," .github/reports/dependency_versions_latest.csv 2>/dev/null || echo "0")
             unchanged_count=$(grep -c ",Unchanged," .github/reports/dependency_versions_latest.csv 2>/dev/null || echo "0")
             
+            # Parse removed dependencies from JSON
+            if [ -f ".github/reports/removed_dependencies.json" ]; then
+              removed_count=$(python3 -c "import json; print(json.load(open('.github/reports/removed_dependencies.json'))['count'])" 2>/dev/null || echo "0")
+              
+              # Format removed dependencies list for PR body (limit to first 10)
+              removed_list=$(python3 -c "
+import json
+try:
+    data = json.load(open('.github/reports/removed_dependencies.json'))
+    removed = data['removed'][:10]  # First 10
+    lines = []
+    for dep in removed:
+        critical = ' **[CRITICAL]**' if dep.get('Critical') == 'Yes' else ''
+        lines.append(f\"    • **{dep['Dependency Name']}** (was: \`{dep['Version']}\`){critical}\")
+        lines.append(f\"      _from {dep['Source File']}_\")
+    
+    if data['count'] > 10:
+        lines.append(f\"    _... and {data['count'] - 10} more (see CSV for full list)_\")
+    
+    print('\n'.join(lines))
+except:
+    print('    _No removed dependencies_')
+" 2>/dev/null || echo "    _Unable to parse removed dependencies_")
+            else
+              removed_count="0"
+              removed_list="    _No removed dependencies_"
+            fi
+            
             echo "new_deps=$new_count" >> $GITHUB_OUTPUT
             echo "changed_deps=$changed_count" >> $GITHUB_OUTPUT
             echo "unchanged_deps=$unchanged_count" >> $GITHUB_OUTPUT
+            echo "removed_deps=$removed_count" >> $GITHUB_OUTPUT
+            echo "removed_list<<EOF" >> $GITHUB_OUTPUT
+            echo "$removed_list" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
           else
             echo "has_changes=false" >> $GITHUB_OUTPUT
           fi
@@ -96,8 +129,12 @@ jobs:
             ### 📊 Summary
             - **New Dependencies:** ${{ steps.check_changes.outputs.new_deps }}
             - **Changed Versions:** ${{ steps.check_changes.outputs.changed_deps }}
+            - **Removed Dependencies:** ${{ steps.check_changes.outputs.removed_deps }}
             - **Unchanged:** ${{ steps.check_changes.outputs.unchanged_deps }}
             
+            ### 🗑️ Removed Dependencies
+            ${{ steps.check_changes.outputs.removed_list }}
+            
             ### 📋 Files Updated
             - ✅ `.github/reports/dependency_versions_latest.csv` - Latest dependency snapshot
             - ✅ `.github/reports/unversioned_dependencies_latest.csv` - Unversioned deps report (if applicable)
@@ -107,6 +144,7 @@ jobs:
             ### ✔️ Review Checklist
             - [ ] Review new dependencies for security/licensing concerns
             - [ ] Check version changes for breaking updates
+            - [ ] Review removed dependencies (intentional?)
             - [ ] Verify unversioned dependencies report
             - [ ] Update baseline count if increase is expected
             

.github/workflows/extract_dependency_versions.py

@@ -1407,15 +1407,55 @@ def sort_key(dep):
             new_count = sum(1 for d in self.dependencies if d['Status'] == 'New')
             changed_count = sum(1 for d in self.dependencies if d['Status'] == 'Changed')
             unchanged_count = sum(1 for d in self.dependencies if d['Status'] == 'Unchanged')
+            removed = self.get_removed_dependencies()
 
             print(f"✓ Written {len(self.dependencies)} dependencies to {output_path}")
             print(f"  Changes since previous version:")
             print(f"    New: {new_count}")
             print(f"    Changed: {changed_count}")
+            print(f"    Removed: {len(removed)}")
             print(f"    Unchanged: {unchanged_count}")
+            
+            if removed:
+                print(f"\n  Removed dependencies:")
+                for dep in removed[:10]:  # Show first 10
+                    critical_flag = " [CRITICAL]" if dep['Critical'] == 'Yes' else ""
+                    print(f"    • {dep['Dependency Name']} (was: {dep['Version']}){critical_flag}")
+                    print(f"      from {dep['Source File']}")
+                if len(removed) > 10:
+                    print(f"    ... and {len(removed) - 10} more")
         else:
             print(f"✓ Written {len(self.dependencies)} dependencies to {output_path}")
 
+    def get_removed_dependencies(self) -> List[Dict[str, str]]:
+        """
+        Detect dependencies that were in the previous CSV but not in the current extraction.
+        Returns list of removed dependencies with their previous information.
+        """
+        if not self.previous_latest_dependencies:
+            return []
+        
+        # Build set of current dependency keys
+        current_keys = set()
+        for dep in self.dependencies:
+            key = f"{dep['Component']}:{dep['Category']}:{dep['Dependency Name']}"
+            current_keys.add(key)
+        
+        # Find dependencies in previous but not in current
+        removed = []
+        for prev_key, prev_dep in self.previous_latest_dependencies.items():
+            if prev_key not in current_keys:
+                removed.append({
+                    'Component': prev_dep.get('Component', ''),
+                    'Category': prev_dep.get('Category', ''),
+                    'Dependency Name': prev_dep.get('Dependency Name', ''),
+                    'Version': prev_dep.get('Version', ''),
+                    'Source File': prev_dep.get('Source File', ''),
+                    'Critical': prev_dep.get('Critical', 'No')
+                })
+        
+        return removed
+    
     def write_unversioned_report(self, output_path: Path) -> None:
         """Write a separate report of unversioned dependencies."""
         unversioned = [
@@ -1591,6 +1631,11 @@ def main():
         action="store_true",
         help="Generate separate report of unversioned dependencies"
     )
+    parser.add_argument(
+        "--report-removed",
+        type=str,
+        help="Output removed dependencies to JSON file (e.g., removed.json)"
+    )
     parser.add_argument(
         "--config",
         type=Path,
@@ -1752,6 +1797,17 @@ def main():
         unversioned_path = output_path.parent / f"{output_path.stem}_unversioned{output_path.suffix}"
         extractor.write_unversioned_report(unversioned_path)
 
+    # Write removed dependencies report if requested
+    if args.report_removed:
+        removed_deps = extractor.get_removed_dependencies()
+        removed_path = Path(args.report_removed)
+        with open(removed_path, 'w') as f:
+            json.dump({
+                'count': len(removed_deps),
+                'removed': removed_deps
+            }, f, indent=2)
+        print(f"✓ Written {len(removed_deps)} removed dependencies to {removed_path}")
+    
     # Print summary
     extractor.print_summary()