Add security audits for Rust and Python dependencies; enhance scraper and workflow tests

Author: ursisterbtw

.github/workflows/security.yml

@@ -0,0 +1,108 @@
+name: Security Scanning
+
+on:
+  push:
+    branches: [ main, dev ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 0 * * 0'  # Weekly on Sundays
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  semgrep:
+    name: Semgrep Security Scan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: returntocorp/semgrep-action@v1
+        with:
+          config: >-
+            p/security-audit
+            p/secrets
+            p/owasp-top-ten
+            p/python
+            p/rust
+
+  dependency-review:
+    name: Dependency Review
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v4
+        with:
+          fail-on-severity: moderate
+
+  cargo-audit:
+    name: Rust Security Audit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install cargo-audit
+        run: cargo install cargo-audit
+
+      - name: Run cargo-audit
+        run: cargo audit --deny warnings
+
+  pip-audit:
+    name: Python Security Audit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Create virtual environment
+        run: uv venv .venv
+
+      - name: Install dependencies
+        run: |
+          .venv/bin/uv pip install --upgrade pip
+          .venv/bin/uv pip install -r requirements.txt
+          .venv/bin/uv pip install pip-audit
+
+      - name: Run pip-audit
+        run: .venv/bin/pip-audit --desc
+
+  bandit:
+    name: Bandit Security Linting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install bandit
+        run: pip install bandit[toml]
+
+      - name: Run bandit
+        run: bandit -r RAGnificent/ -ll -f json -o bandit-report.json
+        continue-on-error: true
+
+      - name: Upload bandit results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: bandit-security-report
+          path: bandit-report.json

.github/workflows/unified-ci.yml

@@ -37,6 +37,11 @@ jobs:
 
       - name: Run tests
         run: cargo test --verbose
+      
+      - name: Security audit for Rust dependencies
+        run: |
+          cargo install cargo-audit --features=fix || true
+          cargo audit
 
   python-tests:
     name: Python ${{ matrix.python-version }} Tests
@@ -95,3 +100,13 @@ jobs:
         run: |
           .venv/bin/uv pip install build
           .venv/bin/python -m build
+      
+      - name: Security scan with pip-audit
+        run: |
+          .venv/bin/uv pip install pip-audit
+          .venv/bin/pip-audit
+      
+      - name: Security scan with bandit
+        run: |
+          .venv/bin/uv pip install bandit[toml]
+          .venv/bin/bandit -r RAGnificent/ -ll

RAGnificent/core/security.py

@@ -161,7 +161,7 @@ def sanitize_headers(headers: Dict[str, str]) -> Dict[str, str]:
         return {}
 
     # Build a mapping of lowercase -> original key to preserve original casing
-    original_keys: Dict[str, str] = {k.lower(): k for k in headers.keys()}
+    original_keys: Dict[str, str] = {k.lower(): k for k in headers}
 
     sensitive_headers = {
         "authorization",