fix: Handle various HTML encodings in documentation parsing (#58)

jamesbrink · web-flow · commit bfc084da4d7b · 2025-10-18T13:49:53.000-07:00
Fix encoding errors when parsing Home Manager and nix-darwin documentation
that may be served with different encodings (windows-1252, UTF-8 with BOM,
ISO-8859-1).

Changes:
- Use resp.content instead of resp.text in parse_html_options()
- Let BeautifulSoup handle encoding detection automatically
- Add test coverage for windows-1252, UTF-8 with BOM, and ISO-8859-1
- Update all existing tests to use mock_resp.content

This resolves intermittent "unknown encoding: windows-1252" errors when
fetching documentation from CDN edge servers with different configurations.

Fixes errors like:
- "Failed to fetch docs: unknown encoding: windows-1252"
diff --git a/mcp_nixos/server.py b/mcp_nixos/server.py
@@ -249,7 +249,9 @@ def parse_html_options(url: str, query: str = "", prefix: str = "", limit: int =
     try:
         resp = requests.get(url, timeout=30)  # Increase timeout for large docs
         resp.raise_for_status()
-        soup = BeautifulSoup(resp.text, "html.parser")
+        # Use resp.content to let BeautifulSoup handle encoding detection
+        # This prevents encoding errors like "unknown encoding: windows-1252"
+        soup = BeautifulSoup(resp.content, "html.parser")
         options = []
 
         # Get all dt elements
diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py
@@ -108,7 +108,7 @@ def test_parse_html_options_large_document(self, mock_get):
 
         mock_resp = Mock()
         mock_resp.raise_for_status = Mock()
-        mock_resp.text = large_html
+        mock_resp.content = large_html.encode("utf-8")
         mock_get.return_value = mock_resp
 
         # Should respect limit
@@ -135,7 +135,7 @@ def test_parse_html_options_malformed_html(self, mock_get):
 
         mock_resp = Mock()
         mock_resp.raise_for_status = Mock()
-        mock_resp.text = malformed_html
+        mock_resp.content = malformed_html.encode("utf-8")
         mock_get.return_value = mock_resp
 
         options = parse_html_options("http://test.com")
@@ -160,7 +160,7 @@ def test_parse_html_options_special_characters(self, mock_get):
 
         mock_resp = Mock()
         mock_resp.raise_for_status = Mock()
-        mock_resp.text = html_with_entities
+        mock_resp.content = html_with_entities.encode("utf-8")
         mock_get.return_value = mock_resp
 
         options = parse_html_options("http://test.com")
diff --git a/tests/test_evals.py b/tests/test_evals.py
@@ -124,7 +124,7 @@ async def test_complete_firefox_installation_flow(self, mock_get, mock_post):
 
         # Step 3: Search Home Manager options
         hm_resp = Mock()
-        hm_resp.text = """
+        hm_resp.content = b"""
         <html>
             <dt>programs.firefox.enable</dt>
             <dd>
diff --git a/tests/test_flakes.py b/tests/test_flakes.py
@@ -249,7 +249,7 @@ async def test_flake_search_error_handling(self, mock_post):
         """Test flake search error handling."""
         mock_response = MagicMock()
         mock_response.status_code = 500
-        mock_response.text = "Internal Server Error"
+        mock_response.content = b"Internal Server Error"
 
         # Create an HTTPError with a response attribute
         http_error = requests.HTTPError("500 Server Error")
@@ -314,7 +314,7 @@ async def test_home_manager_stats_with_data(self, mock_get):
         """
 
         mock_get.return_value.status_code = 200
-        mock_get.return_value.text = mock_html
+        mock_get.return_value.content = mock_html.encode("utf-8")
 
         result = await home_manager_stats()
 
@@ -329,7 +329,7 @@ async def test_home_manager_stats_with_data(self, mock_get):
     async def test_home_manager_stats_error_handling(self, mock_get):
         """Test home_manager_stats error handling."""
         mock_get.return_value.status_code = 404
-        mock_get.return_value.text = "Not Found"
+        mock_get.return_value.content = b"Not Found"
 
         result = await home_manager_stats()
 
@@ -359,7 +359,7 @@ async def test_darwin_stats_with_data(self, mock_get):
         """
 
         mock_get.return_value.status_code = 200
-        mock_get.return_value.text = mock_html
+        mock_get.return_value.content = mock_html.encode("utf-8")
 
         result = await darwin_stats()
 
@@ -374,7 +374,7 @@ async def test_darwin_stats_with_data(self, mock_get):
     async def test_darwin_stats_error_handling(self, mock_get):
         """Test darwin_stats error handling."""
         mock_get.return_value.status_code = 500
-        mock_get.return_value.text = "Server Error"
+        mock_get.return_value.content = b"Server Error"
 
         result = await darwin_stats()
 
@@ -402,7 +402,7 @@ async def test_stats_with_complex_categories(self, mock_get):
         """
 
         mock_get.return_value.status_code = 200
-        mock_get.return_value.text = mock_html
+        mock_get.return_value.content = mock_html.encode("utf-8")
 
         result = await home_manager_stats()
 
@@ -416,7 +416,7 @@ async def test_stats_with_complex_categories(self, mock_get):
     async def test_stats_with_empty_html(self, mock_get):
         """Test stats functions with empty HTML."""
         mock_get.return_value.status_code = 200
-        mock_get.return_value.text = "<html><body></body></html>"
+        mock_get.return_value.content = b"<html><body></body></html>"
 
         result = await home_manager_stats()
 
@@ -546,7 +546,7 @@ async def test_combined_workflow_stats_and_search(self, mock_post, mock_get):
         """
 
         mock_get.return_value.status_code = 200
-        mock_get.return_value.text = stats_html
+        mock_get.return_value.content = stats_html.encode("utf-8")
 
         stats_result = await home_manager_stats()
 
diff --git a/tests/test_mcp_tools.py b/tests/test_mcp_tools.py
@@ -148,7 +148,7 @@ async def test_parse_html_options_type_extraction(self, mock_get):
         """Test that type information is not properly extracted from HTML."""
         # Mock HTML response with proper structure
         mock_response = MagicMock()
-        mock_response.text = """
+        mock_response.content = """.encode("utf-8")
         <html>
         <body>
             <dt>programs.git.enable</dt>
diff --git a/tests/test_plain_text_output.py b/tests/test_plain_text_output.py
@@ -143,7 +143,7 @@ async def test_home_manager_search_plain_text(self, mock_get):
         """Test home_manager_search returns plain text."""
         # Mock HTML response
         mock_response = Mock()
-        mock_response.text = """
+        mock_response.content = """.encode("utf-8")
         <html>
             <dt>programs.git.enable</dt>
             <dd>
@@ -168,7 +168,7 @@ async def test_home_manager_info_plain_text(self, mock_get):
         """Test home_manager_info returns plain text."""
         # Mock HTML response
         mock_response = Mock()
-        mock_response.text = """
+        mock_response.content = """.encode("utf-8")
         <html>
             <dt>programs.git.enable</dt>
             <dd>
@@ -215,7 +215,7 @@ async def test_home_manager_list_options_plain_text(self, mock_get):
         """Test home_manager_list_options returns plain text."""
         # Mock HTML response
         mock_response = Mock()
-        mock_response.text = """
+        mock_response.content = """.encode("utf-8")
         <html>
             <dt>programs.git.enable</dt>
             <dd><p>Enable git</p></dd>
@@ -238,7 +238,7 @@ async def test_darwin_search_plain_text(self, mock_get):
         """Test darwin_search returns plain text."""
         # Mock HTML response
         mock_response = Mock()
-        mock_response.text = """
+        mock_response.content = """.encode("utf-8")
         <html>
             <dt>system.defaults.dock.autohide</dt>
             <dd>
@@ -263,7 +263,7 @@ async def test_no_results_plain_text(self, mock_get):
         """Test empty results return appropriate plain text."""
         # Mock empty HTML response
         mock_response = Mock()
-        mock_response.text = "<html></html>"
+        mock_response.content = b"<html></html>"
         mock_response.raise_for_status = Mock()
         mock_get.return_value = mock_response
 
diff --git a/tests/test_server.py b/tests/test_server.py
@@ -129,7 +129,7 @@ def test_es_query_missing_hits(self, mock_post):
     def test_parse_html_options_success(self, mock_get):
         """Test successful HTML parsing."""
         mock_resp = Mock()
-        mock_resp.text = """
+        html_content = """
         <html>
             <dt>programs.git.enable</dt>
             <dd>
@@ -143,6 +143,7 @@ def test_parse_html_options_success(self, mock_get):
             </dd>
         </html>
         """
+        mock_resp.content = html_content.encode("utf-8")
         mock_resp.raise_for_status = Mock()
         mock_get.return_value = mock_resp
 
@@ -156,14 +157,15 @@ def test_parse_html_options_success(self, mock_get):
     def test_parse_html_options_with_query(self, mock_get):
         """Test HTML parsing with query filter."""
         mock_resp = Mock()
-        mock_resp.text = """
+        html_content = """
         <html>
             <dt>programs.git.enable</dt>
             <dd><p>Enable git</p></dd>
             <dt>programs.vim.enable</dt>
             <dd><p>Enable vim</p></dd>
         </html>
         """
+        mock_resp.content = html_content.encode("utf-8")
         mock_resp.raise_for_status = Mock()
         mock_get.return_value = mock_resp
 
@@ -175,14 +177,15 @@ def test_parse_html_options_with_query(self, mock_get):
     def test_parse_html_options_with_prefix(self, mock_get):
         """Test HTML parsing with prefix filter."""
         mock_resp = Mock()
-        mock_resp.text = """
+        html_content = """
         <html>
             <dt>programs.git.enable</dt>
             <dd><p>Enable git</p></dd>
             <dt>services.nginx.enable</dt>
             <dd><p>Enable nginx</p></dd>
         </html>
         """
+        mock_resp.content = html_content.encode("utf-8")
         mock_resp.raise_for_status = Mock()
         mock_get.return_value = mock_resp
 
@@ -194,7 +197,7 @@ def test_parse_html_options_with_prefix(self, mock_get):
     def test_parse_html_options_empty_response(self, mock_get):
         """Test HTML parsing with empty response."""
         mock_resp = Mock()
-        mock_resp.text = "<html></html>"
+        mock_resp.content = b"<html></html>"
         mock_resp.raise_for_status = Mock()
         mock_get.return_value = mock_resp
 
@@ -217,13 +220,92 @@ def test_parse_html_options_limit(self, mock_get):
         options_html = ""
         for i in range(10):
             options_html += f"<dt>option.{i}</dt><dd><p>desc{i}</p></dd>"
-        mock_resp.text = f"<html>{options_html}</html>"
+        mock_resp.content = f"<html>{options_html}</html>".encode()
         mock_resp.raise_for_status = Mock()
         mock_get.return_value = mock_resp
 
         result = parse_html_options("http://test.com", limit=5)
         assert len(result) == 5
 
+    @patch("mcp_nixos.server.requests.get")
+    def test_parse_html_options_windows_1252_encoding(self, mock_get):
+        """Test HTML parsing with windows-1252 encoding."""
+        # Create HTML content with special characters
+        html_content = """
+        <html>
+            <head><meta charset="windows-1252"></head>
+            <dt>programs.git.userName</dt>
+            <dd>
+                <p>Git user name with special chars: café</p>
+                <span class="term">Type: string</span>
+            </dd>
+        </html>
+        """
+
+        mock_resp = Mock()
+        # Simulate windows-1252 encoded content
+        mock_resp.content = html_content.encode("windows-1252")
+        mock_resp.encoding = "windows-1252"
+        mock_resp.raise_for_status = Mock()
+        mock_get.return_value = mock_resp
+
+        # Should not raise encoding errors
+        result = parse_html_options("http://test.com")
+        assert len(result) == 1
+        assert result[0]["name"] == "programs.git.userName"
+        assert "café" in result[0]["description"]
+
+    @patch("mcp_nixos.server.requests.get")
+    def test_parse_html_options_utf8_with_bom(self, mock_get):
+        """Test HTML parsing with UTF-8 BOM."""
+        html_content = """
+        <html>
+            <dt>programs.neovim.enable</dt>
+            <dd>
+                <p>Enable Neovim with unicode: 你好</p>
+                <span class="term">Type: boolean</span>
+            </dd>
+        </html>
+        """
+
+        mock_resp = Mock()
+        # Add UTF-8 BOM at the beginning
+        mock_resp.content = b"\xef\xbb\xbf" + html_content.encode("utf-8")
+        mock_resp.encoding = "utf-8-sig"
+        mock_resp.raise_for_status = Mock()
+        mock_get.return_value = mock_resp
+
+        result = parse_html_options("http://test.com")
+        assert len(result) == 1
+        assert result[0]["name"] == "programs.neovim.enable"
+        assert "你好" in result[0]["description"]
+
+    @patch("mcp_nixos.server.requests.get")
+    def test_parse_html_options_iso_8859_1_encoding(self, mock_get):
+        """Test HTML parsing with ISO-8859-1 encoding."""
+        html_content = """
+        <html>
+            <head><meta charset="iso-8859-1"></head>
+            <dt>services.nginx.virtualHosts</dt>
+            <dd>
+                <p>Nginx config with special: naïve résumé</p>
+            </dd>
+        </html>
+        """
+
+        mock_resp = Mock()
+        # Simulate ISO-8859-1 encoded content
+        mock_resp.content = html_content.encode("iso-8859-1")
+        mock_resp.encoding = "iso-8859-1"
+        mock_resp.raise_for_status = Mock()
+        mock_get.return_value = mock_resp
+
+        result = parse_html_options("http://test.com")
+        assert len(result) == 1
+        assert result[0]["name"] == "services.nginx.virtualHosts"
+        assert "naïve" in result[0]["description"]
+        assert "résumé" in result[0]["description"]
+
 
 class TestNixOSTools:
     """Test all NixOS tools."""
@@ -516,8 +598,10 @@ async def test_home_manager_stats(self, mock_get):
         </body>
         </html>
         """
-        mock_get.return_value.status_code = 200
-        mock_get.return_value.text = mock_html
+        mock_resp = Mock()
+        mock_resp.content = mock_html.encode("utf-8")
+        mock_resp.raise_for_status = Mock()
+        mock_get.return_value = mock_resp
 
         result = await home_manager_stats()
         assert "Home Manager Statistics:" in result
@@ -604,8 +688,10 @@ async def test_darwin_stats(self, mock_get):
         </body>
         </html>
         """
-        mock_get.return_value.status_code = 200
-        mock_get.return_value.text = mock_html
+        mock_resp = Mock()
+        mock_resp.content = mock_html.encode("utf-8")
+        mock_resp.raise_for_status = Mock()
+        mock_get.return_value = mock_resp
 
         result = await darwin_stats()
         assert "nix-darwin Statistics:" in result
@@ -664,7 +750,7 @@ async def test_special_characters_in_query(self, mock_query):
     def test_malformed_html_response(self, mock_get):
         """Test parsing malformed HTML."""
         mock_resp = Mock()
-        mock_resp.text = "<html><dt>broken"  # Malformed HTML
+        mock_resp.content = b"<html><dt>broken"  # Malformed HTML
         mock_resp.raise_for_status = Mock()
         mock_get.return_value = mock_resp
 
diff --git a/uv.lock b/uv.lock
