fix: ensure the decoded text from the document is utf-8 compat (#223)

resolves #222

---------

Signed-off-by: Anupam Kumar <[email protected]>

Author: kyteinsky

.github/workflows/integration-test.yml

@@ -61,7 +61,7 @@ jobs:
       # do not stop on another job's failure
       fail-fast: false
       matrix:
-        php-versions: [ '8.1' ]
+        php-versions: [ '8.2' ]
         databases: [ 'pgsql' ]
         server-versions: [ 'stable32', 'master' ]
 

context_chat_backend/chain/ingest/doc_loader.py

@@ -129,7 +129,7 @@ def decode_source(source: UploadFile) -> str | None:
 		if _loader_map.get(mimetype):
 			result = _loader_map[mimetype](source.file)
 			source.file.close()
-			return result
+			return result.encode('utf-8', 'ignore').decode('utf-8', 'ignore')
 
 		result = source.file.read().decode('utf-8', 'ignore')
 		source.file.close()