Merge pull request #208 from OpenUpSA/elastic-bottleneck

flatten isolate data

Author: desafinadude

helpers.py

@@ -93,11 +93,24 @@ def bulk_send_to_elastic(documents):
 
     bulk_lines = []
     for doc in documents:
-        doc_id = doc.get("id")
-        action = {"index": {"_id": doc_id}} if doc_id else {"index": {}}
-        bulk_lines.append(json.dumps(action))
-        # Use json_serial for datetime/date serialization
-        bulk_lines.append(json.dumps(doc, default=json_serial))
+            # Flatten isolate_data if present (same logic as send_to_elastic2)
+            if 'isolate_data' in doc and doc['isolate_data']:
+                isolate_data = doc['isolate_data']
+                if isinstance(isolate_data, str):
+                    try:
+                        isolate_data = json.loads(isolate_data)
+                    except Exception:
+                        isolate_data = {}
+                if isinstance(isolate_data, dict):
+                    for key, value in isolate_data.items():
+                        if key not in doc:
+                            doc[key] = value
+                del doc['isolate_data']
+            doc_id = doc.get("id")
+            action = {"index": {"_id": doc_id}} if doc_id else {"index": {}}
+            bulk_lines.append(json.dumps(action))
+            # Use json_serial for datetime/date serialization
+            bulk_lines.append(json.dumps(doc, default=json_serial))
     bulk_data = "\n".join(bulk_lines) + "\n"
 
     url = f"{settings.ELASTICSEARCH_URL}/{settings.ELASTICSEARCH_INDEX}/_bulk"

test/test_submissions_validation.py

@@ -976,11 +976,13 @@ def test_validate_submission_sends_validated_isolates_to_elasticsearch(
         setup_tsv_rows(tsv_rows)
 
         url = get_validation_endpoint(public_project1["id"], submission_id)
-        response = make_request(client, "POST", url, token=org1_admin_token)
-
-        assert response.status_code == 200
-        # Verify send_to_elastic2 was called for each validated isolate
-        assert mock_validation_stack["elastic"].call_count == 2
+        with patch("app.bulk_send_to_elastic") as mock_bulk:
+            response = make_request(client, "POST", url, token=org1_admin_token)
+            assert response.status_code == 200
+            # Verify bulk_send_to_elastic was called once with both isolates
+            mock_bulk.assert_called_once()
+            args, kwargs = mock_bulk.call_args
+            assert len(args[0]) == 2
 
     finally:
         cleanup_submission(submission_id)