refact tests

.claude/settings.local.json

@@ -0,0 +1,14 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(docker logs:*)",
+      "Bash(docker compose stop:*)",
+      "Bash(docker compose rm:*)",
+      "Bash(docker volume:*)",
+      "Bash(docker compose:*)",
+      "Bash(uv run pytest:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

.github/workflows/test-emulators.yaml

@@ -19,7 +19,7 @@ concurrency:
 jobs:
   unit-integration-e2e:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
 
     steps:
     - uses: actions/checkout@v5
@@ -37,7 +37,14 @@ jobs:
         uv sync --all-extras --frozen
 
     - name: Clean up any existing Docker volumes
-      run: docker compose down -v || true
+      run: |
+        # Only clean up if services are already running
+        if docker compose ps -q 2>/dev/null | grep -q .; then
+          echo "Cleaning up existing services and volumes..."
+          docker compose down -v
+        else
+          echo "No existing services to clean up."
+        fi
 
     - name: Pre-build selected images (incl. Postgres 18)
       run: bash scripts/prebuild-images.sh a2a-inspector firebase-emulator postgres
@@ -47,12 +54,20 @@ jobs:
 
     - name: Wait for services to be ready
       run: bash scripts/wait-for-services.sh --default 90 --a2a 180 --postgres 150
-
+
+    - name: Test Elasticsearch separately
+      timeout-minutes: 2
+      run: bash scripts/test-elasticsearch.sh
+
     - name: Run unit/integration tests (non-e2e)
+      timeout-minutes: 5
       run: bash scripts/run-tests-fast.sh
 
     - name: Run e2e tests
-      run: bash scripts/run-tests-e2e.sh | tee e2e.log
+      timeout-minutes: 20
+      run: |
+        set -o pipefail
+        bash scripts/run-tests-e2e.sh | tee e2e.log
 
     - name: Summarize skipped e2e tests
       if: always()

docker-compose.yaml

@@ -300,7 +300,7 @@ services:
       - cli  # Only run when explicitly requested
   # Elasticsearch
   elasticsearch:
-    image: elasticsearch:8.19.5
+    image: elasticsearch:9.2.1
     container_name: elasticsearch-emulator
     ports:
       - "${ELASTICSEARCH_PORT:-9200}:9200"  # REST API
@@ -309,6 +309,9 @@ services:
       - discovery.type=single-node
       - xpack.security.enabled=false
       - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      - "bootstrap.memory_lock=false"
+      - "cluster.routing.allocation.disk.threshold_enabled=false"
+      - "action.destructive_requires_name=false"
     volumes:
       - elasticsearch_data:/usr/share/elasticsearch/data
     healthcheck:

elasticsearch-cli/main.go

@@ -16,9 +16,10 @@ import (
 )
 
 var (
-	host   string
-	port   string
-	client *http.Client
+	host    string
+	port    string
+	client  *http.Client
+	verbose bool
 )
 
 func init() {
@@ -32,8 +33,11 @@ func init() {
 		port = "9200"
 	}
 
+	// Enable verbose logging for debugging (especially in CI)
+	verbose = os.Getenv("ES_CLI_VERBOSE") == "1" || os.Getenv("ES_CLI_VERBOSE") == "true"
+
 	client = &http.Client{
-		Timeout: 30 * time.Second,
+		Timeout: 60 * time.Second,
 	}
 }
 
@@ -236,6 +240,14 @@ func executeCommand(command string) {
 		return
 	}
 
+	// Wait for index shards to be ready after PUT (index creation)
+	if method == "PUT" && strings.HasPrefix(path, "/") && !strings.Contains(path, "/_") {
+		indexName := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
+		if indexName != "" {
+			waitForIndexReady(indexName)
+		}
+	}
+
 	// Pretty print JSON response
 	var prettyJSON bytes.Buffer
 	if err := json.Indent(&prettyJSON, []byte(resp), "", "  "); err != nil {
@@ -247,15 +259,65 @@ func executeCommand(command string) {
 	fmt.Printf("\nTime: %v\n", time.Since(start))
 }
 
+func waitForIndexReady(indexName string) {
+	// Wait for index shards to be ready (yellow or green status)
+	// Use a longer timeout for CI environments and poll for readiness
+	maxRetries := 60 // 60 seconds total
+
+	if verbose {
+		fmt.Printf("[VERBOSE] Waiting for index '%s' to be ready (max %ds)...\n", indexName, maxRetries)
+	}
+
+	for i := 0; i < maxRetries; i++ {
+		healthPath := fmt.Sprintf("/_cluster/health/%s", indexName)
+		start := time.Now()
+		resp, err := makeRequest("GET", healthPath, nil)
+		elapsed := time.Since(start)
+
+		if err != nil {
+			if verbose {
+				fmt.Printf("[VERBOSE] Retry %d/%d: Health check failed after %v: %v\n", i+1, maxRetries, elapsed, err)
+			}
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		result := gjson.Parse(resp)
+		status := result.Get("status").String()
+		initializingShards := result.Get("initializing_shards").Int()
+		activeShards := result.Get("active_shards").Int()
+
+		if verbose {
+			fmt.Printf("[VERBOSE] Retry %d/%d: status=%s, initializing_shards=%d, active_shards=%d (took %v)\n",
+				i+1, maxRetries, status, initializingShards, activeShards, elapsed)
+		}
+
+		if (status == "green" || status == "yellow") && initializingShards == 0 {
+			if verbose {
+				fmt.Printf("[VERBOSE] Index '%s' ready after %d attempts (%.2fs total)\n", indexName, i+1, float64(i+1))
+			}
+			return // Index is ready
+		}
+
+		time.Sleep(1 * time.Second)
+	}
+
+	// Log warning but don't fail - the index might still become available
+	fmt.Printf("Warning: Index %s not ready after %d seconds\n", indexName, maxRetries)
+}
+
 func makeRequest(method, path string, body []byte) (string, error) {
 	url := fmt.Sprintf("http://%s:%s%s", host, port, path)
+	start := time.Now()
 
 	var req *http.Request
 	var err error
 
 	if body != nil {
 		req, err = http.NewRequest(method, url, bytes.NewBuffer(body))
-		req.Header.Set("Content-Type", "application/json")
+		if err == nil {
+			req.Header.Set("Content-Type", "application/json")
+		}
 	} else {
 		req, err = http.NewRequest(method, url, nil)
 	}
@@ -264,17 +326,41 @@ func makeRequest(method, path string, body []byte) (string, error) {
 		return "", err
 	}
 
+	if verbose {
+		bodyPreview := ""
+		if body != nil && len(body) > 0 {
+			if len(body) > 100 {
+				bodyPreview = fmt.Sprintf(" (body: %d bytes)", len(body))
+			} else {
+				bodyPreview = fmt.Sprintf(" (body: %s)", string(body))
+			}
+		}
+		fmt.Printf("[VERBOSE] Request: %s %s%s\n", method, path, bodyPreview)
+	}
+
 	resp, err := client.Do(req)
+	elapsed := time.Since(start)
+
 	if err != nil {
+		if verbose {
+			fmt.Printf("[VERBOSE] Request failed after %v: %v\n", elapsed, err)
+		}
 		return "", err
 	}
 	defer resp.Body.Close()
 
 	respBody, err := io.ReadAll(resp.Body)
 	if err != nil {
+		if verbose {
+			fmt.Printf("[VERBOSE] Failed to read response body after %v: %v\n", elapsed, err)
+		}
 		return "", err
 	}
 
+	if verbose {
+		fmt.Printf("[VERBOSE] Response: HTTP %d (took %v, %d bytes)\n", resp.StatusCode, elapsed, len(respBody))
+	}
+
 	if resp.StatusCode >= 400 {
 		return string(respBody), fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
 	}

justfile

@@ -41,14 +41,22 @@ up nobuild='no':
     fi
 
 # Wait for services
-wait default='60' a2a='180':
-    @bash scripts/wait-for-services.sh --default {{default}} --a2a {{a2a}}
+wait default='30' a2a='60' postgres='60':
+    @bash scripts/wait-for-services.sh --default {{default}} --a2a {{a2a}} --postgres {{postgres}}
+
+# Clean up volumes (use with caution - deletes all data)
+clean-volumes:
+    @echo '⚠️  Cleaning up Docker volumes...'
+    docker compose down -v || true
+    @echo '✅ Volumes cleaned.'
 
 # One-shot: prebuild -> up -> wait
 start:
+    @echo '🧹 Cleaning up old volumes...'
+    @docker compose down -v || true
     @bash scripts/prebuild-images.sh a2a-inspector firebase-emulator postgres
     @bash scripts/start-services.sh
-    @bash scripts/wait-for-services.sh --default 60 --a2a 180
+    @bash scripts/wait-for-services.sh --default 30 --a2a 60 --postgres 60
 
 # Stop emulators (with Firebase export)
 stop:
@@ -90,7 +98,7 @@ lint path='tests/' opts='--fix':
     @echo '🔍 Linting code with ruff...'
     uv run ruff check '{{path}}' '{{opts}}'
     @echo 'Semgrep linting...'
-    uv run semgrep --config .semgrep/
+    uv run semgrep --config .semgrep/ --error
     @echo '✅ Linting finished.'
 
 

pyproject.toml

@@ -22,10 +22,15 @@ dev = [
     "docker>=7.1.0",
     "pytest>=8.4.1",
     "pytest-asyncio>=0.25.0",
+    "pytest-timeout>=2.3.1",
     "ruff>=0.12.4",
 ]
 
 [tool.pytest.ini_options]
 markers = [
     "e2e: end-to-end tests that require Docker and running emulators",
 ]
+# Default timeout for all tests (can be overridden per test)
+timeout = 180
+# Timeout method: 'thread' is more compatible with async tests
+timeout_method = "thread"

scripts/run-tests-e2e.sh

@@ -9,6 +9,8 @@ if ! command -v uv >/dev/null 2>&1; then
 fi
 
 echo "Running E2E tests"
+# E2E tests run sequentially due to Docker client state management
+# Parallel execution (-n auto) causes Docker API 404 errors
 uv run pytest tests/e2e -v -m e2e -ra
 echo "Done."
 

scripts/test-elasticsearch.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Test Elasticsearch functionality
+# Usage: bash scripts/test-elasticsearch.sh
+
+echo "Testing Elasticsearch..."
+
+# 1. Check cluster health
+echo "1. Checking cluster health..."
+HEALTH_RESPONSE=$(curl -s http://localhost:9200/_cluster/health)
+echo "Response: $HEALTH_RESPONSE"
+
+STATUS=$(echo "$HEALTH_RESPONSE" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+echo "Cluster status: $STATUS"
+
+if [[ "$STATUS" != "green" && "$STATUS" != "yellow" ]]; then
+  echo "ERROR: Cluster status is not green or yellow" >&2
+  exit 1
+fi
+
+# 2. Check if shards are initialized
+INITIALIZING_SHARDS=$(echo "$HEALTH_RESPONSE" | grep -o '"initializing_shards":[0-9]*' | cut -d':' -f2)
+echo "Initializing shards: $INITIALIZING_SHARDS"
+
+if [[ "$INITIALIZING_SHARDS" != "0" ]]; then
+  echo "WARNING: Shards are still initializing" >&2
+fi
+
+# 3. Create a test index
+TEST_INDEX="es_health_check_$(date +%s)"
+echo ""
+echo "2. Creating test index: $TEST_INDEX"
+CREATE_RESPONSE=$(curl -s -X PUT "http://localhost:9200/$TEST_INDEX" \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "settings": {
+      "number_of_shards": 1,
+      "number_of_replicas": 0
+    }
+  }')
+echo "Response: $CREATE_RESPONSE"
+
+# 4. Wait for index to be ready
+echo ""
+echo "3. Waiting for index to be ready..."
+MAX_WAIT=30
+for i in $(seq 1 $MAX_WAIT); do
+  INDEX_HEALTH=$(curl -s "http://localhost:9200/_cluster/health/$TEST_INDEX?wait_for_status=yellow&timeout=1s")
+  INDEX_STATUS=$(echo "$INDEX_HEALTH" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+
+  if [[ "$INDEX_STATUS" == "green" || "$INDEX_STATUS" == "yellow" ]]; then
+    echo "Index is ready (status: $INDEX_STATUS)"
+    break
+  fi
+
+  if [[ $i -eq $MAX_WAIT ]]; then
+    echo "ERROR: Index not ready after ${MAX_WAIT}s" >&2
+    exit 1
+  fi
+
+  sleep 1
+done
+
+# 5. Insert a test document
+echo ""
+echo "4. Inserting test document..."
+INSERT_RESPONSE=$(curl -s -X POST "http://localhost:9200/$TEST_INDEX/_doc" \
+  -H 'Content-Type: application/json' \
+  -d '{"test": "document", "timestamp": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"}')
+echo "Response: $INSERT_RESPONSE"
+
+# 6. Search for the document
+echo ""
+echo "5. Searching for test document..."
+sleep 1  # Wait for indexing
+SEARCH_RESPONSE=$(curl -s -X GET "http://localhost:9200/$TEST_INDEX/_search" \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "query": {
+      "match_all": {}
+    }
+  }')
+echo "Response: $SEARCH_RESPONSE"
+
+HITS=$(echo "$SEARCH_RESPONSE" | grep -o '"total":{"value":[0-9]*' | grep -o '[0-9]*$')
+echo "Total hits: $HITS"
+
+if [[ "$HITS" != "1" ]]; then
+  echo "ERROR: Expected 1 hit, got $HITS" >&2
+  exit 1
+fi
+
+# 7. Delete the test index
+echo ""
+echo "6. Cleaning up test index..."
+DELETE_RESPONSE=$(curl -s -X DELETE "http://localhost:9200/$TEST_INDEX")
+echo "Response: $DELETE_RESPONSE"
+
+echo ""
+echo "✓ All Elasticsearch tests passed!"