Skip to content

Commit f5048c6

Browse files
wietzesuijkerwietzesuijker
committed
feat: multi-mission integration tests and examples
1 parent 80ae06a commit f5048c6

File tree

10 files changed

+690
-623
lines changed

10 files changed

+690
-623
lines changed

‎examples/s1_quickstart.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env python3
2+
"""Quick S1 GRD to GeoZarr conversion example.
3+
4+
Demonstrates end-to-end S1 pipeline:
5+
1. Fetch S1 item from STAC
6+
2. Convert to GeoZarr
7+
3. Register in STAC catalog
8+
4. Augment with preview links
9+
"""
10+
11+
import subprocess
12+
import sys
13+
from pathlib import Path
14+
15+
16+
def run_s1_pipeline(
17+
stac_url: str = "https://stac.core.eopf.eodc.eu",
18+
item_id: str = "S1C_IW_GRDH_1SDV_20251008T163126_20251008T163151_004473_008DBA_9AB4",
19+
output_dir: Path = Path("./s1_output"),
20+
) -> int:
21+
"""Run S1 GRD pipeline locally."""
22+
23+
output_dir.mkdir(exist_ok=True)
24+
geozarr_path = output_dir / f"{item_id}_geozarr.zarr"
25+
26+
print(f"🛰️ Processing S1 item: {item_id}")
27+
28+
# Step 1: Get source URL
29+
print("\n1️⃣ Fetching STAC item...")
30+
cmd = [
31+
"python",
32+
"scripts/get_zarr_url.py",
33+
f"{stac_url}/collections/sentinel-1-l1-grd/items/{item_id}",
34+
]
35+
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
36+
source_url = result.stdout.strip()
37+
print(f" Source: {source_url}")
38+
39+
# Step 2: Convert to GeoZarr
40+
print("\n2️⃣ Converting to GeoZarr...")
41+
cmd = [
42+
"eopf-geozarr",
43+
"convert",
44+
source_url,
45+
str(geozarr_path),
46+
"--groups",
47+
"/measurements",
48+
"--gcp-group",
49+
"/conditions/gcp",
50+
"--spatial-chunk",
51+
"2048",
52+
"--verbose",
53+
]
54+
subprocess.run(cmd, check=True)
55+
print(f" ✓ Created: {geozarr_path}")
56+
57+
# Step 3: Validate
58+
print("\n3️⃣ Validating GeoZarr...")
59+
cmd = ["eopf-geozarr", "validate", str(geozarr_path)]
60+
subprocess.run(cmd, check=True)
61+
print(" ✓ Valid GeoZarr")
62+
63+
print("\n✅ S1 pipeline complete!")
64+
print(f" Output: {geozarr_path}")
65+
print("\n Next steps:")
66+
print(" - Upload to S3")
67+
print(" - Register in STAC catalog")
68+
print(" - View in titiler-eopf")
69+
70+
return 0
71+
72+
73+
if __name__ == "__main__":
74+
try:
75+
sys.exit(run_s1_pipeline())
76+
except subprocess.CalledProcessError as e:
77+
print(f"\n❌ Pipeline failed: {e}", file=sys.stderr)
78+
sys.exit(1)
79+
except KeyboardInterrupt:
80+
print("\n⚠️ Interrupted", file=sys.stderr)
81+
sys.exit(130)

‎scripts/test_s1_e2e.sh‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#!/bin/bash
2+
# Test S1 GRD end-to-end pipeline in devseed-staging namespace
3+
#
4+
# This script:
5+
# 1. Applies the workflow template
6+
# 2. Publishes an S1 test payload via AMQP
7+
# 3. Waits for workflow completion
8+
# 4. Shows logs and verifies STAC item was created
9+
10+
set -euo pipefail
11+
12+
# Set kubeconfig
13+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
14+
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
15+
export KUBECONFIG="${KUBECONFIG:-$PROJECT_ROOT/.work/kubeconfig}"
16+
17+
if [ ! -f "$KUBECONFIG" ]; then
18+
echo "❌ Kubeconfig not found at: $KUBECONFIG"
19+
echo "Please set KUBECONFIG environment variable or create .work/kubeconfig"
20+
exit 1
21+
fi
22+
23+
NAMESPACE="${NAMESPACE:-devseed-staging}"
24+
PAYLOAD_FILE="${PAYLOAD_FILE:-workflows/examples/payload-s1.json}"
25+
TIMEOUT="${TIMEOUT:-600}" # 10 minutes
26+
27+
echo "=========================================="
28+
echo "S1 GRD Pipeline E2E Test"
29+
echo "=========================================="
30+
echo "Kubeconfig: $KUBECONFIG"
31+
echo "Namespace: $NAMESPACE"
32+
echo "Payload: $PAYLOAD_FILE"
33+
echo "Timeout: ${TIMEOUT}s"
34+
echo ""
35+
36+
# Step 1: Apply workflow template
37+
echo "📝 Applying workflow template..."
38+
kubectl -n "$NAMESPACE" apply -f workflows/template.yaml
39+
echo "✅ Template applied"
40+
echo ""
41+
42+
# Step 2: Publish AMQP message
43+
echo "📤 Publishing test payload..."
44+
kubectl -n "$NAMESPACE" delete job amqp-publish-once --ignore-not-found=true
45+
kubectl -n "$NAMESPACE" delete configmap amqp-payload --ignore-not-found=true
46+
kubectl -n "$NAMESPACE" create configmap amqp-payload --from-file=body.json="$PAYLOAD_FILE"
47+
kubectl -n "$NAMESPACE" apply -f workflows/amqp-publish-once.yaml
48+
echo "⏳ Waiting for publish job..."
49+
kubectl -n "$NAMESPACE" wait --for=condition=complete --timeout=120s job/amqp-publish-once
50+
echo "✅ Payload published"
51+
echo ""
52+
53+
# Step 3: Get latest workflow
54+
echo "🔍 Finding triggered workflow..."
55+
sleep 3 # Give sensor time to create workflow
56+
WORKFLOW=$(kubectl -n "$NAMESPACE" get wf --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)
57+
if [ -z "$WORKFLOW" ]; then
58+
echo "❌ No workflow found!"
59+
exit 1
60+
fi
61+
echo "✅ Workflow: $WORKFLOW"
62+
echo ""
63+
64+
# Step 4: Wait for completion
65+
echo "⏳ Waiting for workflow completion (timeout: ${TIMEOUT}s)..."
66+
START_TIME=$(date +%s)
67+
while true; do
68+
PHASE=$(kubectl -n "$NAMESPACE" get wf "$WORKFLOW" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown")
69+
ELAPSED=$(($(date +%s) - START_TIME))
70+
71+
echo " [${ELAPSED}s] Phase: $PHASE"
72+
73+
case "$PHASE" in
74+
Succeeded)
75+
echo "✅ Workflow succeeded!"
76+
break
77+
;;
78+
Failed|Error)
79+
echo "❌ Workflow failed!"
80+
break
81+
;;
82+
Unknown)
83+
echo "❌ Workflow disappeared!"
84+
exit 1
85+
;;
86+
esac
87+
88+
if [ $ELAPSED -ge $TIMEOUT ]; then
89+
echo "⏰ Timeout reached!"
90+
break
91+
fi
92+
93+
sleep 5
94+
done
95+
echo ""
96+
97+
# Step 5: Show workflow details
98+
echo "=========================================="
99+
echo "Workflow Details"
100+
echo "=========================================="
101+
kubectl -n "$NAMESPACE" get wf "$WORKFLOW" -o jsonpath='
102+
Name: {.metadata.name}
103+
Status: {.status.phase}
104+
Started: {.status.startedAt}
105+
Finished: {.status.finishedAt}
106+
Duration: {.status.estimatedDuration}
107+
108+
Parameters:
109+
source_url: {.spec.arguments.parameters[?(@.name=="source_url")].value}
110+
item_id: {.spec.arguments.parameters[?(@.name=="item_id")].value}
111+
collection: {.spec.arguments.parameters[?(@.name=="register_collection")].value}
112+
'
113+
echo ""
114+
echo ""
115+
116+
# Step 6: Show pod logs
117+
echo "=========================================="
118+
echo "Pod Logs"
119+
echo "=========================================="
120+
PODS=$(kubectl -n "$NAMESPACE" get pods -l workflows.argoproj.io/workflow="$WORKFLOW" -o name 2>/dev/null || true)
121+
if [ -z "$PODS" ]; then
122+
echo "⚠️ No pods found"
123+
else
124+
for POD in $PODS; do
125+
POD_NAME=$(basename "$POD")
126+
TEMPLATE=$(kubectl -n "$NAMESPACE" get pod "$POD_NAME" -o jsonpath='{.metadata.labels.workflows\.argoproj\.io/template}' 2>/dev/null || echo "unknown")
127+
echo ""
128+
echo "--- $POD_NAME ($TEMPLATE) ---"
129+
kubectl -n "$NAMESPACE" logs "$POD_NAME" --tail=100 -c main 2>/dev/null || echo "No logs available"
130+
done
131+
fi
132+
echo ""
133+
134+
# Step 7: Verify STAC item
135+
echo "=========================================="
136+
echo "STAC Item Verification"
137+
echo "=========================================="
138+
ITEM_ID=$(kubectl -n "$NAMESPACE" get wf "$WORKFLOW" -o jsonpath='{.spec.arguments.parameters[?(@.name=="item_id")].value}')
139+
COLLECTION=$(kubectl -n "$NAMESPACE" get wf "$WORKFLOW" -o jsonpath='{.spec.arguments.parameters[?(@.name=="register_collection")].value}')
140+
STAC_URL="https://api.explorer.eopf.copernicus.eu/stac/collections/$COLLECTION/items/$ITEM_ID"
141+
142+
echo "Checking: $STAC_URL"
143+
ITEM_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$STAC_URL")
144+
if [ "$ITEM_STATUS" = "200" ]; then
145+
echo "✅ STAC item exists!"
146+
echo ""
147+
curl -s "$STAC_URL" | jq '{
148+
id: .id,
149+
collection: .collection,
150+
geometry: .geometry.type,
151+
assets: [.assets | keys[]],
152+
links: [.links[] | select(.rel=="xyz" or .rel=="viewer" or .rel=="tilejson") | {rel, href}]
153+
}'
154+
else
155+
echo "❌ STAC item not found (HTTP $ITEM_STATUS)"
156+
fi
157+
echo ""
158+
159+
echo "=========================================="
160+
echo "Test Summary"
161+
echo "=========================================="
162+
echo "Workflow: $WORKFLOW"
163+
echo "Status: $PHASE"
164+
echo "STAC Item: $ITEM_STATUS"
165+
echo ""
166+
if [ "$PHASE" = "Succeeded" ] && [ "$ITEM_STATUS" = "200" ]; then
167+
echo "🎉 END-TO-END TEST PASSED!"
168+
exit 0
169+
else
170+
echo "❌ END-TO-END TEST FAILED"
171+
exit 1
172+
fi

0 commit comments

Comments
 (0)