diff --git a/.github/actions/setup-integration-test-env/action.yml b/.github/actions/setup-integration-test-env/action.yml index 87c59d2f..a1b256f6 100644 --- a/.github/actions/setup-integration-test-env/action.yml +++ b/.github/actions/setup-integration-test-env/action.yml @@ -80,7 +80,7 @@ runs: env: TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:${{ inputs.origin-port }} TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret - TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY: integration-test-secret-key + TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-32-bytes TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 186569da..3ea74e4c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,14 @@ jobs: - name: Run tests run: cargo test --workspace + - name: Verify Fastly WASM release build + env: + TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:8080 + TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret + TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-32-bytes + TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" + run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 + test-typescript: name: vitest runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index ec76ee46..b5e2b6f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -366,7 +366,7 @@ both runtime behavior and build/tooling changes. | `crates/trusted-server-core/src/tsjs.rs` | Script tag generation with module IDs | | `crates/trusted-server-core/src/html_processor.rs` | Injects `"#; - let params = OwnedProcessResponseParams { - content_encoding: String::new(), - origin_host: "origin.example.com".to_string(), - origin_url: "https://origin.example.com".to_string(), - request_host: "proxy.example.com".to_string(), - request_scheme: "https".to_string(), - content_type: "text/html".to_string(), - }; - - let mut output = Vec::new(); - stream_publisher_body( - Body::from(html.to_vec()), - &mut output, - ¶ms, - &settings, - ®istry, - ) - .expect("should process RSC push"); - - let processed = String::from_utf8(output).expect("valid UTF-8"); - assert!( - !processed.contains("__ts_rsc_payload_"), - "placeholder must be substituted before reaching output. Got: {processed}" - ); - assert!( - processed.contains("proxy.example.com/page"), - "origin URL must be rewritten in the substituted payload. Got: {processed}" - ); - assert!( - !processed.contains("origin.example.com"), - "origin host must not leak. Got: {processed}" - ); - } } diff --git a/crates/trusted-server-core/src/request_signing/endpoints.rs b/crates/trusted-server-core/src/request_signing/endpoints.rs index 3b14317e..04638db0 100644 --- a/crates/trusted-server-core/src/request_signing/endpoints.rs +++ b/crates/trusted-server-core/src/request_signing/endpoints.rs @@ -7,7 +7,7 @@ use error_stack::{Report, ResultExt}; use fastly::{Request, Response}; use serde::{Deserialize, Serialize}; -use crate::error::TrustedServerError; +use crate::error::{IntoHttpResponse, TrustedServerError}; use crate::platform::RuntimeServices; use crate::request_signing::discovery::TrustedServerDiscovery; use crate::request_signing::rotation::KeyRotationManager; @@ -52,18 +52,27 @@ pub fn handle_trusted_server_discovery( .with_body(json)) } +/// JSON request body for the signature verification endpoint. #[derive(Debug, Deserialize, Serialize)] pub struct VerifySignatureRequest { + /// Canonical payload that was signed. pub payload: String, + /// Base64-encoded Ed25519 signature to verify. pub signature: String, + /// Key identifier used to look up the public JWK. pub kid: String, } +/// JSON response body for the signature verification endpoint. #[derive(Debug, Deserialize, Serialize)] pub struct VerifySignatureResponse { + /// Whether signature verification succeeded. pub verified: bool, + /// Key identifier that was used during verification. pub kid: String, + /// Human-readable verification result summary. pub message: String, + /// Error detail when verification fails unexpectedly. #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } @@ -73,9 +82,11 @@ pub struct VerifySignatureResponse { /// /// # Errors /// -/// Returns an error if the request body cannot be parsed as JSON or if verification fails. +/// Returns an error if the request body cannot be parsed as JSON or if the +/// response body cannot be serialized. pub fn handle_verify_signature( _settings: &Settings, + services: &RuntimeServices, mut req: Request, ) -> Result> { let body = req.take_body_str(); @@ -88,6 +99,7 @@ pub fn handle_verify_signature( verify_req.payload.as_bytes(), &verify_req.signature, &verify_req.kid, + services, ); let response = match verification_result { @@ -103,12 +115,15 @@ pub fn handle_verify_signature( message: "Signature verification failed".into(), error: Some("Invalid signature".into()), }, - Err(e) => VerifySignatureResponse { - verified: false, - kid: verify_req.kid, - message: "Verification error".into(), - error: Some(format!("{}", e)), - }, + Err(e) => { + log::warn!("signature verification failed: {e}"); + VerifySignatureResponse { + verified: false, + kid: verify_req.kid, + message: "Verification error".into(), + error: Some("internal verification error".into()), + } + } }; let response_json = serde_json::to_string(&response).map_err(|e| { @@ -122,42 +137,101 @@ pub fn handle_verify_signature( .with_body(response_json)) } +/// JSON request body for the key-rotation endpoint. #[derive(Debug, Deserialize, Serialize)] pub struct RotateKeyRequest { + /// Optional explicit key identifier for the new signing key. #[serde(skip_serializing_if = "Option::is_none")] pub kid: Option, } +/// JSON response body for the key-rotation endpoint. #[derive(Debug, Deserialize, Serialize)] pub struct RotateKeyResponse { + /// Whether the rotation operation succeeded. pub success: bool, + /// Human-readable summary of the rotation result. pub message: String, + /// Newly generated or supplied key identifier. pub new_kid: String, + /// Previously active key identifier, if one existed. pub previous_kid: Option, + /// Active key identifiers after the rotation completes. pub active_kids: Vec, + /// Public JWK associated with the newly active key. pub jwk: serde_json::Value, + /// Error detail when rotation fails. #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } -/// Rotates the current active kid by generating and saving a new one +struct SigningStoreIds<'a> { + config_store_id: &'a str, + secret_store_id: &'a str, +} + +const MAX_KID_LENGTH: usize = 128; + +fn signing_store_ids( + settings: &Settings, +) -> Result, Report> { + settings + .request_signing + .as_ref() + .map(|setting| SigningStoreIds { + config_store_id: setting.config_store_id.as_str(), + secret_store_id: setting.secret_store_id.as_str(), + }) + .ok_or_else(|| { + TrustedServerError::Configuration { + message: "missing signing storage configuration".to_string(), + } + .into() + }) +} + +fn validate_kid(kid: &str) -> Result<(), Report> { + if kid.is_empty() || kid.len() > MAX_KID_LENGTH { + return Err(Report::new(TrustedServerError::BadRequest { + message: format!("kid must be 1..={MAX_KID_LENGTH} characters"), + })); + } + + if !kid + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ':')) + { + return Err(Report::new(TrustedServerError::BadRequest { + message: "kid must contain only ASCII alphanumerics, '-', '_', '.', ':'".into(), + })); + } + + Ok(()) +} + +/// Rotates the current active kid by generating and saving a new one. +/// +/// # Response contract +/// +/// Returns `200 OK` with `success: true` on success, `400 Bad Request` for an +/// invalid operator-supplied `kid`, or `500 Internal Server Error` when rotation +/// fails. Failure responses include `success: false` and a populated `error` +/// field. Unlike [`handle_verify_signature`], the error field contains internal +/// detail — this is intentional because this endpoint is auth-gated and +/// operator-facing only. /// /// # Errors /// -/// Returns an error if the request signing settings are missing, JSON parsing fails, or key rotation fails. +/// Returns an error if the request signing settings are missing or JSON parsing fails. pub fn handle_rotate_key( settings: &Settings, + services: &RuntimeServices, mut req: Request, ) -> Result> { - let (config_store_id, secret_store_id) = match &settings.request_signing { - Some(setting) => (&setting.config_store_id, &setting.secret_store_id), - None => { - return Err(TrustedServerError::Configuration { - message: "missing signing storage configuration".to_string(), - } - .into()); - } - }; + let SigningStoreIds { + config_store_id, + secret_store_id, + } = signing_store_ids(settings)?; let body = req.take_body_str(); let rotate_req: RotateKeyRequest = if body.is_empty() { @@ -168,13 +242,15 @@ pub fn handle_rotate_key( })? }; - let manager = KeyRotationManager::new(config_store_id, secret_store_id).change_context( - TrustedServerError::Configuration { - message: "failed to create KeyRotationManager".into(), - }, - )?; + let manager = KeyRotationManager::new(config_store_id, secret_store_id); + let validation_result = if let Some(kid) = rotate_req.kid.as_deref() { + validate_kid(kid) + } else { + Ok(()) + }; + let result = validation_result.and_then(|()| manager.rotate_key(services, rotate_req.kid)); - match manager.rotate_key(rotate_req.kid) { + match result { Ok(result) => { let jwk_value = serde_json::to_value(&result.jwk).map_err(|e| { Report::new(TrustedServerError::Configuration { @@ -203,6 +279,7 @@ pub fn handle_rotate_key( .with_body(response_json)) } Err(e) => { + let status = e.current_context().status_code(); let response = RotateKeyResponse { success: false, message: "Key rotation failed".to_string(), @@ -219,49 +296,64 @@ pub fn handle_rotate_key( }) })?; - Ok(Response::from_status(500) + Ok(Response::from_status(status) .with_content_type(fastly::mime::APPLICATION_JSON) .with_body(response_json)) } } } +/// JSON request body for the key-deactivation endpoint. #[derive(Debug, Deserialize, Serialize)] pub struct DeactivateKeyRequest { + /// Key identifier to deactivate or delete. pub kid: String, + /// Whether the key should be deleted from storage after deactivation. #[serde(default)] pub delete: bool, } +/// JSON response body for the key-deactivation endpoint. #[derive(Debug, Deserialize, Serialize)] pub struct DeactivateKeyResponse { + /// Whether the deactivation or deletion succeeded. pub success: bool, + /// Human-readable summary of the operation result. pub message: String, + /// Key identifier that was deactivated or deleted. pub deactivated_kid: String, + /// Whether the key was deleted from storage. pub deleted: bool, + /// Active key identifiers remaining after the operation. pub remaining_active_kids: Vec, + /// Error detail when the operation fails. #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } -/// Deactivates an active key +/// Deactivates or deletes an active signing key. +/// +/// # Response contract +/// +/// Returns `200 OK` with `success: true` on success, `400 Bad Request` for an +/// invalid operator-supplied `kid`, or `500 Internal Server Error` when +/// deactivation fails. Failure responses include `success: false` and a populated +/// `error` field. Like [`handle_rotate_key`] and unlike +/// [`handle_verify_signature`], the error field contains internal detail — this +/// is intentional because this endpoint is auth-gated and operator-facing only. /// /// # Errors /// -/// Returns an error if the request signing settings are missing, JSON parsing fails, or key deactivation fails. +/// Returns an error if the request signing settings are missing or JSON parsing fails. pub fn handle_deactivate_key( settings: &Settings, + services: &RuntimeServices, mut req: Request, ) -> Result> { - let (config_store_id, secret_store_id) = match &settings.request_signing { - Some(setting) => (&setting.config_store_id, &setting.secret_store_id), - None => { - return Err(TrustedServerError::Configuration { - message: "missing signing storage configuration".to_string(), - } - .into()); - } - }; + let SigningStoreIds { + config_store_id, + secret_store_id, + } = signing_store_ids(settings)?; let body = req.take_body_str(); let deactivate_req: DeactivateKeyRequest = @@ -269,21 +361,19 @@ pub fn handle_deactivate_key( message: "invalid JSON request body".into(), })?; - let manager = KeyRotationManager::new(config_store_id, secret_store_id).change_context( - TrustedServerError::Configuration { - message: "failed to create KeyRotationManager".into(), - }, - )?; + let manager = KeyRotationManager::new(config_store_id, secret_store_id); - let result = if deactivate_req.delete { - manager.delete_key(&deactivate_req.kid) - } else { - manager.deactivate_key(&deactivate_req.kid) - }; + let result = validate_kid(&deactivate_req.kid).and_then(|()| { + if deactivate_req.delete { + manager.delete_key(services, &deactivate_req.kid) + } else { + manager.deactivate_key(services, &deactivate_req.kid) + } + }); match result { Ok(()) => { - let remaining_keys = manager.list_active_keys().unwrap_or_else(|e| { + let remaining_keys = manager.list_active_keys(services).unwrap_or_else(|e| { log::warn!("failed to list active keys after deactivation: {}", e); vec![] }); @@ -312,6 +402,7 @@ pub fn handle_deactivate_key( .with_body(response_json)) } Err(e) => { + let status = e.current_context().status_code(); let response = DeactivateKeyResponse { success: false, message: if deactivate_req.delete { @@ -331,7 +422,7 @@ pub fn handle_deactivate_key( }) })?; - Ok(Response::from_status(500) + Ok(Response::from_status(status) .with_content_type(fastly::mime::APPLICATION_JSON) .with_body(response_json)) } @@ -340,10 +431,8 @@ pub fn handle_deactivate_key( #[cfg(test)] mod tests { - use error_stack::Report; - use crate::platform::{ - test_support::{build_services_with_config, noop_services}, + test_support::{build_request_signing_services, build_services_with_config, noop_services}, PlatformConfigStore, PlatformError, StoreId, StoreName, }; @@ -373,19 +462,19 @@ mod tests { Err(Report::new(PlatformError::Unsupported)) } } + #[test] fn test_handle_verify_signature_valid() { let settings = crate::test_support::tests::create_test_settings(); + let services = build_request_signing_services(); - // First, create a valid signature let payload = "test message"; - let signer = crate::request_signing::RequestSigner::from_config() - .expect("should create signer from config"); + let signer = crate::request_signing::RequestSigner::from_services(&services) + .expect("should create signer from services"); let signature = signer .sign(payload.as_bytes()) .expect("should sign payload"); - // Create verification request let verify_req = VerifySignatureRequest { payload: payload.to_string(), signature, @@ -396,9 +485,8 @@ mod tests { let mut req = Request::new(Method::POST, "https://test.com/verify-signature"); req.set_body(body); - // Handle the request - let mut resp = - handle_verify_signature(&settings, req).expect("should handle verification request"); + let mut resp = handle_verify_signature(&settings, &services, req) + .expect("should handle verification request"); assert_eq!(resp.get_status(), StatusCode::OK); assert_eq!( resp.get_content_type(), @@ -406,12 +494,11 @@ mod tests { "should return application/json content type" ); - // Parse response let resp_body = resp.take_body_str(); let verify_resp: VerifySignatureResponse = serde_json::from_str(&resp_body).expect("should deserialize verify response"); - assert!(verify_resp.verified, "Signature should be verified"); + assert!(verify_resp.verified, "should verify a valid signature"); assert_eq!(verify_resp.kid, signer.kid); assert!(verify_resp.error.is_none()); } @@ -419,15 +506,15 @@ mod tests { #[test] fn test_handle_verify_signature_invalid() { let settings = crate::test_support::tests::create_test_settings(); - let signer = crate::request_signing::RequestSigner::from_config() - .expect("should create signer from config"); + let services = build_request_signing_services(); + + let signer = crate::request_signing::RequestSigner::from_services(&services) + .expect("should create signer from services"); - // Create a signature for a different payload let wrong_signature = signer .sign(b"different payload") .expect("should sign different payload"); - // Create request with signature that does not match the payload let verify_req = VerifySignatureRequest { payload: "test message".to_string(), signature: wrong_signature, @@ -438,9 +525,8 @@ mod tests { let mut req = Request::new(Method::POST, "https://test.com/verify-signature"); req.set_body(body); - // Handle the request - let mut resp = - handle_verify_signature(&settings, req).expect("should handle verification request"); + let mut resp = handle_verify_signature(&settings, &services, req) + .expect("should handle verification request"); assert_eq!(resp.get_status(), StatusCode::OK); assert_eq!( resp.get_content_type(), @@ -448,16 +534,59 @@ mod tests { "should return application/json content type" ); - // Parse response let resp_body = resp.take_body_str(); let verify_resp: VerifySignatureResponse = serde_json::from_str(&resp_body).expect("should deserialize verify response"); - assert!(!verify_resp.verified, "Invalid signature should not verify"); + assert!( + !verify_resp.verified, + "should not verify an invalid signature" + ); assert_eq!(verify_resp.kid, signer.kid); assert!(verify_resp.error.is_some()); } + #[test] + fn test_handle_verify_signature_hides_internal_error_details() { + let settings = crate::test_support::tests::create_test_settings(); + + let verify_req = VerifySignatureRequest { + payload: "test message".to_string(), + signature: "any-signature".to_string(), + kid: "missing-kid".to_string(), + }; + + let body = serde_json::to_string(&verify_req).expect("should serialize verify request"); + let mut req = Request::new(Method::POST, "https://test.com/verify-signature"); + req.set_body(body); + + let services = noop_services(); + let mut resp = handle_verify_signature(&settings, &services, req) + .expect("should return a verification response for internal errors"); + + assert_eq!(resp.get_status(), StatusCode::OK, "should return 200 OK"); + + let resp_body = resp.take_body_str(); + let verify_resp: VerifySignatureResponse = + serde_json::from_str(&resp_body).expect("should deserialize verify response"); + + assert!( + !verify_resp.verified, + "should mark internal verification errors as unverified" + ); + assert_eq!(verify_resp.kid, "missing-kid"); + assert_eq!(verify_resp.message, "Verification error"); + assert_eq!( + verify_resp.error.as_deref(), + Some("internal verification error"), + "should return a generic error to unauthenticated callers" + ); + assert!( + !resp_body.contains("failed"), + "should not leak internal error details in the response body" + ); + } + #[test] fn test_handle_verify_signature_malformed_request() { let settings = crate::test_support::tests::create_test_settings(); @@ -465,30 +594,36 @@ mod tests { let mut req = Request::new(Method::POST, "https://test.com/verify-signature"); req.set_body("not valid json"); - // Should return an error response - let result = handle_verify_signature(&settings, req); + let result = handle_verify_signature(&settings, &noop_services(), req); assert!(result.is_err(), "Malformed JSON should error"); } #[test] fn test_handle_rotate_key_with_empty_body() { let settings = crate::test_support::tests::create_test_settings(); - let req = Request::new(Method::POST, "https://test.com/admin/keys/rotate"); - - let result = handle_rotate_key(&settings, req); - match result { - Ok(mut resp) => { - let body = resp.take_body_str(); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - log::debug!( - "Rotation response: success={}, message={}", - response.success, - response.message - ); - } - Err(e) => log::debug!("Expected error in test environment: {}", e), - } + let req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/rotate"); + + let mut resp = handle_rotate_key(&settings, &noop_services(), req) + .expect("should return a response even when stores are unavailable"); + + assert_eq!( + resp.get_status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should return 500 when store writes fail" + ); + + let body = resp.take_body_str(); + let response: RotateKeyResponse = + serde_json::from_str(&body).expect("should deserialize rotate response"); + + assert!( + !response.success, + "should report failure when store writes fail" + ); + assert!( + response.error.is_some(), + "should include error detail in failure response" + ); } #[test] @@ -500,35 +635,80 @@ mod tests { }; let body_json = serde_json::to_string(&req_body).expect("should serialize rotate request"); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/rotate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/rotate"); req.set_body(body_json); - let result = handle_rotate_key(&settings, req); - match result { - Ok(mut resp) => { - let body = resp.take_body_str(); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - log::debug!( - "Custom KID rotation: success={}, new_kid={}", - response.success, - response.new_kid - ); - } - Err(e) => log::debug!("Expected error in test environment: {}", e), - } + let mut resp = handle_rotate_key(&settings, &noop_services(), req) + .expect("should return a response even when stores are unavailable"); + + assert_eq!( + resp.get_status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should return 500 when store writes fail" + ); + + let body = resp.take_body_str(); + let response: RotateKeyResponse = + serde_json::from_str(&body).expect("should deserialize rotate response"); + + assert!( + !response.success, + "should report failure when store writes fail" + ); + assert!( + response.error.is_some(), + "should include error detail in failure response" + ); } #[test] fn test_handle_rotate_key_invalid_json() { let settings = crate::test_support::tests::create_test_settings(); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/rotate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/rotate"); req.set_body("invalid json"); - let result = handle_rotate_key(&settings, req); + let result = handle_rotate_key(&settings, &noop_services(), req); assert!(result.is_err(), "Invalid JSON should return error"); } + #[test] + fn test_handle_rotate_key_rejects_invalid_kid() { + let settings = crate::test_support::tests::create_test_settings(); + + let req_body = RotateKeyRequest { + kid: Some("bad,kid".to_string()), + }; + + let body_json = serde_json::to_string(&req_body).expect("should serialize rotate request"); + let mut req = Request::new(Method::POST, "https://test.com/admin/keys/rotate"); + req.set_body(body_json); + + let mut resp = handle_rotate_key(&settings, &noop_services(), req) + .expect("should return a response for invalid kid"); + + assert_eq!( + resp.get_status(), + StatusCode::BAD_REQUEST, + "should reject malformed kid as a bad request" + ); + + let body = resp.take_body_str(); + let response: RotateKeyResponse = + serde_json::from_str(&body).expect("should deserialize rotate response"); + + assert!( + !response.success, + "should report failure when supplied kid is invalid" + ); + assert!( + response + .error + .as_deref() + .is_some_and(|error| error.contains("kid must contain only")), + "should explain the kid character restrictions" + ); + } + #[test] fn test_handle_deactivate_key_request() { let settings = crate::test_support::tests::create_test_settings(); @@ -540,23 +720,30 @@ mod tests { let body_json = serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/deactivate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/deactivate"); req.set_body(body_json); - let result = handle_deactivate_key(&settings, req); - match result { - Ok(mut resp) => { - let body = resp.take_body_str(); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - log::debug!( - "Deactivate response: success={}, message={}", - response.success, - response.message - ); - } - Err(e) => log::debug!("Expected error in test environment: {}", e), - } + let mut resp = handle_deactivate_key(&settings, &noop_services(), req) + .expect("should return a response even when stores are unavailable"); + + assert_eq!( + resp.get_status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should return 500 when active-kids cannot be read" + ); + + let body = resp.take_body_str(); + let response: DeactivateKeyResponse = + serde_json::from_str(&body).expect("should deserialize deactivate response"); + + assert!( + !response.success, + "should report failure when store reads fail" + ); + assert!( + response.error.is_some(), + "should include error detail in failure response" + ); } #[test] @@ -570,35 +757,112 @@ mod tests { let body_json = serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/deactivate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/deactivate"); req.set_body(body_json); - let result = handle_deactivate_key(&settings, req); - match result { - Ok(mut resp) => { - let body = resp.take_body_str(); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - log::debug!( - "Delete response: success={}, deleted={}", - response.success, - response.deleted - ); - } - Err(e) => log::debug!("Expected error in test environment: {}", e), - } + let mut resp = handle_deactivate_key(&settings, &noop_services(), req) + .expect("should return a response even when stores are unavailable"); + + assert_eq!( + resp.get_status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should return 500 when active-kids cannot be read" + ); + + let body = resp.take_body_str(); + let response: DeactivateKeyResponse = + serde_json::from_str(&body).expect("should deserialize deactivate response"); + + assert!( + !response.success, + "should report failure when store reads fail" + ); + assert!( + !response.deleted, + "should not report deletion when the operation failed" + ); + assert!( + response.error.is_some(), + "should include error detail in failure response" + ); } #[test] fn test_handle_deactivate_key_invalid_json() { let settings = crate::test_support::tests::create_test_settings(); - let mut req = Request::new(Method::POST, "https://test.com/admin/keys/deactivate"); + let mut req = Request::new(Method::POST, "https://test.com/_ts/admin/keys/deactivate"); req.set_body("invalid json"); - let result = handle_deactivate_key(&settings, req); + let result = handle_deactivate_key(&settings, &noop_services(), req); assert!(result.is_err(), "Invalid JSON should return error"); } + #[test] + fn test_handle_deactivate_key_rejects_invalid_kid() { + let settings = crate::test_support::tests::create_test_settings(); + + let req_body = DeactivateKeyRequest { + kid: "bad kid".to_string(), + delete: false, + }; + + let body_json = + serde_json::to_string(&req_body).expect("should serialize deactivate request"); + let mut req = Request::new(Method::POST, "https://test.com/admin/keys/deactivate"); + req.set_body(body_json); + + let mut resp = handle_deactivate_key(&settings, &noop_services(), req) + .expect("should return a response for invalid kid"); + + assert_eq!( + resp.get_status(), + StatusCode::BAD_REQUEST, + "should reject malformed kid as a bad request" + ); + + let body = resp.take_body_str(); + let response: DeactivateKeyResponse = + serde_json::from_str(&body).expect("should deserialize deactivate response"); + + assert!( + !response.success, + "should report failure when supplied kid is invalid" + ); + assert!( + response + .error + .as_deref() + .is_some_and(|error| error.contains("kid must contain only")), + "should explain the kid character restrictions" + ); + } + + #[test] + fn validate_kid_accepts_allowed_operator_supplied_ids() { + validate_kid("azAZ09-_.:").expect("should accept allowed kid characters"); + } + + #[test] + fn validate_kid_rejects_empty_ids() { + let result = validate_kid(""); + + assert!(result.is_err(), "should reject empty kid values"); + } + + #[test] + fn validate_kid_rejects_overlong_ids() { + let result = validate_kid(&"a".repeat(129)); + + assert!(result.is_err(), "should reject kids longer than 128 chars"); + } + + #[test] + fn validate_kid_rejects_csv_separator() { + let result = validate_kid("kid-a,kid-b"); + + assert!(result.is_err(), "should reject commas in kid values"); + } + #[test] fn test_rotate_key_request_deserialization() { let json = r#"{"kid":"custom-key"}"#; @@ -624,32 +888,14 @@ mod tests { "https://test.com/.well-known/trusted-server.json", ); - let services = noop_services(); - let result = handle_trusted_server_discovery(&settings, &services, req); - match result { - Ok(mut resp) => { - assert_eq!(resp.get_status(), StatusCode::OK); - assert_eq!( - resp.get_content_type(), - Some(fastly::mime::APPLICATION_JSON), - "should return application/json content type" - ); - let body = resp.take_body_str(); - - // Parse the discovery document - let discovery: serde_json::Value = - serde_json::from_str(&body).expect("should parse discovery document"); - - // Verify structure - only version and jwks - assert_eq!(discovery["version"], "1.0"); - assert!(discovery["jwks"].is_object()); - - // Verify no extra fields - assert!(discovery.get("endpoints").is_none()); - assert!(discovery.get("capabilities").is_none()); - } - Err(e) => log::debug!("Expected error in test environment: {}", e), - } + // noop_services() config store always returns Err, so the discovery + // handler propagates the error rather than absorbing it into a 500. + let result = handle_trusted_server_discovery(&settings, &noop_services(), req); + + assert!( + result.is_err(), + "should propagate store errors when JWKS cannot be retrieved" + ); } #[test] diff --git a/crates/trusted-server-core/src/request_signing/jwks.rs b/crates/trusted-server-core/src/request_signing/jwks.rs index 5c4dda94..8d206639 100644 --- a/crates/trusted-server-core/src/request_signing/jwks.rs +++ b/crates/trusted-server-core/src/request_signing/jwks.rs @@ -3,8 +3,6 @@ //! This module provides functionality for generating, storing, and retrieving //! Ed25519 keypairs in JWK format for request signing. -use std::sync::LazyLock; - use ed25519_dalek::{SigningKey, VerifyingKey}; use error_stack::{Report, ResultExt}; use jose_jwk::{ @@ -14,11 +12,8 @@ use jose_jwk::{ use rand::rngs::OsRng; use crate::error::TrustedServerError; -use crate::platform::{RuntimeServices, StoreName}; -use crate::request_signing::JWKS_CONFIG_STORE_NAME; - -static JWKS_STORE_NAME: LazyLock = - LazyLock::new(|| StoreName::from(JWKS_CONFIG_STORE_NAME)); +use crate::platform::RuntimeServices; +use crate::request_signing::{read_active_kids, JWKS_STORE_NAME}; /// An Ed25519 keypair used for request signing. pub struct Keypair { @@ -75,25 +70,12 @@ impl Keypair { /// cannot be read. The underlying [`crate::platform::PlatformError`] is /// preserved as context in the error chain. pub fn get_active_jwks(services: &RuntimeServices) -> Result> { - let active_kids_str = services - .config_store() - .get(&JWKS_STORE_NAME, "active-kids") - .change_context(TrustedServerError::Configuration { - message: "failed to read active-kids from config store".into(), - }) - .attach("while fetching active kids list")?; - - let active_kids: Vec<&str> = active_kids_str - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .collect(); - + let active_kids = read_active_kids(services)?; let mut jwks = Vec::new(); for kid in active_kids { let jwk = services .config_store() - .get(&JWKS_STORE_NAME, kid) + .get(&JWKS_STORE_NAME, &kid) .change_context(TrustedServerError::Configuration { message: format!("failed to get JWK for kid: {}", kid), })?; diff --git a/crates/trusted-server-core/src/request_signing/mod.rs b/crates/trusted-server-core/src/request_signing/mod.rs index 41507940..d5d0e79c 100644 --- a/crates/trusted-server-core/src/request_signing/mod.rs +++ b/crates/trusted-server-core/src/request_signing/mod.rs @@ -5,15 +5,27 @@ //! //! # Store names vs store IDs //! -//! Fastly stores have two identifiers: +//! Platform stores have two identifiers: //! //! - **Store name** ([`JWKS_CONFIG_STORE_NAME`], [`SIGNING_SECRET_STORE_NAME`]): -//! used at the edge for reads via `ConfigStore::open` / `SecretStore::open`. -//! These are configured in `fastly.toml`. +//! used for runtime reads via [`crate::platform::PlatformConfigStore::get`] +//! and [`crate::platform::PlatformSecretStore::get_bytes`] through +//! [`crate::platform::RuntimeServices`]. These names are configured in +//! `fastly.toml` for the Fastly adapter. //! -//! - **Store ID** (`RequestSigning::config_store_id`, `RequestSigning::secret_store_id`): -//! used by the Fastly management API for writes (creating, updating, and -//! deleting items). These are set in `trusted-server.toml`. +//! - **Store ID**: used for write operations via +//! [`crate::platform::PlatformConfigStore::put`] / +//! [`crate::platform::PlatformConfigStore::delete`] and +//! [`crate::platform::PlatformSecretStore::create`] / +//! [`crate::platform::PlatformSecretStore::delete`]. These identifiers come +//! from the request-signing settings in `trusted-server.toml`. + +use std::sync::LazyLock; + +use error_stack::{Report, ResultExt}; + +use crate::error::TrustedServerError; +use crate::platform::{RuntimeServices, StoreName}; pub mod discovery; pub mod endpoints; @@ -21,20 +33,97 @@ pub mod jwks; pub mod rotation; pub mod signing; -/// Config store name for JWKS public keys (edge reads via `ConfigStore::open`). +/// Config store name for JWKS public keys used by runtime read operations. /// /// This must match the store name declared in `fastly.toml` under /// `[local_server.config_stores]`. pub const JWKS_CONFIG_STORE_NAME: &str = "jwks_store"; -/// Secret store name for Ed25519 signing keys (edge reads via `SecretStore::open`). +/// Secret store name for Ed25519 signing keys used by runtime read operations. /// /// This must match the store name declared in `fastly.toml` under /// `[local_server.secret_stores]`. pub const SIGNING_SECRET_STORE_NAME: &str = "signing_keys"; +/// Lazily constructed [`StoreName`] for JWKS config-store reads. +pub(crate) static JWKS_STORE_NAME: LazyLock = + LazyLock::new(|| StoreName::from(JWKS_CONFIG_STORE_NAME)); + +/// Lazily constructed [`StoreName`] for signing-key secret-store reads. +pub(crate) static SIGNING_STORE_NAME: LazyLock = + LazyLock::new(|| StoreName::from(SIGNING_SECRET_STORE_NAME)); + +fn parse_active_kids(active_kids: &str) -> Vec { + active_kids + .split(',') + .map(|kid| kid.trim().to_string()) + .filter(|kid| !kid.is_empty()) + .collect() +} + +fn read_active_kids(services: &RuntimeServices) -> Result, Report> { + services + .config_store() + .get(&JWKS_STORE_NAME, "active-kids") + .change_context(TrustedServerError::Configuration { + message: "failed to read active-kids from config store".into(), + }) + .attach("while fetching active kids list") + .map(|active_kids| parse_active_kids(&active_kids)) +} + pub use discovery::*; pub use endpoints::*; pub use jwks::*; pub use rotation::*; pub use signing::*; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_active_kids_splits_comma_separated_kids() { + let result = parse_active_kids("kid-a,kid-b,kid-c"); + assert_eq!(result, vec!["kid-a", "kid-b", "kid-c"]); + } + + #[test] + fn parse_active_kids_trims_whitespace_around_each_kid() { + let result = parse_active_kids(" kid-a , kid-b "); + assert_eq!(result, vec!["kid-a", "kid-b"]); + } + + #[test] + fn parse_active_kids_skips_empty_segments() { + let result = parse_active_kids("kid-a,,kid-b"); + assert_eq!(result, vec!["kid-a", "kid-b"]); + } + + #[test] + fn parse_active_kids_skips_whitespace_only_segments() { + let result = parse_active_kids(" kid-a , , kid-b "); + assert_eq!(result, vec!["kid-a", "kid-b"]); + } + + #[test] + fn parse_active_kids_returns_empty_for_empty_string() { + let result = parse_active_kids(""); + assert!(result.is_empty(), "should return no kids for empty input"); + } + + #[test] + fn parse_active_kids_returns_empty_for_only_commas() { + let result = parse_active_kids(",,,"); + assert!( + result.is_empty(), + "should return no kids when input is only commas" + ); + } + + #[test] + fn parse_active_kids_handles_single_kid() { + let result = parse_active_kids("only-kid"); + assert_eq!(result, vec!["only-kid"]); + } +} diff --git a/crates/trusted-server-core/src/request_signing/rotation.rs b/crates/trusted-server-core/src/request_signing/rotation.rs index 252ccd59..f78ac882 100644 --- a/crates/trusted-server-core/src/request_signing/rotation.rs +++ b/crates/trusted-server-core/src/request_signing/rotation.rs @@ -1,66 +1,58 @@ //! Key rotation management for request signing. //! -//! This module provides functionality for rotating signing keys, managing key lifecycle, -//! and storing keys in Fastly Config and Secret stores. +//! This module provides functionality for rotating signing keys, managing key +//! lifecycle, and storing keys via platform store primitives through +//! [`RuntimeServices`]. use base64::{engine::general_purpose, Engine}; +use chrono::Utc; use ed25519_dalek::SigningKey; use error_stack::{Report, ResultExt}; use jose_jwk::Jwk; +use uuid::Uuid; +use super::{read_active_kids, Keypair}; use crate::error::TrustedServerError; -use crate::request_signing::JWKS_CONFIG_STORE_NAME; -use crate::storage::{FastlyApiClient, FastlyConfigStore}; - -use super::Keypair; +use crate::platform::{RuntimeServices, StoreId}; +use crate::request_signing::JWKS_STORE_NAME; +/// Result of a key rotation operation. #[derive(Debug, Clone)] pub struct KeyRotationResult { + /// Newly generated or supplied key identifier. pub new_kid: String, + /// Previously active key identifier, if one existed. pub previous_kid: Option, + /// Active key identifiers after rotation completes. pub active_kids: Vec, + /// Public JWK associated with the newly active key. pub jwk: Jwk, } -#[allow(deprecated)] +/// Manages signing key lifecycle using platform store primitives. +/// +/// Reads use the edge-visible store name ([`super::JWKS_CONFIG_STORE_NAME`]). +/// Writes use the management API store identifiers supplied at construction. pub struct KeyRotationManager { - /// Edge-side config store for reading JWKS (uses store name). - config_store: FastlyConfigStore, - /// Management API client for writing to stores (uses store IDs). - api_client: FastlyApiClient, - /// Fastly API store ID for config store writes. - config_store_id: String, - /// Fastly API store ID for secret store writes. - secret_store_id: String, + /// Management API store ID for config store writes. + config_store_id: StoreId, + /// Management API store ID for secret store writes. + secret_store_id: StoreId, } -#[allow(deprecated)] impl KeyRotationManager { /// Creates a new key rotation manager. /// - /// The `config_store_id` and `secret_store_id` are Fastly management API + /// The `config_store_id` and `secret_store_id` are platform management API /// identifiers used for write operations. Edge reads use the store names - /// defined in [`JWKS_CONFIG_STORE_NAME`] and [`crate::request_signing::SIGNING_SECRET_STORE_NAME`]. - /// - /// # Errors - /// - /// Returns an error if the API client cannot be initialized. - pub fn new( - config_store_id: impl Into, - secret_store_id: impl Into, - ) -> Result> { - let config_store_id = config_store_id.into(); - let secret_store_id = secret_store_id.into(); - - let config_store = FastlyConfigStore::new(JWKS_CONFIG_STORE_NAME); - let api_client = FastlyApiClient::new()?; - - Ok(Self { - config_store, - api_client, - config_store_id, - secret_store_id, - }) + /// defined in [`super::JWKS_CONFIG_STORE_NAME`] and + /// [`crate::request_signing::SIGNING_SECRET_STORE_NAME`]. + #[must_use] + pub fn new(config_store_id: &str, secret_store_id: &str) -> Self { + Self { + config_store_id: StoreId::from(config_store_id), + secret_store_id: StoreId::from(secret_store_id), + } } /// Rotates the signing key by generating a new keypair and storing it. @@ -70,24 +62,83 @@ impl KeyRotationManager { /// Returns an error if key storage or update operations fail. pub fn rotate_key( &self, + services: &RuntimeServices, kid: Option, ) -> Result> { - let new_kid = kid.unwrap_or_else(generate_date_based_kid); + let previous_kid = services + .config_store() + .get(&JWKS_STORE_NAME, "current-kid") + .ok(); + let active_kids = read_active_kids(services).unwrap_or_default(); + let new_kid = match kid { + Some(kid) => { + if self.key_exists(services, &kid, &active_kids) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("kid '{}' already exists; choose a unique kid", kid), + })); + } + kid + } + None => self.generate_unique_date_based_kid(services, &active_kids), + }; let keypair = Keypair::generate(); let jwk = keypair.get_jwk(new_kid.clone()); - let previous_kid = self.config_store.get("current-kid").ok(); - self.store_private_key(&new_kid, &keypair.signing_key)?; - self.store_public_jwk(&new_kid, &jwk)?; + // Step 1: write private key. Nothing to roll back on failure. + self.store_private_key(services, &new_kid, &keypair.signing_key)?; + + // Step 2: write public JWK. Roll back the private key on failure so no + // orphaned key material is left in the secret store. + if let Err(err) = self.store_public_jwk(services, &new_kid, &jwk) { + if let Err(rollback_err) = services + .secret_store() + .delete(&self.secret_store_id, &new_kid) + { + log::warn!( + "rotate_key: rollback of private key '{}' failed after JWK write error: {}", + new_kid, + rollback_err + ); + } + return Err(err); + } - let active_kids = match &previous_kid { - Some(prev) if prev != &new_kid => vec![prev.clone(), new_kid.clone()], - _ => vec![new_kid.clone()], - }; + let mut active_kids = active_kids; + if !active_kids.iter().any(|kid| kid == &new_kid) { + active_kids.push(new_kid.clone()); + } - self.update_current_kid(&new_kid)?; - self.update_active_kids(&active_kids)?; + // Step 3: publish the new kid in active-kids BEFORE flipping current-kid. + // Roll back both artifacts on failure so the new kid never appears in JWKS + // without a reachable private key. + if let Err(err) = self.update_active_kids(services, &active_kids) { + if let Err(rollback_err) = services + .config_store() + .delete(&self.config_store_id, &new_kid) + { + log::warn!( + "rotate_key: rollback of JWK '{}' failed after active-kids write error: {}", + new_kid, + rollback_err + ); + } + if let Err(rollback_err) = services + .secret_store() + .delete(&self.secret_store_id, &new_kid) + { + log::warn!( + "rotate_key: rollback of private key '{}' failed after active-kids write error: {}", + new_kid, + rollback_err + ); + } + return Err(err); + } + + // Step 4: flip current-kid last. A failure here leaves the old kid still + // active and the new kid visible in JWKS but unused — a recoverable state. + self.update_current_kid(services, &new_kid)?; Ok(KeyRotationResult { new_kid, @@ -97,50 +148,88 @@ impl KeyRotationManager { }) } + fn key_exists(&self, services: &RuntimeServices, kid: &str, active_kids: &[String]) -> bool { + active_kids.iter().any(|active_kid| active_kid == kid) + || services.config_store().get(&JWKS_STORE_NAME, kid).is_ok() + } + + fn generate_unique_date_based_kid( + &self, + services: &RuntimeServices, + active_kids: &[String], + ) -> String { + let base_kid = generate_date_based_kid(); + if !self.key_exists(services, &base_kid, active_kids) { + return base_kid; + } + + format!("{base_kid}-{}", Uuid::new_v4().simple()) + } + fn store_private_key( &self, + services: &RuntimeServices, kid: &str, signing_key: &SigningKey, ) -> Result<(), Report> { - let key_bytes = signing_key.as_bytes(); - let key_b64 = general_purpose::STANDARD.encode(key_bytes); - - self.api_client - .create_secret(&self.secret_store_id, kid, &key_b64) + // The platform secret-store write interface is string-based, so signing + // keys are persisted as base64 text. The Fastly adapter applies its own + // transport-level base64 encoding when calling the management API. + let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); + + services + .secret_store() + .create(&self.secret_store_id, kid, &key_b64) .change_context(TrustedServerError::Configuration { - message: format!("Failed to store private key '{}'", kid), + message: format!("failed to store private key '{}'", kid), }) } - fn store_public_jwk(&self, kid: &str, jwk: &Jwk) -> Result<(), Report> { + fn store_public_jwk( + &self, + services: &RuntimeServices, + kid: &str, + jwk: &Jwk, + ) -> Result<(), Report> { let jwk_json = serde_json::to_string(jwk).map_err(|e| { Report::new(TrustedServerError::Configuration { - message: format!("Failed to serialize JWK: {}", e), + message: format!("failed to serialize JWK: {}", e), }) })?; - self.api_client - .update_config_item(&self.config_store_id, kid, &jwk_json) + services + .config_store() + .put(&self.config_store_id, kid, &jwk_json) .change_context(TrustedServerError::Configuration { - message: format!("Failed to store public JWK '{}'", kid), + message: format!("failed to store public JWK '{}'", kid), }) } - fn update_current_kid(&self, kid: &str) -> Result<(), Report> { - self.api_client - .update_config_item(&self.config_store_id, "current-kid", kid) + fn update_current_kid( + &self, + services: &RuntimeServices, + kid: &str, + ) -> Result<(), Report> { + services + .config_store() + .put(&self.config_store_id, "current-kid", kid) .change_context(TrustedServerError::Configuration { - message: "Failed to update current-kid".into(), + message: "failed to update current-kid".into(), }) } - fn update_active_kids(&self, active_kids: &[String]) -> Result<(), Report> { + fn update_active_kids( + &self, + services: &RuntimeServices, + active_kids: &[String], + ) -> Result<(), Report> { let active_kids_str = active_kids.join(","); - self.api_client - .update_config_item(&self.config_store_id, "active-kids", &active_kids_str) + services + .config_store() + .put(&self.config_store_id, "active-kids", &active_kids_str) .change_context(TrustedServerError::Configuration { - message: "Failed to update active-kids".into(), + message: "failed to update active-kids".into(), }) } @@ -149,16 +238,11 @@ impl KeyRotationManager { /// # Errors /// /// Returns an error if the active keys cannot be retrieved from the config store. - pub fn list_active_keys(&self) -> Result, Report> { - let active_kids_str = self.config_store.get("active-kids")?; - - let active_kids: Vec = active_kids_str - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - - Ok(active_kids) + pub fn list_active_keys( + &self, + services: &RuntimeServices, + ) -> Result, Report> { + read_active_kids(services) } /// Deactivates a key by removing it from the active keys list. @@ -166,18 +250,23 @@ impl KeyRotationManager { /// # Errors /// /// Returns an error if this would deactivate the last active key, or if the update fails. - pub fn deactivate_key(&self, kid: &str) -> Result<(), Report> { - let mut active_kids = self.list_active_keys()?; + pub fn deactivate_key( + &self, + services: &RuntimeServices, + kid: &str, + ) -> Result<(), Report> { + self.ensure_not_current_key(services, kid, "deactivate")?; + let mut active_kids = self.list_active_keys(services)?; active_kids.retain(|k| k != kid); if active_kids.is_empty() { return Err(Report::new(TrustedServerError::Configuration { - message: "Cannot deactivate the last active key".into(), + message: "cannot deactivate the last active key".into(), })); } - self.update_active_kids(&active_kids) + self.update_active_kids(services, &active_kids) } /// Deletes a key by deactivating it and removing it from storage. @@ -185,79 +274,466 @@ impl KeyRotationManager { /// # Errors /// /// Returns an error if deactivation fails or if the key cannot be deleted from storage. - pub fn delete_key(&self, kid: &str) -> Result<(), Report> { - self.deactivate_key(kid)?; - - self.api_client - .delete_config_item(&self.config_store_id, kid) + pub fn delete_key( + &self, + services: &RuntimeServices, + kid: &str, + ) -> Result<(), Report> { + self.ensure_not_current_key(services, kid, "delete")?; + self.deactivate_key(services, kid)?; + + // Delete the private key first. A failure here leaves the JWK in the + // config store but no private key — the key is verifiable but cannot + // sign, which is safer than orphaned key material with no JWK. Both + // deletes treat 404 as success so retries converge after partial failures. + services + .secret_store() + .delete(&self.secret_store_id, kid) .change_context(TrustedServerError::Configuration { - message: "Failed to delete JWK from ConfigStore".into(), + message: "failed to delete signing key from secret store".into(), })?; - self.api_client - .delete_secret(&self.secret_store_id, kid) + services + .config_store() + .delete(&self.config_store_id, kid) .change_context(TrustedServerError::Configuration { - message: "Failed to delete secret from SecretStore".into(), + message: "failed to delete JWK from config store".into(), })?; Ok(()) } + + fn ensure_not_current_key( + &self, + services: &RuntimeServices, + kid: &str, + operation: &str, + ) -> Result<(), Report> { + if services + .config_store() + .get(&JWKS_STORE_NAME, "current-kid") + .is_ok_and(|current| current == kid) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "cannot {operation} '{kid}' because it is the current signing key; rotate first" + ), + })); + } + + Ok(()) + } } +/// Generates a date-based key ID in the format `ts-YYYY-MM-DD`. #[must_use] pub fn generate_date_based_kid() -> String { - use chrono::Utc; format!("ts-{}", Utc::now().format("%Y-%m-%d")) } #[cfg(test)] mod tests { + use std::collections::HashMap; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + + use error_stack::Report; + + use crate::platform::test_support::build_services_with_config_and_secret; + use crate::platform::{ + PlatformConfigStore, PlatformError, PlatformSecretStore, StoreId, StoreName, + }; use crate::request_signing::Keypair; use super::*; - #[test] - fn test_generate_date_based_kid() { - let kid = generate_date_based_kid(); - // Verify format: ts-YYYY-MM-DD - assert!(kid.starts_with("ts-")); - assert!(kid.len() >= 13); - let parts: Vec<&str> = kid.split('-').collect(); - assert_eq!(parts.len(), 4); - assert_eq!(parts[0], "ts"); + // --------------------------------------------------------------------------- + // Spy stores: record put/create/delete calls, serve preset get values + // --------------------------------------------------------------------------- + + #[derive(Clone)] + struct SpyConfigStore { + inner: Arc, } - #[test] - fn test_key_rotation_manager_creation() { - let result = KeyRotationManager::new("test-config-store-id", "test-secret-store-id"); - match result { - Ok(manager) => { - assert_eq!(manager.config_store_id, "test-config-store-id"); - assert_eq!(manager.secret_store_id, "test-secret-store-id"); + struct SpyConfigStoreInner { + data: Mutex>, + puts: Mutex>, + deletes: Mutex>, + /// Fail `put` after this many successful calls. `usize::MAX` means never fail. + fail_after_n_puts: AtomicUsize, + } + + impl SpyConfigStore { + fn new(initial: HashMap) -> Self { + Self { + inner: Arc::new(SpyConfigStoreInner { + data: Mutex::new(initial), + puts: Mutex::new(vec![]), + deletes: Mutex::new(vec![]), + fail_after_n_puts: AtomicUsize::new(usize::MAX), + }), } - Err(e) => { - println!("Expected error in test environment: {}", e); + } + + /// Returns a store whose `put` succeeds for the first `n` calls, then + /// returns an error. Use `n = 0` to fail immediately. + fn with_put_failure_after(n: usize) -> Self { + Self { + inner: Arc::new(SpyConfigStoreInner { + data: Mutex::new(HashMap::new()), + puts: Mutex::new(vec![]), + deletes: Mutex::new(vec![]), + fail_after_n_puts: AtomicUsize::new(n), + }), } } + + fn puts(&self) -> Vec<(String, String, String)> { + self.inner.puts.lock().expect("should lock puts").clone() + } + + fn deletes(&self) -> Vec<(String, String)> { + self.inner + .deletes + .lock() + .expect("should lock deletes") + .clone() + } } - #[test] - fn test_list_active_keys() { - let result = KeyRotationManager::new("test-config-store-id", "test-secret-store-id"); - if let Ok(manager) = result { - match manager.list_active_keys() { - Ok(keys) => { - assert!(!keys.is_empty(), "Should have at least one active key"); - } - Err(e) => println!("Expected error in test environment: {}", e), + impl PlatformConfigStore for SpyConfigStore { + fn get(&self, _: &StoreName, key: &str) -> Result> { + self.inner + .data + .lock() + .expect("should lock data") + .get(key) + .cloned() + .ok_or_else(|| Report::new(PlatformError::ConfigStore)) + } + + fn put( + &self, + store_id: &StoreId, + key: &str, + value: &str, + ) -> Result<(), Report> { + let remaining = self.inner.fail_after_n_puts.load(Ordering::SeqCst); + if remaining == 0 { + return Err(Report::new(PlatformError::ConfigStore)); + } + if remaining != usize::MAX { + self.inner.fail_after_n_puts.fetch_sub(1, Ordering::SeqCst); } + self.inner.puts.lock().expect("should lock puts").push(( + store_id.to_string(), + key.to_string(), + value.to_string(), + )); + self.inner + .data + .lock() + .expect("should lock data") + .insert(key.to_string(), value.to_string()); + Ok(()) } + + fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + self.inner + .deletes + .lock() + .expect("should lock deletes") + .push((store_id.to_string(), key.to_string())); + self.inner + .data + .lock() + .expect("should lock data") + .remove(key); + Ok(()) + } + } + + #[derive(Clone)] + struct SpySecretStore { + inner: Arc, + } + + struct SpySecretStoreInner { + creates: Mutex>, + deletes: Mutex>, + /// Fail `create` after this many successful calls. `usize::MAX` means never fail. + fail_after_n_creates: AtomicUsize, } + impl SpySecretStore { + fn new() -> Self { + Self { + inner: Arc::new(SpySecretStoreInner { + creates: Mutex::new(vec![]), + deletes: Mutex::new(vec![]), + fail_after_n_creates: AtomicUsize::new(usize::MAX), + }), + } + } + + /// Returns a store whose `create` succeeds for the first `n` calls, then + /// returns an error. Use `n = 0` to fail immediately. + fn with_create_failure_after(n: usize) -> Self { + Self { + inner: Arc::new(SpySecretStoreInner { + creates: Mutex::new(vec![]), + deletes: Mutex::new(vec![]), + fail_after_n_creates: AtomicUsize::new(n), + }), + } + } + + fn creates(&self) -> Vec<(String, String, String)> { + self.inner + .creates + .lock() + .expect("should lock creates") + .clone() + } + + fn deletes(&self) -> Vec<(String, String)> { + self.inner + .deletes + .lock() + .expect("should lock deletes") + .clone() + } + } + + impl PlatformSecretStore for SpySecretStore { + fn get_bytes(&self, _: &StoreName, _: &str) -> Result, Report> { + Err(Report::new(PlatformError::SecretStore)) + } + + fn create( + &self, + store_id: &StoreId, + name: &str, + value: &str, + ) -> Result<(), Report> { + let remaining = self.inner.fail_after_n_creates.load(Ordering::SeqCst); + if remaining == 0 { + return Err(Report::new(PlatformError::SecretStore)); + } + if remaining != usize::MAX { + self.inner + .fail_after_n_creates + .fetch_sub(1, Ordering::SeqCst); + } + self.inner + .creates + .lock() + .expect("should lock creates") + .push((store_id.to_string(), name.to_string(), value.to_string())); + Ok(()) + } + + fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { + self.inner + .deletes + .lock() + .expect("should lock deletes") + .push((store_id.to_string(), name.to_string())); + Ok(()) + } + } + + // --------------------------------------------------------------------------- + // Tests + // --------------------------------------------------------------------------- + #[test] - fn test_key_rotation_result_structure() { - let jwk = Keypair::generate().get_jwk("test-key".to_string()); + fn generate_date_based_kid_has_correct_format() { + let kid = generate_date_based_kid(); + assert!(kid.starts_with("ts-"), "should start with 'ts-'"); + assert!(kid.len() >= 13, "should be at least 13 characters"); + let parts: Vec<&str> = kid.split('-').collect(); + assert_eq!(parts.len(), 4, "should have 4 dash-separated parts"); + assert_eq!(parts[0], "ts", "first part should be 'ts'"); + } + + #[test] + fn new_is_infallible_and_stores_ids() { + let manager = KeyRotationManager::new("cfg-store-123", "sec-store-456"); + assert_eq!( + manager.config_store_id.as_ref(), + "cfg-store-123", + "should store config_store_id" + ); + assert_eq!( + manager.secret_store_id.as_ref(), + "sec-store-456", + "should store secret_store_id" + ); + } + + #[test] + fn rotate_key_stores_private_key_via_secret_store_create() { + let config_store = SpyConfigStore::new(HashMap::new()); + let secret_store = SpySecretStore::new(); + let services = build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.rotate_key(&services, Some("new-kid".to_string())); + + assert!(result.is_ok(), "should succeed when stores accept writes"); + let rotation = result.expect("should produce rotation result"); + assert_eq!(rotation.new_kid, "new-kid", "should use the provided kid"); + assert!( + rotation.active_kids.contains(&"new-kid".to_string()), + "should include new kid in active kids" + ); + } + #[test] + fn rotate_key_preserves_existing_active_kids() { + let mut data = HashMap::new(); + data.insert("current-kid".to_string(), "kid-b".to_string()); + data.insert("active-kids".to_string(), "kid-a, kid-b".to_string()); + + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let rotation = manager + .rotate_key(&services, Some("kid-c".to_string())) + .expect("should rotate key successfully"); + + assert_eq!( + rotation.active_kids, + vec![ + "kid-a".to_string(), + "kid-b".to_string(), + "kid-c".to_string() + ], + "should preserve previously active keys and append the new kid" + ); + + let active_kids = manager + .list_active_keys(&services) + .expect("should read back updated active kids"); + assert_eq!( + active_kids, + vec![ + "kid-a".to_string(), + "kid-b".to_string(), + "kid-c".to_string() + ], + "should store the full active kid list after rotation" + ); + } + + #[test] + fn rotate_key_does_not_reactivate_deactivated_previous_kid() { + let mut data = HashMap::new(); + data.insert("current-kid".to_string(), "kid-a".to_string()); + data.insert("active-kids".to_string(), "kid-b".to_string()); + + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let rotation = manager + .rotate_key(&services, Some("kid-c".to_string())) + .expect("should rotate key successfully"); + + assert_eq!( + rotation.active_kids, + vec!["kid-b".to_string(), "kid-c".to_string()], + "should not resurrect a previous kid that is no longer active" + ); + } + + #[test] + fn rotate_key_rejects_explicit_kid_that_is_already_active() { + let mut data = HashMap::new(); + data.insert("current-kid".to_string(), "kid-b".to_string()); + data.insert("active-kids".to_string(), "kid-a,kid-b".to_string()); + + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = + build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.rotate_key(&services, Some("kid-a".to_string())); + + assert!( + result.is_err(), + "should reject explicit rotation to an existing kid" + ); + assert!( + secret_store.creates().is_empty(), + "should reject duplicate kids before writing private key material" + ); + assert!( + config_store.puts().is_empty(), + "should reject duplicate kids before writing config store entries" + ); + } + + #[test] + fn rotate_key_uniquifies_generated_kid_when_date_based_kid_is_active() { + let base_kid = generate_date_based_kid(); + let mut data = HashMap::new(); + data.insert("current-kid".to_string(), base_kid.clone()); + data.insert("active-kids".to_string(), base_kid.clone()); + + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let rotation = manager + .rotate_key(&services, None) + .expect("should rotate with a uniquified generated kid"); + + assert_ne!( + rotation.new_kid, base_kid, + "should not reuse an active date-based kid" + ); + assert!( + rotation.new_kid.starts_with(&format!("{base_kid}-")), + "should preserve the date-based kid prefix for generated collisions" + ); + assert!( + rotation.active_kids.contains(&base_kid), + "should keep the existing kid active" + ); + assert!( + rotation.active_kids.contains(&rotation.new_kid), + "should add the uniquified generated kid" + ); + } + + #[test] + fn deactivate_key_fails_when_only_one_key_remains() { + let mut data = HashMap::new(); + data.insert("active-kids".to_string(), "only-key".to_string()); + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.deactivate_key(&services, "only-key"); + + assert!( + result.is_err(), + "should fail to deactivate the last active key" + ); + } + + #[test] + fn key_rotation_result_structure_is_valid() { + let jwk = Keypair::generate().get_jwk("test-key".to_string()); let result = KeyRotationResult { new_kid: "ts-2024-01-01".to_string(), previous_kid: Some("ts-2023-12-31".to_string()), @@ -270,4 +746,142 @@ mod tests { assert_eq!(result.active_kids.len(), 2); assert_eq!(result.jwk.prm.kid, Some("test-key".to_string())); } + + #[test] + fn rotate_key_fails_when_private_key_store_write_fails() { + let config_store = SpyConfigStore::new(HashMap::new()); + let secret_store = SpySecretStore::with_create_failure_after(0); + let services = build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.rotate_key(&services, Some("new-kid".to_string())); + + assert!( + result.is_err(), + "should fail when the secret store rejects the private key write" + ); + } + + #[test] + fn rotate_key_rolls_back_secret_when_jwk_write_fails() { + let config_store = SpyConfigStore::with_put_failure_after(0); + let secret_store = SpySecretStore::new(); + let services = + build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.rotate_key(&services, Some("rollback-kid".to_string())); + + assert!(result.is_err(), "should fail when JWK write fails"); + assert_eq!( + secret_store.deletes(), + vec![("sec-id".to_string(), "rollback-kid".to_string())], + "should roll back private key material after JWK write failure" + ); + assert!( + config_store.deletes().is_empty(), + "should not roll back a JWK that was never stored" + ); + } + + #[test] + fn rotate_key_rolls_back_secret_and_jwk_when_active_kids_write_fails() { + let config_store = SpyConfigStore::with_put_failure_after(1); + let secret_store = SpySecretStore::new(); + let services = + build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.rotate_key(&services, Some("rollback-kid".to_string())); + + assert!(result.is_err(), "should fail when active-kids write fails"); + assert_eq!( + config_store.deletes(), + vec![("cfg-id".to_string(), "rollback-kid".to_string())], + "should roll back the stored JWK after active-kids write failure" + ); + assert_eq!( + secret_store.deletes(), + vec![("sec-id".to_string(), "rollback-kid".to_string())], + "should roll back private key material after active-kids write failure" + ); + } + + #[test] + fn deactivate_key_rejects_current_kid() { + let mut data = HashMap::new(); + data.insert("current-kid".to_string(), "kid-a".to_string()); + data.insert("active-kids".to_string(), "kid-a,kid-b".to_string()); + + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = + build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.deactivate_key(&services, "kid-a"); + + assert!(result.is_err(), "should reject deactivating current-kid"); + assert!( + config_store.puts().is_empty(), + "should reject current-kid deactivation before updating active-kids" + ); + assert!( + secret_store.deletes().is_empty(), + "should not touch secret store during failed deactivation" + ); + } + + #[test] + fn delete_key_rejects_current_kid_before_deleting_storage() { + let mut data = HashMap::new(); + data.insert("current-kid".to_string(), "kid-a".to_string()); + data.insert("active-kids".to_string(), "kid-a,kid-b".to_string()); + + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = + build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.delete_key(&services, "kid-a"); + + assert!(result.is_err(), "should reject deleting current-kid"); + assert!( + secret_store.deletes().is_empty(), + "should reject current-kid deletion before deleting private key material" + ); + assert!( + config_store.deletes().is_empty(), + "should reject current-kid deletion before deleting JWK storage" + ); + } + + #[test] + fn delete_key_removes_secret_before_jwk() { + let mut data = HashMap::new(); + data.insert("active-kids".to_string(), "kid-a, kid-b".to_string()); + data.insert( + "kid-a".to_string(), + r#"{"kty":"OKP","crv":"Ed25519"}"#.to_string(), + ); + + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + manager + .delete_key(&services, "kid-a") + .expect("should delete key successfully"); + + // After deletion, the JWK entry should be gone from the config store. + let jwk_gone = services + .config_store() + .get(&crate::request_signing::JWKS_STORE_NAME, "kid-a"); + assert!( + jwk_gone.is_err(), + "should remove JWK from the config store after deletion" + ); + } } diff --git a/crates/trusted-server-core/src/request_signing/signing.rs b/crates/trusted-server-core/src/request_signing/signing.rs index a30226bb..176f75e1 100644 --- a/crates/trusted-server-core/src/request_signing/signing.rs +++ b/crates/trusted-server-core/src/request_signing/signing.rs @@ -1,7 +1,7 @@ //! Request signing and verification utilities. //! //! This module provides Ed25519-based signing and verification of HTTP requests -//! using keys stored in Fastly Config and Secret stores. +//! using keys stored via platform store primitives. use base64::{engine::general_purpose, Engine}; use ed25519_dalek::{Signature, Signer as Ed25519Signer, SigningKey, Verifier, VerifyingKey}; @@ -9,42 +9,52 @@ use error_stack::{Report, ResultExt}; use serde::Serialize; use crate::error::TrustedServerError; -use crate::request_signing::{JWKS_CONFIG_STORE_NAME, SIGNING_SECRET_STORE_NAME}; -use crate::storage::{FastlyConfigStore, FastlySecretStore}; +use crate::platform::RuntimeServices; +use crate::request_signing::{JWKS_STORE_NAME, SIGNING_STORE_NAME}; /// Retrieves the current active key ID from the config store. /// /// # Errors /// /// Returns an error if the config store cannot be accessed or the current-kid key is not found. -#[allow(deprecated)] -pub fn get_current_key_id() -> Result> { - let store = FastlyConfigStore::new(JWKS_CONFIG_STORE_NAME); - store.get("current-kid") +pub fn get_current_key_id( + services: &RuntimeServices, +) -> Result> { + services + .config_store() + .get(&JWKS_STORE_NAME, "current-kid") + .change_context(TrustedServerError::Configuration { + message: "failed to read current-kid from config store".into(), + }) } -fn parse_ed25519_signing_key(key_bytes: Vec) -> Result> { - let bytes = if key_bytes.len() > 32 { - general_purpose::STANDARD.decode(&key_bytes).map_err(|_| { - Report::new(TrustedServerError::Configuration { - message: "Failed to decode base64 key".into(), - }) - })? - } else { - key_bytes - }; +/// Parses an Ed25519 signing key from secret-store bytes. +/// +/// Request-signing rotation always stores private keys as standard base64 text +/// via [`crate::request_signing::rotation::KeyRotationManager`]. A non-base64 +/// value in the secret store indicates data corruption and is surfaced as an +/// explicit error rather than silently falling back to a length heuristic. +fn parse_ed25519_signing_key(key_bytes: &[u8]) -> Result> { + let bytes = general_purpose::STANDARD.decode(key_bytes).map_err(|_| { + Report::new(TrustedServerError::Configuration { + message: "signing key is not valid base64 — corrupt key material in secret store" + .into(), + }) + })?; let key_array: [u8; 32] = bytes.try_into().map_err(|_| { Report::new(TrustedServerError::Configuration { - message: "Invalid key length (expected 32 bytes for Ed25519)".into(), + message: "signing key must be 32 bytes after base64 decoding".into(), }) })?; Ok(SigningKey::from_bytes(&key_array)) } +/// Signs request payloads using the current Ed25519 private key. pub struct RequestSigner { key: SigningKey, + /// Key identifier associated with the loaded private key. pub kid: String, } @@ -68,9 +78,13 @@ struct SigningPayload<'a> { /// Parameters for enhanced request signing #[derive(Debug, Clone)] pub struct SigningParams { + /// Request identifier to bind into the signature payload. pub request_id: String, + /// Host header value expected by the receiving service. pub request_host: String, + /// Request scheme bound into the signature payload. pub request_scheme: String, + /// Signature timestamp in Unix milliseconds. pub timestamp: u64, } @@ -91,8 +105,8 @@ impl SigningParams { /// Builds the canonical payload string for signing. /// - /// The payload is a JSON-serialized [`SigningPayload`] to prevent signature - /// confusion attacks that could exploit delimiter-based formats. + /// The payload is JSON-serialized to prevent signature confusion attacks + /// that could exploit delimiter-based formats. /// /// # Errors /// @@ -115,26 +129,25 @@ impl SigningParams { } impl RequestSigner { - /// Creates a `RequestSigner` from the current key ID stored in config. + /// Creates a `RequestSigner` from the current key ID stored in platform stores. /// /// # Errors /// /// Returns an error if the key ID cannot be retrieved or the key cannot be parsed. - #[allow(deprecated)] - pub fn from_config() -> Result> { - let config_store = FastlyConfigStore::new(JWKS_CONFIG_STORE_NAME); + pub fn from_services(services: &RuntimeServices) -> Result> { let key_id = - config_store - .get("current-kid") - .change_context(TrustedServerError::Configuration { - message: "Failed to get current-kid".into(), - })?; - - let secret_store = FastlySecretStore::new(SIGNING_SECRET_STORE_NAME); - let key_bytes = secret_store - .get(&key_id) - .attach(format!("Failed to get signing key for kid: {}", key_id))?; - let signing_key = parse_ed25519_signing_key(key_bytes)?; + get_current_key_id(services).change_context(TrustedServerError::Configuration { + message: "failed to get current-kid".into(), + })?; + + let key_bytes = services + .secret_store() + .get_bytes(&SIGNING_STORE_NAME, &key_id) + .change_context(TrustedServerError::Configuration { + message: format!("failed to get signing key for kid: {}", key_id), + })?; + + let signing_key = parse_ed25519_signing_key(&key_bytes)?; Ok(Self { key: signing_key, @@ -175,17 +188,17 @@ impl RequestSigner { /// # Errors /// /// Returns an error if the JWK cannot be retrieved, parsed, or if signature verification fails. -#[allow(deprecated)] pub fn verify_signature( payload: &[u8], signature_b64: &str, kid: &str, + services: &RuntimeServices, ) -> Result> { - let store = FastlyConfigStore::new(JWKS_CONFIG_STORE_NAME); - let jwk_json = store - .get(kid) + let jwk_json = services + .config_store() + .get(&JWKS_STORE_NAME, kid) .change_context(TrustedServerError::Configuration { - message: format!("Failed to get JWK for kid: {}", kid), + message: format!("failed to get JWK for kid: {}", kid), })?; let jwk: serde_json::Value = serde_json::from_str(&jwk_json).map_err(|e| { @@ -242,88 +255,88 @@ pub fn verify_signature( #[cfg(test)] mod tests { + use crate::platform::test_support::build_request_signing_services; + use super::*; #[test] - fn test_request_signer_sign() { - // Report unwraps print full error chain on test failure - // Note: unwrapping a Report prints it nicely if test fails. - let signer = RequestSigner::from_config().expect("should create signer from config"); + fn from_services_loads_kid_from_config_store() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); + + assert_eq!(signer.kid, "test-kid", "should load kid from config store"); + } + + #[test] + fn sign_produces_non_empty_url_safe_base64_signature() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); + let signature = signer .sign(b"these pretzels are making me thirsty") .expect("should sign payload"); - assert!(!signature.is_empty()); - assert!(signature.len() > 32); - } - #[test] - fn test_request_signer_from_config() { - let signer = RequestSigner::from_config().expect("should create signer from config"); - assert!(!signer.kid.is_empty()); + assert!(!signature.is_empty(), "should produce non-empty signature"); + assert!( + signature.len() > 32, + "should produce a full-length signature" + ); } #[test] - fn test_sign_and_verify() { + fn sign_and_verify_roundtrip_succeeds() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); let payload = b"test payload for verification"; - let signer = RequestSigner::from_config().expect("should create signer from config"); + let signature = signer.sign(payload).expect("should sign payload"); + let verified = verify_signature(payload, &signature, &signer.kid, &services) + .expect("should attempt verification"); - let result = - verify_signature(payload, &signature, &signer.kid).expect("should verify signature"); - assert!(result, "Signature should be valid"); + assert!(verified, "should verify a valid signature"); } #[test] - fn test_verify_invalid_signature() { - let payload = b"test payload"; - let signer = RequestSigner::from_config().expect("should create signer from config"); - - let wrong_signature = signer - .sign(b"different payload") - .expect("should sign different payload"); + fn verify_returns_false_for_wrong_payload() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); + let signature = signer.sign(b"original").expect("should sign"); - let result = verify_signature(payload, &wrong_signature, &signer.kid) + let verified = verify_signature(b"wrong payload", &signature, &signer.kid, &services) .expect("should attempt verification"); - assert!(!result, "Invalid signature should not verify"); - } - - #[test] - fn test_verify_wrong_payload() { - let original_payload = b"original payload"; - let signer = RequestSigner::from_config().expect("should create signer from config"); - let signature = signer - .sign(original_payload) - .expect("should sign original payload"); - let wrong_payload = b"wrong payload"; - let result = verify_signature(wrong_payload, &signature, &signer.kid) - .expect("should attempt verification"); - assert!(!result, "Signature should not verify with wrong payload"); + assert!(!verified, "should not verify signature for wrong payload"); } #[test] - fn test_verify_missing_key() { - let payload = b"test payload"; - let signer = RequestSigner::from_config().expect("should create signer from config"); - let signature = signer.sign(payload).expect("should sign payload"); - let nonexistent_kid = "nonexistent-key-id"; + fn verify_errors_for_unknown_kid() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); + let signature = signer.sign(b"payload").expect("should sign"); + + let result = verify_signature(b"payload", &signature, "nonexistent-kid", &services); - let result = verify_signature(payload, &signature, nonexistent_kid); - assert!(result.is_err(), "Should error for missing key"); + assert!(result.is_err(), "should error for unknown kid"); } #[test] - fn test_verify_malformed_signature() { - let payload = b"test payload"; - let signer = RequestSigner::from_config().expect("should create signer from config"); - let malformed_signature = "not-valid-base64!!!"; + fn verify_errors_for_malformed_signature() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); - let result = verify_signature(payload, malformed_signature, &signer.kid); - assert!(result.is_err(), "Should error for malformed signature"); + let result = verify_signature(b"payload", "not-valid-base64!!!", &signer.kid, &services); + + assert!(result.is_err(), "should error for malformed signature"); } #[test] - fn test_signing_params_build_payload() { + fn signing_params_build_payload_serializes_all_fields() { let params = SigningParams { request_id: "req-123".to_string(), request_host: "example.com".to_string(), @@ -336,6 +349,7 @@ mod tests { .expect("should build payload"); let parsed: serde_json::Value = serde_json::from_str(&payload).expect("should be valid JSON"); + assert_eq!(parsed["version"], SIGNING_VERSION); assert_eq!(parsed["kid"], "kid-abc"); assert_eq!(parsed["host"], "example.com"); @@ -345,46 +359,55 @@ mod tests { } #[test] - fn test_signing_params_new_creates_timestamp() { + fn signing_params_new_creates_recent_timestamp() { let params = SigningParams::new( "req-123".to_string(), "example.com".to_string(), "https".to_string(), ); - assert_eq!(params.request_id, "req-123"); - assert_eq!(params.request_host, "example.com"); - assert_eq!(params.request_scheme, "https"); - // Timestamp should be recent (within last minute), in milliseconds let now_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .unwrap() + .expect("should get system time") .as_millis() as u64; - assert!(params.timestamp <= now_ms); - assert!(params.timestamp >= now_ms - 60_000); + + assert!( + params.timestamp <= now_ms, + "timestamp should not be in the future" + ); + assert!( + params.timestamp >= now_ms - 60_000, + "timestamp should be within the last minute" + ); } #[test] - fn test_sign_request_enhanced() { - let signer = RequestSigner::from_config().unwrap(); + fn sign_request_enhanced_produces_verifiable_signature() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); let params = SigningParams::new( "auction-123".to_string(), "publisher.com".to_string(), "https".to_string(), ); - let signature = signer.sign_request(¶ms).unwrap(); - assert!(!signature.is_empty()); + let signature = signer.sign_request(¶ms).expect("should sign request"); + let payload = params + .build_payload(&signer.kid) + .expect("should build payload"); - // Verify the signature is valid by reconstructing the payload - let payload = params.build_payload(&signer.kid).unwrap(); - let result = verify_signature(payload.as_bytes(), &signature, &signer.kid).unwrap(); - assert!(result, "Enhanced signature should be valid"); + let verified = verify_signature(payload.as_bytes(), &signature, &signer.kid, &services) + .expect("should verify"); + + assert!(verified, "enhanced request signature should be verifiable"); } #[test] - fn test_sign_request_different_params_different_signature() { - let signer = RequestSigner::from_config().unwrap(); + fn sign_request_different_hosts_produce_different_signatures() { + let services = build_request_signing_services(); + let signer = + RequestSigner::from_services(&services).expect("should create signer from services"); let params1 = SigningParams { request_id: "req-1".to_string(), @@ -392,20 +415,19 @@ mod tests { request_scheme: "https".to_string(), timestamp: 1706900000, }; - let params2 = SigningParams { request_id: "req-1".to_string(), - request_host: "host2.com".to_string(), // Different host + request_host: "host2.com".to_string(), request_scheme: "https".to_string(), timestamp: 1706900000, }; - let sig1 = signer.sign_request(¶ms1).unwrap(); - let sig2 = signer.sign_request(¶ms2).unwrap(); + let sig1 = signer.sign_request(¶ms1).expect("should sign params1"); + let sig2 = signer.sign_request(¶ms2).expect("should sign params2"); assert_ne!( sig1, sig2, - "Different hosts should produce different signatures" + "different hosts should produce different signatures" ); } } diff --git a/crates/trusted-server-core/src/rsc_flight.rs b/crates/trusted-server-core/src/rsc_flight.rs index 6bd17366..309e9505 100644 --- a/crates/trusted-server-core/src/rsc_flight.rs +++ b/crates/trusted-server-core/src/rsc_flight.rs @@ -1,7 +1,3 @@ -//! RSC flight data processor. -//! -//! See [`crate::platform`] module doc for platform notes. - use std::io; use crate::host_rewrite::rewrite_bare_host_at_boundaries; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 78549262..a75eb2c8 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -5,6 +5,7 @@ use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize}; use serde_json::Value as JsonValue; use std::collections::HashMap; use std::ops::{Deref, DerefMut}; +use std::str::FromStr; use std::sync::OnceLock; use url::Url; use validator::{Validate, ValidationError}; @@ -19,7 +20,10 @@ pub const ENVIRONMENT_VARIABLE_SEPARATOR: &str = "__"; #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] pub struct Publisher { + #[validate(custom(function = validate_publisher_domain))] pub domain: String, + /// Domain for non-EC cookies. EC cookies use a separate computed domain + /// (see [`ec_cookie_domain`](Self::ec_cookie_domain)). #[validate(custom(function = validate_cookie_domain))] pub cookie_domain: String, #[validate(custom(function = validate_no_trailing_slash))] @@ -34,6 +38,17 @@ impl Publisher { /// Known placeholder values that must not be used in production. pub const PROXY_SECRET_PLACEHOLDERS: &[&str] = &["change-me-proxy-secret", "proxy-secret"]; + /// Returns the EC cookie domain, computed as `.{domain}`. + /// + /// Per spec §5.2, EC cookies derive their domain from + /// `publisher.domain` — **not** from `publisher.cookie_domain`. + /// This ensures the EC cookie is always scoped to the publisher's + /// apex domain regardless of how `cookie_domain` is configured. + #[must_use] + pub fn ec_cookie_domain(&self) -> String { + format!(".{}", self.domain) + } + /// Returns `true` if `proxy_secret` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -203,35 +218,216 @@ impl DerefMut for IntegrationSettings { } } -/// Edge Cookie configuration. -#[allow(unused)] +/// A partner (SSP, DSP, identity vendor) configured in `[[ec.partners]]`. +/// +/// Partners are defined statically in `trusted-server.toml` rather than +/// registered via API. At startup, each partner's `api_token` is hashed +/// (SHA-256) for O(1) auth lookups; the plaintext is never stored at runtime. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +pub struct EcPartner { + /// Unique partner identifier. Must match `^[a-z0-9_-]{1,32}$` and + /// not collide with reserved IDs (`ec`, `ts`, `eids`, etc.). + #[validate(custom(function = EcPartner::validate_id))] + pub id: String, + /// Human-readable partner name. + pub name: String, + /// `OpenRTB` `source.domain` for EID entries (e.g. `"liveramp.com"`). + pub source_domain: String, + /// `OpenRTB` `atype` value (typically 3). + #[serde( + default = "EcPartner::default_openrtb_atype", + deserialize_with = "from_value_or_str" + )] + pub openrtb_atype: u8, + /// Whether this partner's UIDs appear in auction `user.eids`. + #[serde(default, deserialize_with = "from_value_or_str")] + pub bidstream_enabled: bool, + /// Plaintext API token. Hashed at startup for auth lookups. + /// Used by batch sync (inbound) and identify (inbound). + pub api_token: Redacted, + /// Max batch sync API requests per partner per minute. + #[serde( + default = "EcPartner::default_batch_rate_limit", + deserialize_with = "from_value_or_str" + )] + pub batch_rate_limit: u32, + /// Whether server-to-server pull sync is enabled for this partner. + #[serde(default, deserialize_with = "from_value_or_str")] + pub pull_sync_enabled: bool, + /// URL to call for pull sync. Required when `pull_sync_enabled`. + #[serde(default)] + pub pull_sync_url: Option, + /// Allowlist of domains TS may call for this partner's pull sync. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub pull_sync_allowed_domains: Vec, + /// Legacy pull-sync refresh interval retained for config compatibility. + /// + /// EC identity entries no longer store per-partner sync timestamps, so + /// this value is not used by the current fill-missing-only pull sync + /// behavior. + #[serde( + default = "EcPartner::default_pull_sync_ttl_sec", + deserialize_with = "from_value_or_str" + )] + pub pull_sync_ttl_sec: u64, + /// Max pull sync calls per EC hash per partner per hour. + #[serde( + default = "EcPartner::default_pull_sync_rate_limit", + deserialize_with = "from_value_or_str" + )] + pub pull_sync_rate_limit: u32, + /// Outbound bearer token for pull sync requests. + #[serde(default)] + pub ts_pull_token: Option>, +} + +impl EcPartner { + const RESERVED_IDS: &[&str] = &[ + "ec", + "eids", + "ec-consent", + "eids-truncated", + "synthetic", + "ts", + "version", + "env", + ]; + + /// Validates a partner ID for safe use in dynamic headers and cookies. + /// + /// # Errors + /// + /// Returns a validation error when `id` does not match the configured + /// lowercase identifier policy or collides with a reserved name. + pub fn validate_id(id: &str) -> Result<(), ValidationError> { + if id.is_empty() || id.len() > 32 { + return Err(ValidationError::new("invalid_partner_id_length")); + } + if Self::RESERVED_IDS.contains(&id) { + return Err(ValidationError::new("reserved_partner_id")); + } + if !id.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' || byte == b'-' + }) { + return Err(ValidationError::new("invalid_partner_id")); + } + Ok(()) + } + + #[must_use] + pub const fn default_openrtb_atype() -> u8 { + 3 + } + + #[must_use] + pub const fn default_batch_rate_limit() -> u32 { + 60 + } + + #[must_use] + pub const fn default_pull_sync_ttl_sec() -> u64 { + 86400 + } + + #[must_use] + pub const fn default_pull_sync_rate_limit() -> u32 { + 10 + } +} + +/// Edge Cookie (EC) configuration. +/// +/// Mapped from the `[ec]` TOML section. Controls EC identity generation, +/// KV store names, and partner registry. #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -pub struct EdgeCookie { - #[validate(custom(function = EdgeCookie::validate_secret_key))] - pub secret_key: Redacted, +pub struct Ec { + /// Publisher passphrase used as HMAC key for EC generation. + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, + + /// Fastly KV store name for the EC identity graph. + #[serde(default)] + pub ec_store: Option, + + /// Maximum number of concurrent pull-sync requests. + #[serde(default = "Ec::default_pull_sync_concurrency")] + pub pull_sync_concurrency: usize, + + /// Entries with `cluster_size` at or below this value are treated as + /// individual users for identity resolution. B2B publishers should + /// raise this to 50+ since readers are frequently on office networks. + #[serde(default = "Ec::default_cluster_trust_threshold")] + pub cluster_trust_threshold: u32, + + /// Legacy cluster re-check interval retained for config compatibility. + /// + /// EC identity entries no longer store cluster-check timestamps, so this + /// value is not used. `/_ts/api/v1/identify` computes cluster size only + /// when an entry does not already have a stored `cluster_size`. + #[serde(default = "Ec::default_cluster_recheck_secs")] + pub cluster_recheck_secs: u64, + + /// Partners (SSPs, DSPs, identity vendors) for EC identity sync. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + #[validate(nested)] + pub partners: Vec, } -impl EdgeCookie { +impl Ec { /// Known placeholder values that must not be used in production. - pub const SECRET_KEY_PLACEHOLDERS: &[&str] = &["secret-key", "secret_key", "trusted-server"]; + pub const PASSPHRASE_PLACEHOLDERS: &[&str] = &[ + "secret-key", + "secret_key", + "trusted-server", + "trusted-server-placeholder-secret", + ]; + + /// Default maximum concurrent pull-sync requests. + #[must_use] + pub const fn default_pull_sync_concurrency() -> usize { + 3 + } + + /// Default cluster trust threshold. + #[must_use] + pub const fn default_cluster_trust_threshold() -> u32 { + 10 + } + + /// Default cluster re-check interval (1 hour). + #[must_use] + pub const fn default_cluster_recheck_secs() -> u64 { + 3600 + } - /// Returns `true` if `secret_key` matches a known placeholder value + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] - pub fn is_placeholder_secret_key(secret_key: &str) -> bool { - Self::SECRET_KEY_PLACEHOLDERS + pub fn is_placeholder_passphrase(passphrase: &str) -> bool { + Self::PASSPHRASE_PLACEHOLDERS .iter() - .any(|p| p.eq_ignore_ascii_case(secret_key)) + .any(|p| p.eq_ignore_ascii_case(passphrase)) } - /// Validates that the secret key is not empty. + /// Minimum passphrase length for HMAC-SHA256 key strength. + /// + /// The EC passphrase is long-lived keying material for visitor ID + /// derivation. Operators should use a high-entropy random passphrase per + /// the EC setup and key-rotation documentation. + const MIN_PASSPHRASE_LENGTH: usize = 32; + + /// Validates that the passphrase is not empty and meets minimum length. /// /// # Errors /// - /// Returns a validation error if the secret key is empty. - pub fn validate_secret_key(secret_key: &Redacted) -> Result<(), ValidationError> { - if secret_key.expose().is_empty() { - return Err(ValidationError::new("empty_secret_key")); + /// Returns a validation error if the passphrase is empty or shorter + /// than [`Self::MIN_PASSPHRASE_LENGTH`] characters. + pub fn validate_passphrase(passphrase: &Redacted) -> Result<(), ValidationError> { + if passphrase.expose().is_empty() { + return Err(ValidationError::new("empty_passphrase")); + } + if passphrase.expose().len() < Self::MIN_PASSPHRASE_LENGTH { + return Err(ValidationError::new("short_passphrase")); } Ok(()) } @@ -405,7 +601,7 @@ pub struct Settings { pub publisher: Publisher, #[serde(default)] #[validate(nested)] - pub edge_cookie: EdgeCookie, + pub ec: Ec, #[serde(default)] pub integrations: IntegrationSettings, #[serde(default, deserialize_with = "vec_from_seq_or_map")] @@ -425,7 +621,6 @@ pub struct Settings { pub proxy: Proxy, } -#[allow(unused)] impl Settings { /// Creates a new [`Settings`] instance from a pre-built TOML string. /// @@ -444,6 +639,13 @@ impl Settings { settings.proxy.normalize(); settings.consent.validate(); settings.prepare_runtime()?; + + settings.validate().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Configuration validation failed: {err}"), + }) + })?; + settings.validate_admin_coverage()?; Ok(settings) @@ -507,6 +709,31 @@ impl Settings { Ok(()) } + /// Reject settings that still contain known placeholder secrets. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::InsecureDefault`] when one or more secret + /// fields still contain a placeholder value. + pub fn reject_placeholder_secrets(&self) -> Result<(), Report> { + let mut insecure_fields: Vec<&str> = Vec::new(); + + if Ec::is_placeholder_passphrase(self.ec.passphrase.expose()) { + insecure_fields.push("ec.passphrase"); + } + if Publisher::is_placeholder_proxy_secret(self.publisher.proxy_secret.expose()) { + insecure_fields.push("publisher.proxy_secret"); + } + + if insecure_fields.is_empty() { + Ok(()) + } else { + Err(Report::new(TrustedServerError::InsecureDefault { + field: insecure_fields.join(", "), + })) + } + } + /// Resolve the first handler whose regex matches the request path. /// /// # Errors @@ -532,7 +759,8 @@ impl Settings { /// endpoints are always protected by authentication. /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new /// admin routes to `crates/trusted-server-adapter-fastly/src/main.rs`. - pub(crate) const ADMIN_ENDPOINTS: &[&str] = &["/admin/keys/rotate", "/admin/keys/deactivate"]; + pub(crate) const ADMIN_ENDPOINTS: &[&str] = + &["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"]; /// Returns admin endpoint paths that no configured handler covers. /// @@ -576,7 +804,7 @@ impl Settings { Err(Report::new(TrustedServerError::Configuration { message: format!( "No handler covers admin endpoint(s): {}. \ - Add a [[handlers]] entry with a path regex matching /admin/ \ + Add a [[handlers]] entry with a path regex matching /_ts/admin/ \ to protect admin access.", uncovered.join(", ") ), @@ -599,6 +827,33 @@ impl Settings { } } +fn validate_publisher_domain(value: &str) -> Result<(), ValidationError> { + if value.trim() != value || value.is_empty() || value.len() > 253 { + return Err(ValidationError::new("invalid_publisher_domain")); + } + if value.starts_with('.') || value.ends_with('.') || value.contains(['/', ':']) { + return Err(ValidationError::new("invalid_publisher_domain")); + } + + for label in value.split('.') { + if label.is_empty() || label.len() > 63 { + return Err(ValidationError::new("invalid_publisher_domain")); + } + let bytes = label.as_bytes(); + if bytes.first() == Some(&b'-') || bytes.last() == Some(&b'-') { + return Err(ValidationError::new("invalid_publisher_domain")); + } + if !bytes + .iter() + .all(|byte| byte.is_ascii_alphanumeric() || *byte == b'-') + { + return Err(ValidationError::new("invalid_publisher_domain")); + } + } + + Ok(()) +} + fn validate_cookie_domain(value: &str) -> Result<(), ValidationError> { // `=` is excluded: it only has special meaning in the name=value pair, // not within the Domain attribute value. @@ -636,6 +891,19 @@ fn validate_path(value: &str) -> Result<(), ValidationError> { validation_error }) } +fn from_value_or_str<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: DeserializeOwned + FromStr, + T::Err: std::fmt::Display, +{ + let value = JsonValue::deserialize(deserializer)?; + match value { + JsonValue::String(value) => T::from_str(&value).map_err(serde::de::Error::custom), + other => serde_json::from_value(other).map_err(serde::de::Error::custom), + } +} + // Helper: allow Vec fields to deserialize from either a JSON array or a map of numeric indices. // This lets env vars like TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__0=smartadserver work, which the config env source // represents as an object {"0": "value"} rather than a sequence. Also supports string inputs that are @@ -802,11 +1070,19 @@ mod tests { ); assert_eq!(settings.publisher.domain, "test-publisher.com"); assert_eq!(settings.publisher.cookie_domain, ".test-publisher.com"); + assert_eq!( + settings.publisher.ec_cookie_domain(), + ".test-publisher.com", + "EC cookie domain should be computed as .{{domain}}" + ); assert_eq!( settings.publisher.origin_url, "https://origin.test-publisher.com" ); - assert_eq!(settings.edge_cookie.secret_key.expose(), "test-secret-key"); + assert_eq!( + settings.ec.passphrase.expose(), + "test-secret-key-32-bytes-minimum" + ); settings.validate().expect("Failed to validate settings"); } @@ -818,14 +1094,72 @@ mod tests { r#"origin_url = "https://origin.test-publisher.com/""#, ); - let settings = Settings::from_toml(&toml_str).expect("should parse TOML"); - let result = settings.validate(); + let result = Settings::from_toml(&toml_str); assert!( result.is_err(), "origin_url ending with '/' should fail validation" ); } + #[test] + fn validate_rejects_invalid_publisher_domains() { + for domain in [ + "", + ".example.com", + "example.com.", + "https://example.com", + "bad_domain.com", + ] { + let toml_str = crate_test_settings_str().replace( + r#"domain = "test-publisher.com""#, + &format!(r#"domain = "{domain}""#), + ); + + let result = Settings::from_toml(&toml_str); + assert!(result.is_err(), "should reject invalid domain {domain:?}"); + } + } + + #[test] + fn validate_accepts_localhost_publisher_domain() { + let toml_str = crate_test_settings_str().replace( + r#"domain = "test-publisher.com""#, + r#"domain = "localhost""#, + ); + + let settings = Settings::from_toml(&toml_str).expect("should accept localhost domain"); + assert_eq!(settings.publisher.ec_cookie_domain(), ".localhost"); + } + + #[test] + fn validate_rejects_invalid_ec_partner_ids() { + for partner_id in [ + "Upper", + "bad id", + "ec", + "", + "abcdefghijklmnopqrstuvwxyzabcdefg", + ] { + let toml_str = format!( + r#"{} + [[ec.partners]] + id = "{}" + name = "Invalid Partner" + source_domain = "invalid.example.com" + api_token = "invalid-token" + "#, + crate_test_settings_str(), + partner_id, + ); + + let result = Settings::from_toml(&toml_str); + assert!( + result.is_err(), + "should reject invalid partner ID {partner_id:?}" + ); + } + } + #[test] fn prepare_runtime_rejects_invalid_handler_regex() { let toml_str = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "(""#); @@ -853,35 +1187,55 @@ mod tests { } #[test] - fn is_placeholder_secret_key_rejects_all_known_placeholders() { - for placeholder in EdgeCookie::SECRET_KEY_PLACEHOLDERS { + fn is_placeholder_passphrase_rejects_all_known_placeholders() { + for placeholder in Ec::PASSPHRASE_PLACEHOLDERS { assert!( - EdgeCookie::is_placeholder_secret_key(placeholder), - "should detect placeholder secret_key '{placeholder}'" + Ec::is_placeholder_passphrase(placeholder), + "should detect placeholder passphrase '{placeholder}'" ); } } #[test] - fn is_placeholder_secret_key_is_case_insensitive() { + fn is_placeholder_passphrase_is_case_insensitive() { assert!( - EdgeCookie::is_placeholder_secret_key("SECRET-KEY"), - "should detect case-insensitive placeholder secret_key" + Ec::is_placeholder_passphrase("SECRET-KEY"), + "should detect case-insensitive placeholder passphrase" ); assert!( - EdgeCookie::is_placeholder_secret_key("Trusted-Server"), - "should detect mixed-case placeholder secret_key" + Ec::is_placeholder_passphrase("Trusted-Server"), + "should detect mixed-case placeholder passphrase" ); } #[test] - fn is_placeholder_secret_key_accepts_non_placeholder() { + fn is_placeholder_passphrase_accepts_non_placeholder() { assert!( - !EdgeCookie::is_placeholder_secret_key("test-secret-key"), - "should accept non-placeholder secret_key" + !Ec::is_placeholder_passphrase("test-secret-key-32-bytes-minimum"), + "should accept non-placeholder passphrase" + ); + } + + #[test] + fn validate_passphrase_rejects_under_32_characters() { + let passphrase = Redacted::new("a".repeat(31)); + + let err = Ec::validate_passphrase(&passphrase).expect_err("should reject short passphrase"); + + assert_eq!( + err.code.as_ref(), + "short_passphrase", + "should report short passphrase validation error" ); } + #[test] + fn validate_passphrase_accepts_32_characters() { + let passphrase = Redacted::new("a".repeat(32)); + + Ec::validate_passphrase(&passphrase).expect("should accept 32-character passphrase"); + } + #[test] fn is_placeholder_proxy_secret_rejects_all_known_placeholders() { for placeholder in Publisher::PROXY_SECRET_PLACEHOLDERS { @@ -1092,7 +1446,7 @@ mod tests { (path_key_0, Some("^/env-handler")), (username_key_0, Some("env-user")), (password_key_0, Some("env-pass")), - (path_key_1, Some("^/admin")), + (path_key_1, Some("^/_ts/admin")), (username_key_1, Some("admin")), (password_key_1, Some("admin-pass")), ], @@ -1108,6 +1462,156 @@ mod tests { ); } + #[test] + fn test_ec_partners_override_with_indexed_env() { + let toml_str = crate_test_settings_str(); + + let origin_key = format!( + "{}{}PUBLISHER{}ORIGIN_URL", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_id_key = format!( + "{}{}EC{}PARTNERS{}0{}ID", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_name_key = format!( + "{}{}EC{}PARTNERS{}0{}NAME", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_source_domain_key = format!( + "{}{}EC{}PARTNERS{}0{}SOURCE_DOMAIN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_openrtb_atype_key = format!( + "{}{}EC{}PARTNERS{}0{}OPENRTB_ATYPE", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_bidstream_enabled_key = format!( + "{}{}EC{}PARTNERS{}0{}BIDSTREAM_ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_0_api_token_key = format!( + "{}{}EC{}PARTNERS{}0{}API_TOKEN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_id_key = format!( + "{}{}EC{}PARTNERS{}1{}ID", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_name_key = format!( + "{}{}EC{}PARTNERS{}1{}NAME", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_source_domain_key = format!( + "{}{}EC{}PARTNERS{}1{}SOURCE_DOMAIN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_openrtb_atype_key = format!( + "{}{}EC{}PARTNERS{}1{}OPENRTB_ATYPE", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_bidstream_enabled_key = format!( + "{}{}EC{}PARTNERS{}1{}BIDSTREAM_ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + let partner_1_api_token_key = format!( + "{}{}EC{}PARTNERS{}1{}API_TOKEN", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_vars( + [ + (origin_key, Some("https://origin.test-publisher.com")), + (partner_0_id_key, Some("envpartner0")), + (partner_0_name_key, Some("Env Partner 0")), + (partner_0_source_domain_key, Some("envpartner0.example.com")), + (partner_0_openrtb_atype_key, Some("1")), + (partner_0_bidstream_enabled_key, Some("true")), + (partner_0_api_token_key, Some("env-token-0")), + (partner_1_id_key, Some("envpartner1")), + (partner_1_name_key, Some("Env Partner 1")), + (partner_1_source_domain_key, Some("envpartner1.example.com")), + (partner_1_openrtb_atype_key, Some("3")), + (partner_1_bidstream_enabled_key, Some("false")), + (partner_1_api_token_key, Some("env-token-1")), + ], + || { + let settings = Settings::from_toml_and_env(&toml_str) + .expect("Settings should load indexed EC partners from env"); + + assert_eq!(settings.ec.partners.len(), 2); + assert_eq!(settings.ec.partners[0].id, "envpartner0"); + assert_eq!(settings.ec.partners[0].name, "Env Partner 0"); + assert_eq!( + settings.ec.partners[0].source_domain, + "envpartner0.example.com" + ); + assert_eq!(settings.ec.partners[0].openrtb_atype, 1); + assert!(settings.ec.partners[0].bidstream_enabled); + assert_eq!(settings.ec.partners[0].api_token.expose(), "env-token-0"); + assert_eq!(settings.ec.partners[1].id, "envpartner1"); + assert_eq!(settings.ec.partners[1].name, "Env Partner 1"); + assert_eq!( + settings.ec.partners[1].source_domain, + "envpartner1.example.com" + ); + assert_eq!(settings.ec.partners[1].openrtb_atype, 3); + assert!(!settings.ec.partners[1].bidstream_enabled); + assert_eq!(settings.ec.partners[1].api_token.expose(), "env-token-1"); + }, + ); + } + #[test] fn test_invalid_handler_override_fails_during_runtime_preparation() { let toml_str = crate_test_settings_str(); @@ -1815,8 +2319,8 @@ mod tests { origin_url = "https://origin.test-publisher.com" proxy_secret = "unit-test-proxy-secret" - [edge_cookie] - secret_key = "test-secret-key" + [ec] + passphrase = "test-secret-key-32-bytes-minimum" [request_signing] config_store_id = "test-config-store-id" @@ -1836,8 +2340,8 @@ mod tests { .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/admin/keys/rotate", "/admin/keys/deactivate"], - "should report both admin endpoints as uncovered" + vec!["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate",], + "should report all admin endpoints as uncovered" ); } @@ -1849,7 +2353,7 @@ mod tests { .expect("should check admin coverage"); assert!( uncovered.is_empty(), - "should report no uncovered admin endpoints when handler covers /admin" + "should report no uncovered admin endpoints when handler covers /_ts/admin" ); } @@ -1858,7 +2362,7 @@ mod tests { let toml_str = settings_str_without_admin_handler() + r#" [[handlers]] - path = "^/admin/keys/rotate$" + path = "^/_ts/admin/keys/rotate$" username = "admin" password = "secret" "#; @@ -1870,8 +2374,8 @@ mod tests { .expect("should check admin coverage"); assert_eq!( uncovered, - vec!["/admin/keys/deactivate"], - "should detect that only deactivate is uncovered" + vec!["/_ts/admin/keys/deactivate"], + "should detect endpoints not covered by the rotate-only handler" ); } @@ -1941,9 +2445,9 @@ mod tests { .lines() .filter_map(|line| { let trimmed = line.trim(); - // Match arms look like: (Method::POST, "/admin/...") => ... - if trimmed.starts_with('(') && trimmed.contains("\"/admin/") { - let start = trimmed.find("\"/admin/")?; + // Match arms look like: (Method::POST, "/_ts/admin/...") => ... + if trimmed.starts_with('(') && trimmed.contains("\"/_ts/admin/") { + let start = trimmed.find("\"/_ts/admin/")?; let rest = &trimmed[start + 1..]; let end = rest.find('"')?; Some(&rest[..end]) diff --git a/crates/trusted-server-core/src/settings_data.rs b/crates/trusted-server-core/src/settings_data.rs index f69fc7ba..2f0a29fd 100644 --- a/crates/trusted-server-core/src/settings_data.rs +++ b/crates/trusted-server-core/src/settings_data.rs @@ -3,7 +3,7 @@ use error_stack::{Report, ResultExt}; use validator::Validate; use crate::error::TrustedServerError; -use crate::settings::{EdgeCookie, Publisher, Settings}; +use crate::settings::Settings; pub use crate::auction_config_types::AuctionConfig; @@ -40,37 +40,126 @@ pub fn get_settings() -> Result> { ); } - if EdgeCookie::is_placeholder_secret_key(settings.edge_cookie.secret_key.expose()) { - log::warn!( - "INSECURE: edge_cookie.secret_key is set to a default placeholder — \ - HMAC-SHA256 signatures can be forged. \ - Override via TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY at build time" + settings.reject_placeholder_secrets()?; + + Ok(settings) +} + +#[cfg(test)] +mod tests { + use crate::error::TrustedServerError; + use crate::settings::Settings; + use crate::test_support::tests::crate_test_settings_str; + + /// Builds a TOML string with the given secret values swapped in. + /// + /// # Panics + /// + /// Panics if the replacement patterns no longer match the test TOML, + /// which would cause the substitution to silently no-op. + fn toml_with_secrets(passphrase: &str, proxy_secret: &str) -> String { + let original = crate_test_settings_str(); + let after_passphrase = original.replace( + r#"passphrase = "test-secret-key-32-bytes-minimum""#, + &format!(r#"passphrase = "{passphrase}""#), ); + assert_ne!( + after_passphrase, original, + "should have replaced passphrase value" + ); + let result = after_passphrase.replace( + r#"proxy_secret = "unit-test-proxy-secret""#, + &format!(r#"proxy_secret = "{proxy_secret}""#), + ); + assert_ne!( + result, after_passphrase, + "should have replaced proxy_secret value" + ); + result } - if Publisher::is_placeholder_proxy_secret(settings.publisher.proxy_secret.expose()) { - log::warn!( - "INSECURE: publisher.proxy_secret is set to a default placeholder — \ - XChaCha20-Poly1305 encrypted URLs can be decrypted by anyone. \ - Override via TRUSTED_SERVER__PUBLISHER__PROXY_SECRET at build time" + #[test] + fn rejects_placeholder_passphrase() { + let toml = toml_with_secrets("trusted-server-placeholder-secret", "real-proxy-secret"); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder secret_key"); + let root = err.current_context(); + assert!( + matches!(root, TrustedServerError::InsecureDefault { field } if field.contains("ec.passphrase")), + "error should mention ec.passphrase, got: {root}" ); } - Ok(settings) -} + #[test] + fn rejects_placeholder_proxy_secret() { + let toml = toml_with_secrets( + "production-secret-key-32-bytes-min", + "change-me-proxy-secret", + ); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder proxy_secret"); + let root = err.current_context(); + assert!( + matches!(root, TrustedServerError::InsecureDefault { field } if field.contains("publisher.proxy_secret")), + "error should mention publisher.proxy_secret, got: {root}" + ); + } -#[cfg(test)] -mod tests { - use super::*; + #[test] + fn rejects_both_placeholders_in_single_error() { + let toml = toml_with_secrets( + "trusted-server-placeholder-secret", + "change-me-proxy-secret", + ); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject both placeholder secrets"); + let root = err.current_context(); + match root { + TrustedServerError::InsecureDefault { field } => { + assert!( + field.contains("ec.passphrase"), + "error should mention ec.passphrase, got: {field}" + ); + assert!( + field.contains("publisher.proxy_secret"), + "error should mention publisher.proxy_secret, got: {field}" + ); + } + other => panic!("expected InsecureDefault, got: {other}"), + } + } #[test] - fn get_settings_loads_embedded_toml_successfully() { - // The embedded TOML contains placeholder secrets (e.g. "trusted-server", - // "change-me-proxy-secret"). This is expected — production builds override - // them via TRUSTED_SERVER__* env vars at build time. - let settings = get_settings().expect("should load settings from embedded TOML"); - assert!(!settings.publisher.domain.is_empty()); - assert!(!settings.publisher.cookie_domain.is_empty()); - assert!(!settings.publisher.origin_url.is_empty()); + fn accepts_non_placeholder_secrets() { + let toml = toml_with_secrets( + "production-secret-key-32-bytes-min", + "production-proxy-secret", + ); + let settings = Settings::from_toml(&toml).expect("should parse TOML"); + settings + .reject_placeholder_secrets() + .expect("non-placeholder secrets should pass validation"); + } + + /// Smoke-test the full `get_settings()` pipeline (embedded bytes → UTF-8 → + /// parse → validate → placeholder check). The build-time TOML ships with + /// placeholder secrets, so the expected outcome is an [`InsecureDefault`] + /// error — but reaching that error proves every earlier stage succeeded. + #[test] + fn get_settings_rejects_embedded_placeholder_secrets() { + let err = super::get_settings().expect_err("should reject embedded placeholder secrets"); + assert!( + matches!( + err.current_context(), + TrustedServerError::InsecureDefault { .. } + ), + "should fail with InsecureDefault, got: {err}" + ); } } diff --git a/crates/trusted-server-core/src/storage/api_client.rs b/crates/trusted-server-core/src/storage/api_client.rs deleted file mode 100644 index 81a2d57b..00000000 --- a/crates/trusted-server-core/src/storage/api_client.rs +++ /dev/null @@ -1,291 +0,0 @@ -//! Fastly management API client (legacy). -//! -//! This module holds [`FastlyApiClient`], which wraps the Fastly management -//! REST API for write operations on config and secret stores. -//! New code should use [`crate::platform::PlatformConfigStore`] and -//! [`crate::platform::PlatformSecretStore`] write methods instead. -//! This type will be removed once all call sites have migrated. - -use std::io::Read; - -use error_stack::{Report, ResultExt}; -use fastly::{Request, Response}; -use http::StatusCode; - -use crate::backend::BackendConfig; -use crate::error::TrustedServerError; -use crate::storage::secret_store::FastlySecretStore; - -const FASTLY_API_HOST: &str = "https://api.fastly.com"; - -fn build_config_item_payload(value: &str) -> String { - format!("item_value={}", urlencoding::encode(value)) -} - -/// HTTP client for the Fastly management API. -/// -/// Used to perform write operations on config and secret stores via the -/// Fastly REST API. Reads are performed directly through the edge-side SDK. -/// -/// # Migration note -/// -/// This type predates the `platform` abstraction. New code should use -/// [`crate::platform::PlatformConfigStore`] and -/// [`crate::platform::PlatformSecretStore`] write methods instead. -pub struct FastlyApiClient { - api_key: Vec, - base_url: &'static str, - backend_name: String, -} - -impl FastlyApiClient { - /// Creates a new Fastly API client using the default secret store. - /// - /// # Errors - /// - /// Returns an error if the secret store cannot be opened or the API key - /// cannot be retrieved. - pub fn new() -> Result> { - Self::from_secret_store("api-keys", "api_key") - } - - /// Creates a new Fastly API client reading credentials from a specified - /// secret store entry. - /// - /// # Errors - /// - /// Returns an error if the API backend cannot be ensured or the API key - /// cannot be retrieved. - pub fn from_secret_store( - store_name: &str, - key_name: &str, - ) -> Result> { - let backend_name = BackendConfig::from_url("https://api.fastly.com", true)?; - let api_key = FastlySecretStore::new(store_name).get(key_name)?; - - log::debug!("FastlyApiClient initialized"); - - Ok(Self { - api_key, - base_url: FASTLY_API_HOST, - backend_name, - }) - } - - fn make_request( - &self, - method: &str, - path: &str, - body: Option, - content_type: &str, - ) -> Result> { - let url = format!("{}{}", self.base_url, path); - let api_key_str = String::from_utf8_lossy(&self.api_key).to_string(); - - let mut request = match method { - "GET" => Request::get(&url), - "POST" => Request::post(&url), - "PUT" => Request::put(&url), - "DELETE" => Request::delete(&url), - _ => { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("unsupported HTTP method: {}", method), - })) - } - }; - - request = request - .with_header("Fastly-Key", api_key_str) - .with_header("Accept", "application/json"); - - if let Some(body_content) = body { - request = request - .with_header("Content-Type", content_type) - .with_body(body_content); - } - - request.send(&self.backend_name).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to send API request: {}", e), - }) - }) - } - - /// Updates a configuration item in a Fastly config store. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns a non-OK status. - pub fn update_config_item( - &self, - store_id: &str, - key: &str, - value: &str, - ) -> Result<(), Report> { - let path = format!("/resources/stores/config/{}/item/{}", store_id, key); - let payload = build_config_item_payload(value); - - let mut response = self.make_request( - "PUT", - &path, - Some(payload), - "application/x-www-form-urlencoded", - )?; - - let mut buf = String::new(); - response - .get_body_mut() - .read_to_string(&mut buf) - .change_context(TrustedServerError::Configuration { - message: "failed to read config store API response".into(), - })?; - - if response.get_status() == StatusCode::OK { - Ok(()) - } else { - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "failed to update config item: HTTP {} - {}", - response.get_status(), - buf - ), - })) - } - } - - /// Creates a secret in a Fastly secret store. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns a non-OK status. - pub fn create_secret( - &self, - store_id: &str, - secret_name: &str, - secret_value: &str, - ) -> Result<(), Report> { - let path = format!("/resources/stores/secret/{}/secrets", store_id); - let payload = serde_json::json!({ - "name": secret_name, - "secret": secret_value - }); - - let mut response = - self.make_request("POST", &path, Some(payload.to_string()), "application/json")?; - - let mut buf = String::new(); - response - .get_body_mut() - .read_to_string(&mut buf) - .change_context(TrustedServerError::Configuration { - message: "failed to read secret store API response".into(), - })?; - - if response.get_status() == StatusCode::OK { - Ok(()) - } else { - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "failed to create secret: HTTP {} - {}", - response.get_status(), - buf - ), - })) - } - } - - /// Deletes a configuration item from a Fastly config store. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns a non-OK or - /// non-NO_CONTENT status. - pub fn delete_config_item( - &self, - store_id: &str, - key: &str, - ) -> Result<(), Report> { - let path = format!("/resources/stores/config/{}/item/{}", store_id, key); - - let mut response = self.make_request("DELETE", &path, None, "application/json")?; - - let mut buf = String::new(); - response - .get_body_mut() - .read_to_string(&mut buf) - .change_context(TrustedServerError::Configuration { - message: "failed to read config store delete API response".into(), - })?; - - if response.get_status() == StatusCode::OK - || response.get_status() == StatusCode::NO_CONTENT - { - Ok(()) - } else { - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "failed to delete config item: HTTP {} - {}", - response.get_status(), - buf - ), - })) - } - } - - /// Deletes a secret from a Fastly secret store. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns a non-OK or - /// non-NO_CONTENT status. - pub fn delete_secret( - &self, - store_id: &str, - secret_name: &str, - ) -> Result<(), Report> { - let path = format!( - "/resources/stores/secret/{}/secrets/{}", - store_id, secret_name - ); - - let mut response = self.make_request("DELETE", &path, None, "application/json")?; - - let mut buf = String::new(); - response - .get_body_mut() - .read_to_string(&mut buf) - .change_context(TrustedServerError::Configuration { - message: "failed to read secret store delete API response".into(), - })?; - - if response.get_status() == StatusCode::OK - || response.get_status() == StatusCode::NO_CONTENT - { - Ok(()) - } else { - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "failed to delete secret: HTTP {} - {}", - response.get_status(), - buf - ), - })) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn config_item_payload_url_encodes_reserved_characters() { - let payload = build_config_item_payload(r#"value with spaces + symbols &= {"kid":"a+b"}"#); - - assert_eq!( - payload, - "item_value=value%20with%20spaces%20%2B%20symbols%20%26%3D%20%7B%22kid%22%3A%22a%2Bb%22%7D", - "should URL-encode config item values in form payloads" - ); - } -} diff --git a/crates/trusted-server-core/src/storage/kv_store.rs b/crates/trusted-server-core/src/storage/kv_store.rs deleted file mode 100644 index c118005a..00000000 --- a/crates/trusted-server-core/src/storage/kv_store.rs +++ /dev/null @@ -1,570 +0,0 @@ -//! KV Store consent persistence. -//! -//! Stores and retrieves consent data from a KV Store, keyed by EC ID. This -//! provides consent continuity for returning users whose browsers may not -//! have consent cookies on every request. -//! -//! # Storage layout -//! -//! Each entry uses a single JSON body ([`KvConsentEntry`]) containing the raw -//! consent strings, context flags, and a fingerprint for write-on-change -//! detection. -//! -//! # Change detection -//! -//! Writes only occur when consent signals have actually changed. -//! [`consent_fingerprint`] hashes the raw strings into a compact fingerprint -//! stored in the body's `fp` field. On the next request, the existing -//! fingerprint is compared before writing. - -use bytes::Bytes; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -use crate::consent::jurisdiction::Jurisdiction; -use crate::consent::types::{ConsentContext, ConsentSource}; -use crate::platform::PlatformKvStore; - -// --------------------------------------------------------------------------- -// KV body (JSON, stored as value) -// --------------------------------------------------------------------------- - -/// Consent data stored in the KV Store body. -/// -/// Contains the raw consent strings needed to reconstruct a [`ConsentContext`]. -/// Decoded data (TCF, GPP, US Privacy) is not stored — it is re-decoded on -/// read to avoid stale decoded state. -/// -/// The `fp` field holds the consent fingerprint for write-on-change detection. -/// Entries written before PR5 lack this field; `#[serde(default)]` treats them -/// as having an empty fingerprint, which always triggers a self-healing -/// re-write. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct KvConsentEntry { - /// Fingerprint of consent signals for write-on-change detection. - /// - /// Written by [`save_consent_to_kv`]. Entries written before PR5 lack - /// this field; `#[serde(default)]` treats them as having an empty - /// fingerprint, which always triggers a self-healing re-write. - #[serde(default)] - pub fp: String, - /// Raw TC String from `euconsent-v2` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_tc_string: Option, - /// Raw GPP string from `__gpp` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_gpp_string: Option, - /// GPP section IDs (decoded or from `__gpp_sid` cookie). - #[serde(skip_serializing_if = "Option::is_none")] - pub gpp_section_ids: Option>, - /// Raw US Privacy string from `us_privacy` cookie. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_us_privacy: Option, - /// Raw Google Additional Consent (AC) string. - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_ac_string: Option, - - /// Whether GDPR applies to this request. - pub gdpr_applies: bool, - /// Global Privacy Control signal. - pub gpc: bool, - /// Serialized jurisdiction (e.g. `"GDPR"`, `"US-CA"`, `"unknown"`). - pub jurisdiction: String, - - /// When this entry was stored (deciseconds since Unix epoch). - pub stored_at_ds: u64, -} - -// --------------------------------------------------------------------------- -// Conversions -// --------------------------------------------------------------------------- - -/// Builds a [`KvConsentEntry`] from a [`ConsentContext`]. -/// -/// Captures only the raw strings and contextual flags. Decoded data is -/// intentionally omitted — it will be re-decoded on read. The `fp` field is -/// initialized to an empty string and must be set by the caller before writing. -#[must_use] -pub fn entry_from_context(ctx: &ConsentContext, now_ds: u64) -> KvConsentEntry { - KvConsentEntry { - fp: String::new(), - raw_tc_string: ctx.raw_tc_string.clone(), - raw_gpp_string: ctx.raw_gpp_string.clone(), - gpp_section_ids: ctx.gpp_section_ids.clone(), - raw_us_privacy: ctx.raw_us_privacy.clone(), - raw_ac_string: ctx.raw_ac_string.clone(), - gdpr_applies: ctx.gdpr_applies, - gpc: ctx.gpc, - jurisdiction: ctx.jurisdiction.to_string(), - stored_at_ds: now_ds, - } -} - -/// Converts a [`KvConsentEntry`] into [`crate::consent::types::RawConsentSignals`] -/// suitable for re-decoding via [`crate::consent::build_context_from_signals`]. -#[must_use] -pub fn signals_from_entry(entry: &KvConsentEntry) -> crate::consent::types::RawConsentSignals { - crate::consent::types::RawConsentSignals { - raw_tc_string: entry.raw_tc_string.clone(), - raw_gpp_string: entry.raw_gpp_string.clone(), - raw_gpp_sid: entry.gpp_section_ids.as_ref().map(|ids| { - ids.iter() - .map(ToString::to_string) - .collect::>() - .join(",") - }), - raw_us_privacy: entry.raw_us_privacy.clone(), - gpc: entry.gpc, - } -} - -/// Reconstructs a [`ConsentContext`] from a KV Store entry. -/// -/// Re-decodes the raw strings to populate structured fields (TCF, GPP, US -/// Privacy). The `source` is set to [`ConsentSource::KvStore`] and the -/// `jurisdiction` is parsed from the stored string representation. -#[must_use] -pub fn context_from_entry(entry: &KvConsentEntry) -> ConsentContext { - let signals = signals_from_entry(entry); - let mut ctx = crate::consent::build_context_from_signals(&signals); - - // Restore context fields that aren't derived from raw signals. - ctx.gdpr_applies = entry.gdpr_applies; - ctx.gpc = entry.gpc; - ctx.raw_ac_string = entry.raw_ac_string.clone(); - ctx.jurisdiction = parse_jurisdiction(&entry.jurisdiction); - ctx.source = ConsentSource::KvStore; - - ctx -} - -// --------------------------------------------------------------------------- -// Fingerprinting -// --------------------------------------------------------------------------- - -/// Computes a compact fingerprint of the consent signals for change detection. -/// -/// Returns the first 16 hex characters of a SHA-256 hash computed over all -/// raw consent strings and the GPC flag. This is sufficient for detecting -/// changes without storing full hashes. -#[must_use] -pub fn consent_fingerprint(ctx: &ConsentContext) -> String { - let mut hasher = Sha256::new(); - - // Feed each signal into the hash, separated by a sentinel byte to - // prevent ambiguity (e.g., None+Some("x") vs Some("x")+None). - hash_optional(&mut hasher, ctx.raw_tc_string.as_deref()); - hash_optional(&mut hasher, ctx.raw_gpp_string.as_deref()); - hash_optional(&mut hasher, ctx.raw_us_privacy.as_deref()); - hash_optional(&mut hasher, ctx.raw_ac_string.as_deref()); - hasher.update(if ctx.gpc { b"1" } else { b"0" }); - - // Include GPP section IDs so SID-only changes trigger a KV write. - if let Some(sids) = &ctx.gpp_section_ids { - let mut sorted = sids.clone(); - sorted.sort_unstable(); - for sid in &sorted { - hasher.update(sid.to_string().as_bytes()); - hasher.update(b"\xFF"); - } - } else { - hasher.update(b"\x00"); - } - - let result = hasher.finalize(); - hex::encode(&result[..8]) // 16 hex chars = 8 bytes = 64 bits -} - -/// Feeds an optional string into the hasher with sentinel bytes. -fn hash_optional(hasher: &mut Sha256, value: Option<&str>) { - match value { - Some(s) => { - hasher.update(b"\x01"); - hasher.update(s.as_bytes()); - } - None => hasher.update(b"\x00"), - } -} - -/// Parses a jurisdiction string back into a [`Jurisdiction`] enum. -fn parse_jurisdiction(s: &str) -> Jurisdiction { - match s { - "GDPR" => Jurisdiction::Gdpr, - "non-regulated" => Jurisdiction::NonRegulated, - "unknown" => Jurisdiction::Unknown, - s if s.starts_with("US-") => Jurisdiction::UsState(s[3..].to_owned()), - _ => Jurisdiction::Unknown, - } -} - -// --------------------------------------------------------------------------- -// KV Store operations -// --------------------------------------------------------------------------- - -/// Checks whether the stored consent fingerprint matches the current one. -/// -/// Returns `true` when the stored body's `fp` field equals `new_fp`, meaning -/// no write is needed. Returns `false` when the key is absent, the body -/// cannot be deserialized, or the fingerprint differs. -/// -/// Entries written before PR5 have an empty `fp` (via `#[serde(default)]`), -/// which never matches a computed fingerprint and triggers a self-healing -/// re-write. -fn fingerprint_unchanged(store: &dyn PlatformKvStore, key: &str, new_fp: &str) -> bool { - let bytes = match futures::executor::block_on(store.get_bytes(key)) { - Ok(Some(bytes)) => bytes, - _ => return false, - }; - - serde_json::from_slice::(&bytes) - .map(|entry| entry.fp == new_fp) - .unwrap_or(false) -} - -/// Loads consent data from the KV store for a given EC ID. -/// -/// Returns `Some(ConsentContext)` if a valid entry is found, [`None`] if the -/// key does not exist or deserialization fails. Errors are logged but never -/// propagated — KV failures must not break the request pipeline. -/// -/// # Arguments -/// -/// * `store` — KV store opened by the adapter. -/// * `ec_id` — Edge Cookie ID used as the KV key. -#[must_use] -pub fn load_consent_from_kv(store: &dyn PlatformKvStore, ec_id: &str) -> Option { - let bytes = match futures::executor::block_on(store.get_bytes(ec_id)) { - Ok(Some(bytes)) => bytes, - Ok(None) => { - log::debug!("Consent KV lookup miss for '{ec_id}'"); - return None; - } - Err(e) => { - log::debug!("Consent KV lookup error for '{ec_id}': {e}"); - return None; - } - }; - - match serde_json::from_slice::(&bytes) { - Ok(entry) => { - log::info!( - "Loaded consent from KV store for '{ec_id}' (stored_at_ds={})", - entry.stored_at_ds - ); - Some(context_from_entry(&entry)) - } - Err(e) => { - log::warn!("Failed to deserialize consent KV entry for '{ec_id}': {e}"); - None - } - } -} - -/// Saves consent data to the KV store, writing only when signals have changed. -/// -/// Compares the fingerprint of current consent signals against the fingerprint -/// embedded in the stored entry. If they match, the write is skipped. -/// The fingerprint is embedded in the body so no KV metadata is required. -/// -/// # Arguments -/// -/// * `store` — KV store opened by the adapter. -/// * `ec_id` — Edge Cookie ID used as the KV key. -/// * `ctx` — Current request's consent context. -/// * `max_age_days` — TTL for the entry, matching `max_consent_age_days`. -pub fn save_consent_to_kv( - store: &dyn PlatformKvStore, - ec_id: &str, - ctx: &ConsentContext, - max_age_days: u32, -) { - if ctx.is_empty() { - log::debug!("Skipping consent KV write: consent is empty"); - return; - } - - let fp = consent_fingerprint(ctx); - - if fingerprint_unchanged(store, ec_id, &fp) { - log::debug!("Consent unchanged for '{ec_id}' (fp={fp}), skipping write"); - return; - } - - let mut entry = entry_from_context(ctx, crate::consent::now_deciseconds()); - entry.fp = fp.clone(); - - let body = match serde_json::to_vec(&entry) { - Ok(body) => Bytes::from(body), - Err(e) => { - log::warn!("Failed to serialize consent entry for '{ec_id}': {e}"); - return; - } - }; - - let ttl = std::time::Duration::from_secs(u64::from(max_age_days) * 86_400); - - match futures::executor::block_on(store.put_bytes_with_ttl(ec_id, body, ttl)) { - Ok(()) => { - log::info!("Saved consent to KV store for '{ec_id}' (fp={fp}, ttl={max_age_days}d)"); - } - Err(e) => { - log::warn!("Failed to write consent to KV store for '{ec_id}': {e}"); - } - } -} - -/// Deletes a consent entry from the KV store for a given EC ID. -/// -/// Used when a user revokes consent — the existing EC cookie is being -/// expired, so the persisted consent data must also be removed. -/// -/// Errors are logged but never propagated — KV failures must not -/// break the request pipeline. -pub fn delete_consent_from_kv(store: &dyn PlatformKvStore, ec_id: &str) { - match futures::executor::block_on(store.delete(ec_id)) { - Ok(()) => { - log::info!("Deleted consent KV entry for '{ec_id}' (consent revoked)"); - } - Err(e) => { - log::warn!("Failed to delete consent KV entry for '{ec_id}': {e}"); - } - } -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -fn make_test_context() -> ConsentContext { - ConsentContext { - raw_tc_string: Some("CPXxGfAPXxGfA".to_owned()), - raw_gpp_string: Some("DBACNYA~CPXxGfA".to_owned()), - gpp_section_ids: Some(vec![2, 6]), - raw_us_privacy: Some("1YNN".to_owned()), - raw_ac_string: None, - gdpr_applies: true, - tcf: None, - gpp: None, - us_privacy: None, - expired: false, - gpc: false, - jurisdiction: Jurisdiction::Gdpr, - source: ConsentSource::Cookie, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::consent::jurisdiction::Jurisdiction; - use crate::consent::types::{ConsentContext, ConsentSource}; - - #[test] - fn entry_roundtrip() { - let ctx = make_test_context(); - let entry = entry_from_context(&ctx, 1_000_000); - let json = serde_json::to_string(&entry).expect("should serialize"); - let restored: KvConsentEntry = serde_json::from_str(&json).expect("should deserialize"); - - assert_eq!(restored.raw_tc_string, ctx.raw_tc_string); - assert_eq!(restored.raw_gpp_string, ctx.raw_gpp_string); - assert_eq!(restored.gpp_section_ids, ctx.gpp_section_ids); - assert_eq!(restored.raw_us_privacy, ctx.raw_us_privacy); - assert_eq!(restored.gdpr_applies, ctx.gdpr_applies); - assert_eq!(restored.gpc, ctx.gpc); - assert_eq!(restored.jurisdiction, "GDPR"); - assert_eq!(restored.stored_at_ds, 1_000_000); - } - - #[test] - fn kv_consent_entry_roundtrip_preserves_fp() { - let ctx = make_test_context(); - let fp = consent_fingerprint(&ctx); - let mut entry = entry_from_context(&ctx, 1_000_000); - entry.fp = fp.clone(); - let json = serde_json::to_string(&entry).expect("should serialize"); - let restored: KvConsentEntry = serde_json::from_str(&json).expect("should deserialize"); - - assert_eq!( - restored.fp, fp, - "should preserve fingerprint through roundtrip" - ); - } - - #[test] - fn entry_fits_in_2000_bytes() { - let ctx = make_test_context(); - let mut entry = entry_from_context(&ctx, 1_000_000); - entry.fp = consent_fingerprint(&ctx); - let json = serde_json::to_string(&entry).expect("should serialize"); - assert!( - json.len() <= 2000, - "entry JSON must fit in 2000 bytes, was {} bytes", - json.len() - ); - } - - #[test] - fn context_roundtrip_via_entry() { - let original = make_test_context(); - let entry = entry_from_context(&original, 1_000_000); - let restored = context_from_entry(&entry); - - assert_eq!(restored.raw_tc_string, original.raw_tc_string); - assert_eq!(restored.raw_gpp_string, original.raw_gpp_string); - assert_eq!(restored.raw_us_privacy, original.raw_us_privacy); - assert_eq!(restored.gdpr_applies, original.gdpr_applies); - assert_eq!(restored.gpc, original.gpc); - assert_eq!(restored.jurisdiction, original.jurisdiction); - assert_eq!(restored.source, ConsentSource::KvStore); - } - - #[test] - fn fingerprint_deterministic() { - let ctx = make_test_context(); - let fp1 = consent_fingerprint(&ctx); - let fp2 = consent_fingerprint(&ctx); - assert_eq!(fp1, fp2, "fingerprint should be deterministic"); - assert_eq!(fp1.len(), 16, "fingerprint should be 16 hex chars"); - } - - #[test] - fn fingerprint_changes_with_different_signals() { - let ctx1 = make_test_context(); - let mut ctx2 = make_test_context(); - ctx2.raw_tc_string = Some("DIFFERENT_TC_STRING".to_owned()); - - assert_ne!( - consent_fingerprint(&ctx1), - consent_fingerprint(&ctx2), - "different TC strings should produce different fingerprints" - ); - } - - #[test] - fn fingerprint_changes_with_gpc() { - let mut ctx1 = make_test_context(); - ctx1.gpc = false; - let mut ctx2 = make_test_context(); - ctx2.gpc = true; - - assert_ne!( - consent_fingerprint(&ctx1), - consent_fingerprint(&ctx2), - "different GPC values should produce different fingerprints" - ); - } - - #[test] - fn fingerprint_distinguishes_none_from_empty() { - let mut ctx_none = make_test_context(); - ctx_none.raw_tc_string = None; - let mut ctx_empty = make_test_context(); - ctx_empty.raw_tc_string = Some(String::new()); - - assert_ne!( - consent_fingerprint(&ctx_none), - consent_fingerprint(&ctx_empty), - "None vs empty string should produce different fingerprints" - ); - } - - #[test] - fn signals_from_entry_roundtrip() { - let ctx = make_test_context(); - let entry = entry_from_context(&ctx, 1_000_000); - let signals = signals_from_entry(&entry); - - assert_eq!(signals.raw_tc_string, ctx.raw_tc_string); - assert_eq!(signals.raw_gpp_string, ctx.raw_gpp_string); - assert_eq!(signals.raw_us_privacy, ctx.raw_us_privacy); - assert_eq!(signals.gpc, ctx.gpc); - // gpp_sid is serialized as "2,6" from the section IDs - assert_eq!(signals.raw_gpp_sid, Some("2,6".to_owned())); - } - - #[test] - fn parse_jurisdiction_roundtrip() { - assert_eq!(parse_jurisdiction("GDPR"), Jurisdiction::Gdpr); - assert_eq!( - parse_jurisdiction("US-CA"), - Jurisdiction::UsState("CA".to_owned()) - ); - assert_eq!( - parse_jurisdiction("non-regulated"), - Jurisdiction::NonRegulated - ); - assert_eq!(parse_jurisdiction("unknown"), Jurisdiction::Unknown); - assert_eq!( - parse_jurisdiction("something-else"), - Jurisdiction::Unknown, - "unrecognized jurisdiction should default to Unknown" - ); - } - - #[test] - fn empty_entry_serializes_compact() { - let ctx = ConsentContext::default(); - let entry = entry_from_context(&ctx, 0); - let json = serde_json::to_string(&entry).expect("should serialize"); - // With skip_serializing_if = "Option::is_none", omitted fields keep it small. - assert!( - !json.contains("raw_tc_string"), - "None fields should be omitted from JSON" - ); - } - - #[test] - fn entry_preserves_ac_string() { - let mut ctx = make_test_context(); - ctx.raw_ac_string = Some("2~1234.5678~dv.".to_owned()); - let entry = entry_from_context(&ctx, 0); - let restored = context_from_entry(&entry); - - assert_eq!( - restored.raw_ac_string, - Some("2~1234.5678~dv.".to_owned()), - "AC string should survive roundtrip" - ); - } -} - -#[cfg(test)] -mod new_api_tests { - use super::*; - use edgezero_core::key_value_store::NoopKvStore; - - fn noop() -> NoopKvStore { - NoopKvStore - } - - #[test] - fn load_returns_none_when_key_absent() { - let result = load_consent_from_kv(&noop(), "some-ec-id"); - assert!(result.is_none(), "should return None when key is absent"); - } - - #[test] - fn save_does_not_panic_with_noop_store() { - let ctx = make_test_context(); - save_consent_to_kv(&noop(), "some-ec-id", &ctx, 30); - } - - #[test] - fn delete_does_not_panic_with_noop_store() { - delete_consent_from_kv(&noop(), "some-ec-id"); - } - - #[test] - fn kv_consent_entry_missing_fp_deserialises_as_empty() { - let json = r#"{"gdpr_applies":true,"gpc":false,"jurisdiction":"GDPR","stored_at_ds":0}"#; - let entry: KvConsentEntry = - serde_json::from_str(json).expect("should deserialize legacy entry"); - assert_eq!( - entry.fp, - String::new(), - "should default fp to empty string for legacy entries" - ); - } -} diff --git a/crates/trusted-server-core/src/storage/mod.rs b/crates/trusted-server-core/src/storage/mod.rs index ed5ff1ff..62c42ba1 100644 --- a/crates/trusted-server-core/src/storage/mod.rs +++ b/crates/trusted-server-core/src/storage/mod.rs @@ -7,11 +7,8 @@ //! [`crate::platform::PlatformSecretStore`], and the management write methods //! via [`crate::platform::RuntimeServices`]. -pub(crate) mod api_client; pub(crate) mod config_store; -pub mod kv_store; pub(crate) mod secret_store; -pub use api_client::FastlyApiClient; pub use config_store::FastlyConfigStore; pub use secret_store::FastlySecretStore; diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index ec5f8ddf..ccb3b25b 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -5,19 +5,6 @@ //! - Pluggable content processors (text replacement, HTML rewriting, etc.) //! - Memory-efficient streaming //! - UTF-8 boundary handling -//! -//! # Platform notes -//! -//! This module is **platform-agnostic** (verified 2026-03-31; see -//! `docs/superpowers/plans/2026-03-31-pr8-content-rewriting-verification.md`). It has zero -//! `fastly` imports. [`StreamingPipeline::process`] is generic over -//! `R: Read + W: Write` — any reader or writer works, including -//! `fastly::Body` (which implements `std::io::Read`) or standard -//! `std::io::Cursor<&[u8]>`. -//! -//! Future adapters (Cloudflare Workers, Axum, Spin) do not need to implement any compression or -//! streaming interface. See `crate::platform` module doc for the -//! authoritative note. use std::cell::RefCell; use std::io::{self, Read, Write}; diff --git a/crates/trusted-server-core/src/streaming_replacer.rs b/crates/trusted-server-core/src/streaming_replacer.rs index 1e7291e5..faf8f9a2 100644 --- a/crates/trusted-server-core/src/streaming_replacer.rs +++ b/crates/trusted-server-core/src/streaming_replacer.rs @@ -2,8 +2,6 @@ //! //! This module provides functionality for replacing patterns in content //! in streaming fashion, handling content that may be split across multiple chunks. -//! -//! See [`crate::platform`] module doc for platform notes. // Note: std::io::{Read, Write} were previously used by stream_process function // which has been removed in favor of StreamingPipeline diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 8fdfaa85..39432755 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -15,7 +15,7 @@ pub mod tests { password = "pass" [[handlers]] - path = "^/admin" + path = "^/_ts/admin" username = "admin" password = "admin-pass" @@ -34,8 +34,8 @@ pub mod tests { enabled = false rewrite_attributes = ["href", "link", "url"] - [edge_cookie] - secret_key = "test-secret-key" + [ec] + passphrase = "test-secret-key-32-bytes-minimum" [request_signing] config_store_id = "test-config-store-id" secret_store_id = "test-secret-store-id" diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index d66a820c..36ed84c5 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -78,6 +78,7 @@ export default withMermaid( text: 'Core Concepts', items: [ { text: 'Edge Cookies', link: '/guide/edge-cookies' }, + { text: 'EC Setup Guide', link: '/guide/ec-setup-guide' }, { text: 'GDPR Compliance', link: '/guide/gdpr-compliance' }, { text: 'Ad Serving', link: '/guide/ad-serving' }, { diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 880efc83..a44f202b 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -5,6 +5,7 @@ Quick reference for all Trusted Server HTTP endpoints. ## Endpoint Categories - [First-Party Endpoints](#first-party-endpoints) - Core ad serving and proxying +- [Edge Cookie Endpoints](#edge-cookie-endpoints) - Identity sync and enrichment - [Request Signing](#request-signing-endpoints) - Cryptographic signing and key management - [TSJS Library](#tsjs-library-endpoint) - JavaScript library serving - [Integration Endpoints](#integration-endpoints) - Third-party service proxying @@ -47,6 +48,72 @@ curl "https://edge.example.com/first-party/ad?slot=header-banner&w=728&h=90" --- +## Edge Cookie Endpoints + +Partners are configured statically in `[[ec.partners]]` and loaded into an in-memory registry at startup. There is no runtime partner-registration endpoint and the legacy browser pixel sync endpoint has been removed; browser-resolved IDs are ingested through Prebid EID cookies. + +--- + +### GET /\_ts/api/v1/identify + +Returns EC identity plus the authenticated partner's UID and EID for the current user. + +**Auth:** Bearer token (`Authorization: Bearer `) + +**Request:** + +- Uses `ts-ec` cookie and consent signals + +**Response (example):** + +```json +{ + "ec": "954d...e0c3.nZ1GxL", + "consent": "ok", + "degraded": false, + "partner_id": "mocktioneer", + "uid": "mock-user-123", + "eid": { + "source": "formally-vital-lion.edgecompute.app", + "uids": [{ "id": "mock-user-123", "atype": 3 }] + } +} +``` + +--- + +### POST /\_ts/api/v1/batch-sync + +Server-to-server batch sync endpoint for writing EC ID to partner UID mappings. Mapping timestamps are retained in the request schema for compatibility, but they no longer order writes because EC identity entries do not store per-partner sync timestamps. Valid mappings use idempotent last-write-wins semantics. + +**Auth:** Bearer token (`Authorization: Bearer `) + +**Request Body:** + +```json +{ + "mappings": [ + { + "ec_id": "954d8e7398dd993f78e3875ca1ef7841249781240e913157c1f2d6a6c960e0c3.nZ1GxL", + "partner_uid": "mock-user-123", + "timestamp": 1775147300 + } + ] +} +``` + +**Response:** + +```json +{ + "accepted": 1, + "rejected": 0, + "errors": [] +} +``` + +--- + ### POST /third-party/ad Client-side auction endpoint for TSJS library. @@ -329,7 +396,7 @@ curl -X POST https://edge.example.com/verify-signature \ --- -### POST /admin/keys/rotate +### POST /\_ts/admin/keys/rotate Generates and activates a new signing key. @@ -359,7 +426,7 @@ If omitted, auto-generates date-based ID (e.g., `ts-2025-01-15-A`). **Example:** ```bash -curl -X POST https://edge.example.com/admin/keys/rotate \ +curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ -u admin:password \ -H "Content-Type: application/json" ``` @@ -374,7 +441,7 @@ See [Key Rotation Guide](./key-rotation.md) for workflow details. --- -### POST /admin/keys/deactivate +### POST /\_ts/admin/keys/deactivate Deactivates or deletes a signing key. @@ -407,7 +474,7 @@ Deactivates or deletes a signing key. **Example:** ```bash -curl -X POST https://edge.example.com/admin/keys/deactivate \ +curl -X POST https://edge.example.com/_ts/admin/keys/deactivate \ -u admin:password \ -H "Content-Type: application/json" \ -d '{"kid":"ts-2025-01-14-A","delete":true}' @@ -588,7 +655,7 @@ Endpoints under protected paths require HTTP Basic Authentication: ```toml [[handlers]] -path = "^/admin" +path = "^/_ts/admin" username = "admin" password = "secure-password" ``` @@ -596,13 +663,13 @@ password = "secure-password" **Usage:** ```bash -curl -u admin:secure-password https://edge.example.com/admin/keys/rotate +curl -u admin:secure-password https://edge.example.com/_ts/admin/keys/rotate ``` **Protected Endpoints:** -- `/admin/keys/rotate` -- `/admin/keys/deactivate` +- `/_ts/admin/keys/rotate` +- `/_ts/admin/keys/deactivate` - Any paths matching configured `handlers` patterns --- diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 029163bb..1047ef2b 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -23,8 +23,8 @@ cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "your-secure-secret-here" -[edge_cookie] -secret_key = "your-hmac-secret" +[ec] +passphrase = "replace-with-32-plus-byte-random-secret" ``` ### Environment Variable Overrides @@ -37,7 +37,7 @@ at runtime. # Format: TRUSTED_SERVER__SECTION__FIELD export TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com -export TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=your-secret +export TRUSTED_SERVER__EC__PASSPHRASE=replace-with-32-plus-byte-random-secret ``` ### Generate Secure Secrets @@ -60,7 +60,7 @@ openssl rand -base64 32 | Section | Purpose | | ------------------- | -------------------------------------------- | | `[publisher]` | Domain, origin, proxy settings | -| `[edge_cookie]` | Edge Cookie (EC) ID generation | +| `[ec]` | Edge Cookie (EC) ID generation | | `[proxy]` | Proxy SSRF allowlist | | `[request_signing]` | Ed25519 request signing | | `[auction]` | Auction orchestration | @@ -75,8 +75,8 @@ cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "change-me-to-secure-value" -[edge_cookie] -secret_key = "your-hmac-secret-key" +[ec] +passphrase = "replace-with-32-plus-byte-random-secret" [request_signing] enabled = true @@ -153,19 +153,22 @@ Core publisher settings for domain, origin, and proxy configuration. ### `[publisher]` -| Field | Type | Required | Description | -| --------------- | ------ | -------- | ------------------------------------------------------- | -| `domain` | String | Yes | Publisher's domain name | -| `cookie_domain` | String | Yes | Domain for setting cookies (typically with leading dot) | -| `origin_url` | String | Yes | Full URL of publisher origin server | -| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | +| Field | Type | Required | Description | +| --------------- | ------ | -------- | ------------------------------------------------------ | +| `domain` | String | Yes | Publisher's apex domain name | +| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | +| `origin_url` | String | Yes | Full URL of publisher origin server | +| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | + +> **Note:** EC cookies (`ts-ec`) derive their domain automatically as `.{domain}` and +> do not use `cookie_domain`. The `cookie_domain` field is used by other cookie helpers. **Example**: ```toml [publisher] domain = "publisher.com" -cookie_domain = ".publisher.com" # Includes subdomains +cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "change-me-to-secure-random-value" ``` @@ -199,12 +202,12 @@ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=your-secret-here #### `cookie_domain` -**Purpose**: Domain scope for EC cookies. +**Purpose**: Domain scope for non-EC cookies. **Usage**: -- Set on `ts-ec` cookie -- Controls cookie sharing across subdomains +- Used by non-EC cookie helpers for domain scoping +- EC cookies (`ts-ec`) use a separate computed domain derived from `domain` **Format**: Domain with optional leading dot @@ -263,32 +266,46 @@ Changing `proxy_secret` invalidates all existing signed URLs. Plan rotations car ## EC Configuration -Settings for generating privacy-preserving Edge Cookie identifiers. +Settings for generating privacy-preserving Edge Cookie identifiers. The `ec_store` KV store is the only KV-backed EC lifecycle store; it holds identity graph state, minimal consent metadata, partner IDs, and withdrawal tombstones. Consent configuration controls request-local interpretation and forwarding, not separate KV persistence. -### `[edge_cookie]` +### `[ec]` -| Field | Type | Required | Description | -| ------------ | ------ | -------- | ----------------------------- | -| `secret_key` | String | Yes | HMAC secret for ID generation | +| Field | Type | Required | Description | +| ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | +| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | +| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | +| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | +| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | +| `partners` | Array | No | Static partner registry entries | **Example**: ```toml -[edge_cookie] -secret_key = "your-secure-hmac-secret" +[ec] +passphrase = "replace-with-32-plus-byte-random-secret" +ec_store = "ec_identity_store" + +[[ec.partners]] +id = "mocktioneer" +name = "Mocktioneer SSP" +source_domain = "mocktioneer.example" +api_token = "partner-api-token" +bidstream_enabled = true ``` **Environment Override**: ```bash -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=your-secret +TRUSTED_SERVER__EC__PASSPHRASE=your-secret +TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ``` ### Field Details -#### `secret_key` +#### `passphrase` -**Purpose**: HMAC secret for EC ID base generation. +**Purpose**: Publisher passphrase used as HMAC key for EC ID generation. **Security**: @@ -436,7 +453,7 @@ Path-based HTTP Basic Authentication. ```toml # Single handler [[handlers]] -path = "^/admin" +path = "^/_ts/admin" username = "admin" password = "secure-password" @@ -456,7 +473,7 @@ password = "api-pass" ```bash # Handler 0 -TRUSTED_SERVER__HANDLERS__0__PATH="^/admin" +TRUSTED_SERVER__HANDLERS__0__PATH="^/_ts/admin" TRUSTED_SERVER__HANDLERS__0__USERNAME="admin" TRUSTED_SERVER__HANDLERS__0__PASSWORD="secure-password" @@ -474,10 +491,10 @@ TRUSTED_SERVER__HANDLERS__1__PASSWORD="api-pass" ```toml # Exact path -path = "^/admin$" # Only /admin +path = "^/_ts/admin$" # Only /_ts/admin # Prefix match -path = "^/admin" # /admin, /admin/users, /admin/settings +path = "^/_ts/admin" # /_ts/admin, /_ts/admin/users, /_ts/admin/settings # Multiple paths path = "^/(admin|secure|private)" @@ -886,8 +903,8 @@ Configuration is validated at startup: **EC Validation**: -- `secret_key` ≥ 1 character -- `secret_key` ≠ known placeholders (`"secret-key"`, `"secret_key"`, `"trusted-server"` — case-insensitive) +- `passphrase` ≥ 1 character +- `passphrase` ≠ known placeholders (`"secret-key"`, `"secret_key"`, `"trusted-server"` — case-insensitive) **Handler Validation**: @@ -944,7 +961,7 @@ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret_staging) ```bash # All secrets from environment TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret) -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=$(cat /run/secrets/ec_secret) +TRUSTED_SERVER__EC__PASSPHRASE=$(cat /run/secrets/ec_secret) TRUSTED_SERVER__HANDLERS__0__PASSWORD=$(cat /run/secrets/admin_password) ``` @@ -998,7 +1015,7 @@ trusted-server.dev.toml # Development overrides **"Configuration field '...' is set to a known placeholder value"**: -- `edge_cookie.secret_key` cannot be `"secret-key"`, `"secret_key"`, or `"trusted-server"` (case-insensitive) +- `ec.passphrase` cannot be `"secret-key"`, `"secret_key"`, or `"trusted-server"` (case-insensitive) - `publisher.proxy_secret` cannot be `"change-me-proxy-secret"` (case-insensitive) - Must be non-empty - Change to a secure random value (see generation commands above) @@ -1006,7 +1023,7 @@ trusted-server.dev.toml # Development overrides **"Invalid regex"**: - Handler `path` must be valid regex -- Test pattern: `echo "^/admin" | grep -E "^/admin"` +- Test pattern: `echo "^/_ts/admin" | grep -E "^/_ts/admin"` - Escape special characters: `\.`, `\$`, etc. **"Integration configuration could not be parsed"**: diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md new file mode 100644 index 00000000..7dba9a06 --- /dev/null +++ b/docs/guide/ec-setup-guide.md @@ -0,0 +1,212 @@ +# Edge Cookie Setup Guide + +End-to-end setup and verification guide for Edge Cookie (EC) identity flows. + +This guide covers: + +1. Fastly store setup +2. Partner configuration +3. Server-to-server batch sync (`/_ts/api/v1/batch-sync`) +4. Identity verification (`/_ts/api/v1/identify`) +5. Auction bidstream verification (`/auction`) + +## Prerequisites + +- Trusted Server deployed and reachable (example: `https://getpurpose.ai`) +- Access to update `trusted-server.toml` / deployment configuration +- Fastly CLI authenticated (for store verification) +- A valid TCF consent string (`euconsent-v2`) for consent-required requests + +## 1) Required Configuration + +Set EC configuration in `trusted-server.toml`: + +```toml +[ec] +passphrase = "replace-with-32-plus-byte-random-secret" +ec_store = "ec_identity_store" + +[[ec.partners]] +id = "mocktioneer" +name = "Mocktioneer SSP" +source_domain = "formally-vital-lion.edgecompute.app" +api_token = "test-batch-sync-key-2026" +bidstream_enabled = true +``` + +Required behavior assumptions: + +- `passphrase` is long-lived HMAC-SHA256 keying material for EC ID derivation; use a high-entropy random value of at least 32 characters +- `ec_store` is linked to the active Fastly service version +- `ec_store` is the only KV-backed EC lifecycle store; it contains identity graph state, minimal consent metadata, partner IDs, and withdrawal tombstones +- Live consent is interpreted from request cookies, headers, geolocation, and policy defaults rather than a separate consent KV store +- Partners are configured statically in `[[ec.partners]]` and loaded into an in-memory registry at startup +- Partner has `bidstream_enabled = true` if you want `user.ext.eids` in bidstream + +## 2) Configure Demo Variables + +```bash +TS_BASE_URL="https://getpurpose.ai" +MOCK_SSP_URL="https://formally-vital-lion.edgecompute.app" + +PARTNER_ID="mocktioneer" +PARTNER_NAME="Mocktioneer SSP" +PARTNER_API_KEY="test-batch-sync-key-2026" + +# Optional: use a real browser EC if already present +EC_ID="<64hex.6chars>" + +TCF_CONSENT="" +PARTNER_UID="mock-user-$(date +%s)" +``` + +## 3) Configure Partner + +Partners are configured in `trusted-server.toml` and loaded at startup: + +```toml +[[ec.partners]] +id = "mocktioneer" +name = "Mocktioneer SSP" +source_domain = "formally-vital-lion.edgecompute.app" +api_token = "test-batch-sync-key-2026" +bidstream_enabled = true +``` + +Deploy/restart after changing partner configuration. + +## 4) Acquire or Reuse EC Cookie + +If you already have an EC from browser traffic, reuse it. + +Otherwise, attempt generation with consent: + +```bash +curl -si "${TS_BASE_URL}/" \ + -H "Cookie: euconsent-v2=${TCF_CONSENT}" +``` + +Look for: + +- `Set-Cookie: ts-ec=<64hex.6chars>` + +## 5) Batch Sync (S2S) + +Endpoint: `POST /_ts/api/v1/batch-sync` + +Important: request field is `ec_id` (full `{64hex}.{6alnum}` value). The `timestamp` field remains required for API compatibility, but it no longer orders writes because EC identity entries do not store per-partner sync timestamps. Valid mappings are idempotent last-write-wins: unchanged UIDs are accepted without a write, and different UIDs replace the stored value. + +```bash +BATCH_UID="${PARTNER_UID}-batch" +NOW_TS="$(date +%s)" + +curl -X POST "${TS_BASE_URL}/_ts/api/v1/batch-sync" \ + -H "Authorization: Bearer ${PARTNER_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{ + \"mappings\": [{ + \"ec_id\": \"${EC_ID}\", + \"partner_uid\": \"${BATCH_UID}\", + \"timestamp\": ${NOW_TS} + }] + }" | python3 -m json.tool +``` + +Expected: + +```json +{ + "accepted": 1, + "rejected": 0, + "errors": [] +} +``` + +## 6) Verify Identity + +Endpoint: `GET /_ts/api/v1/identify` + +```bash +curl -s "${TS_BASE_URL}/_ts/api/v1/identify" \ + -H "Authorization: Bearer ${PARTNER_API_KEY}" \ + -H "Cookie: ts-ec=${EC_ID}; euconsent-v2=${TCF_CONSENT}" | python3 -m json.tool +``` + +Expected shape: + +```json +{ + "ec": "", + "consent": "ok", + "degraded": false, + "partner_id": "mocktioneer", + "uid": "mock-user-123", + "eid": { + "source": "formally-vital-lion.edgecompute.app", + "uids": [{ "id": "mock-user-123", "atype": 3 }] + }, + "cluster_size": 12 +} +``` + +## 7) Verify Auction Bidstream Enrichment + +Endpoint: `POST /auction` + +```bash +curl -si -X POST "${TS_BASE_URL}/auction" \ + -H "Cookie: ts-ec=${EC_ID}; euconsent-v2=${TCF_CONSENT}" \ + -H "Content-Type: application/json" \ + -d '{"adUnits":[{"code":"test","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}' +``` + +Check response headers: + +- `x-ts-ec` +- `x-ts-ec-consent` +- `x-ts-eids` + For returning users, ordinary page views should include `x-ts-ec` but should not refresh `Set-Cookie: ts-ec=...`. A `Set-Cookie` header is expected when the EC is newly generated. + +Decode `x-ts-eids`: + +```bash +echo "" | base64 -d | python3 -m json.tool +``` + +Expected decoded payload contains: + +- `source = formally-vital-lion.edgecompute.app` +- `uids[0].id = ` + +## 8) Fastly KV Operational Checks + +List stores: + +```bash +fastly kv-store list +``` + +Check service resource links for active version: + +```bash +fastly resource-link list --service-id --version +``` + +Inspect EC identity entry: + +```bash +fastly kv-store-entry get --store-id --key "${EC_ID}" +``` + +If batch sync returns `ineligible`, check whether the KV entry is missing or has `consent.ok = false` from a withdrawal tombstone. + +## 9) Troubleshooting Quick Map + +| Symptom | Likely Cause | Check | +| ----------------------------------------------------- | ------------------------------ | --------------------------------------------------------------------------- | +| `invalid_token` on batch sync | Wrong partner API key | Re-register partner with known API key | +| `missing field ec_id` | Wrong request schema | Use `ec_id` field | +| `/_ts/api/v1/identify` returns `{"consent":"denied"}` | No consent for current request | Send consent cookie | +| No `uid` in `/_ts/api/v1/identify` | No successful sync yet | Run batch sync or ensure Prebid EID ingestion has populated the partner UID | + +See also: [Edge Cookies](/guide/edge-cookies), [Configuration](/guide/configuration), [API Reference](/guide/api-reference) diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index d0e31e2e..7f3f1dad 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -8,6 +8,8 @@ Edge Cookies (EC) are privacy-safe identifiers generated on a first site visit u Trusted Server surfaces the current EC ID via response headers and a first-party cookie. For the exact header and cookie names, see the [API Reference](/guide/api-reference). +For full operational onboarding (partner configuration, batch sync, identify, and auction verification), use the [EC Setup Guide](/guide/ec-setup-guide). + ## How They Work ### HMAC-Based Generation @@ -18,9 +20,144 @@ EC IDs use HMAC (Hash-based Message Authentication Code) to generate a determini **IP normalization**: IPv6 addresses are normalized to a /64 prefix before hashing. +### Request Lifecycle + +Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction) — read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. + +```mermaid +sequenceDiagram + participant B as Browser + participant TS as Trusted Server + participant KV as KV Store + + B->>TS: Request (ts-ec cookie + consent signals) + Note over TS: Phase 1: Pre-routing
Read EC from cookie/header
Build consent context
Extract device signals + + alt First Visit (no EC cookie) + Note over TS: Phase 2: Routing (organic only)
generate_if_needed() + TS->>TS: HMAC-SHA256(IP) + random suffix + TS->>KV: Create entry (consent, geo, device) + Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies + TS-->>B: Response + Set-Cookie: ts-ec=... + else Return Visit (EC cookie present) + Note over TS: Phase 2: Routing
EC exists — skip generation + Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies + TS-->>B: Response + x-ts-ec header
(no cookie refresh) + end + + Note over TS,KV: Phase 4: Post-send (background)
Dispatch pull-sync to partners +``` + +### Response Finalization + +After routing completes, the server evaluates consent state and cookie presence to decide what to do with the EC cookie on the response. + +```mermaid +flowchart TD + Start[ec_finalize_response] --> ConsentCheck{Consent
allows EC?} + + ConsentCheck -- "No" --> ExplicitWithdrawal{Explicit
withdrawal?} + ExplicitWithdrawal -- "Yes" --> CookiePresent{Cookie was
present?} + CookiePresent -- "Yes" --> Withdraw["Expire ts-ec cookie
Write withdrawal tombstone in ec_identity_store (24h TTL)
Strip all x-ts-* headers"] + CookiePresent -- "No" --> HeaderOnly["Strip all x-ts-* headers only
(no cookie expiry or KV tombstone)"] + ExplicitWithdrawal -- "No" --> HeaderOnly + + ConsentCheck -- "Yes" --> WasPresent{EC was present
in request?} + WasPresent -- "Yes, not generated" --> Returning["Ingest Prebid EID cookies
Set x-ts-ec header only
(no cookie or KV TTL refresh)"] + WasPresent -- "No, just generated" --> NewEc["Ingest Prebid EID cookies
Set ts-ec cookie + x-ts-ec header"] +``` + +When consent cannot be verified for the current request — for example, unknown jurisdiction or missing/undecodable consent signals in a regulated region — Trusted Server fails closed for EC use by stripping EC headers, but it does **not** treat that as authoritative revocation of an already-issued EC. + +## Consent Model + +EC creation is gated by jurisdiction. The server detects jurisdiction from geolocation data attached to the request and applies the appropriate consent framework. Live consent comes from request-local signals (`euconsent-v2`, `__gpp`, `__gpp_sid`, `us_privacy`, `Sec-GPC`) plus geolocation and policy defaults; there is no separate consent KV fallback. + +```mermaid +flowchart TD + Start[Detect Jurisdiction] --> J{Jurisdiction?} + + J -- "GDPR
(EU/UK)" --> TCF{TCF string
present?} + TCF -- "Yes" --> P1{Purpose 1
granted?} + P1 -- "Yes" --> Allow([Allow EC]) + P1 -- "No" --> Deny([Deny EC]) + TCF -- "No" --> Deny + + J -- "US State" --> GPC{GPC header
set?} + GPC -- "Yes" --> Deny + GPC -- "No" --> USTCF{TCF from CMP
e.g. Didomi?} + USTCF -- "Yes" --> USP1{Purpose 1
granted?} + USP1 -- "Yes" --> Allow + USP1 -- "No" --> Deny + USTCF -- "No" --> USP{US Privacy
string?} + USP -- "Yes" --> OptOut{Opt-out
sale?} + OptOut -- "No" --> Allow + OptOut -- "Yes" --> Deny + USP -- "No" --> Deny + + J -- "Non-regulated" --> Allow + J -- "Unknown
(no geo data)" --> Deny +``` + +- **GDPR**: Opt-in required. TCF Purpose 1 (store/access device) must be explicitly consented. +- **US State**: Opt-out model with three-tier fallback — GPC always blocks, then TCF if a CMP uses it, then US Privacy string, then fail-closed. +- **Non-regulated**: EC always allowed. +- **Unknown**: Fail-closed when jurisdiction cannot be determined. + +The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, partner IDs, a minimal consent snapshot used for EC entry metadata, and withdrawal tombstones. Consent interpretation for each request remains based on the live request signals listed above. + +## Partner Sync Channels + +Partner identities flow into the KV identity graph through three channels. Each writes to the same `ids` map in the KV entry via idempotent upsert logic: unchanged UIDs are accepted without a KV write, while different UIDs replace the stored value. + +```mermaid +flowchart LR + subgraph Browser-initiated + Prebid["Prebid EID Cookies
ts-eids + sharedId
Passive cookie ingestion"] + end + + subgraph Server-initiated + Batch["Batch Sync (S2S)
POST /_ts/api/v1/batch-sync
Partner POST + Bearer auth"] + Pull["Pull Sync (Background)
TS calls partner URL
Post-send on organic routes"] + end + + Prebid --> KV[(KV Identity Graph
ids map)] + Batch --> KV + Pull --> KV +``` + +### Prebid EID Cookie Flow + +The `ts-eids` cookie bridges client-side Prebid user ID modules with the server-side identity graph. + +```mermaid +sequenceDiagram + participant Prebid as Prebid.js + participant TSJS as TSJS Prebid Module + participant B as Browser Cookie Jar + participant TS as Trusted Server + participant KV as KV Store + + Prebid->>Prebid: Auction completes + Prebid->>TSJS: bidsBackHandler fires + TSJS->>Prebid: getUserIdsAsEids() + Prebid-->>TSJS: [{source, uids: [{id, atype}]}] + TSJS->>TSJS: Base64 encode full OpenRTB-style EID array
[{source, uids:[{id, atype, ext?}]}] + TSJS->>B: document.cookie = "ts-eids=..." + + Note over B,TS: Next page request + B->>TS: Request with ts-eids cookie + TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners + TS->>KV: upsert_partner_id() per match
(skips write when UID unchanged) +``` + +Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shape in `ts-eids`. The server remains backward-compatible with earlier flattened `{source, id, atype}` cookies during rollout, but new cookies use the structured `uids[]` form. + +The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. + ## Configuration -Configure EC secrets in `trusted-server.toml`. See the full [Configuration Reference](/guide/configuration) for the `[edge_cookie]` section and environment variable overrides. +Configure EC settings in `trusted-server.toml`. See the full [Configuration Reference](/guide/configuration) for the `[ec]` section and environment variable overrides. ## Privacy Considerations @@ -35,8 +172,18 @@ Configure EC secrets in `trusted-server.toml`. See the full [Configuration Refer 2. Rotate secret keys periodically 3. Monitor ID collision rates +## Runtime Behavior Notes + +- Returning requests with consent and an existing `ts-ec` receive an `x-ts-ec` response header only; ordinary page views do not refresh the EC cookie or KV TTL. +- Newly generated ECs receive both `Set-Cookie: ts-ec=...` and `x-ts-ec`. +- When consent is blocked but not explicitly withdrawn, Trusted Server strips EC response headers for that request but leaves any existing `ts-ec` cookie intact; cookie expiry and tombstones happen only on explicit withdrawal. +- `/_ts/api/v1/identify` is read-oriented and returns identity enrichment for the authenticated partner. It computes `cluster_size` only when the EC entry does not already store one. +- `/_ts/api/v1/batch-sync` writes mappings into the EC identity graph. Mapping timestamps are retained for API compatibility but no longer order writes; valid mappings use idempotent last-write-wins semantics. +- Pull sync fills missing partner UIDs only. Existing partner UIDs are not periodically refreshed because EC entries no longer store per-partner sync timestamps. + ## Next Steps +- Follow the [EC Setup Guide](/guide/ec-setup-guide) - Learn about [GDPR Compliance](/guide/gdpr-compliance) - Configure [Ad Serving](/guide/ad-serving) - Learn about [Collective Sync](/guide/collective-sync) for cross-publisher data sharing details and diagrams diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index 99f611aa..7d244bb7 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -69,7 +69,7 @@ proxy_secret = "change-me-to-random-string" - `publisher.domain` - `publisher.origin_url` - `publisher.proxy_secret` -- `edge_cookie.secret_key` +- `ec.passphrase` --- @@ -141,17 +141,17 @@ Failed to generate EC ID: HMAC error **Solution:** -1. Ensure `secret_key` is set in `trusted-server.toml`: +1. Ensure `passphrase` is set in `trusted-server.toml`: ```toml -[edge_cookie] -secret_key = "your-secure-hmac-secret" +[ec] +passphrase = "replace-with-32-plus-byte-random-secret" ``` 2. Or set via environment variable: ```bash -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=your-secure-hmac-secret +TRUSTED_SERVER__EC__PASSPHRASE=replace-with-32-plus-byte-random-secret ``` --- @@ -245,7 +245,9 @@ curl -w "%{time_total}\n" https://upstream-service.example.com Warning: Cookie not set due to domain mismatch ``` -**Cause:** `publisher.cookie_domain` doesn't match request domain +**Cause:** `publisher.cookie_domain` doesn't match request domain. +Note: EC cookies (`ts-ec`) use a computed domain from `publisher.domain`, +not `cookie_domain`. **Solution:** @@ -402,7 +404,7 @@ Signing key not found: ts-2025-01-A 3. Run key rotation to generate new key: ```bash -curl -X POST https://edge.example.com/admin/keys/rotate \ +curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ -u admin:password ``` @@ -425,7 +427,7 @@ curl -X POST https://edge.example.com/admin/keys/rotate \ 1. Initialize keys using rotation endpoint: ```bash -curl -X POST https://edge.example.com/admin/keys/rotate \ +curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ -u admin:password ``` diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index 2a1edd79..4ca1aa24 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -94,8 +94,45 @@ fastly secret-store create --name signing_keys Note the store IDs - you'll need them for your `trusted-server.toml` configuration. +## Create EC KV Store + +Edge Cookie flows require one KV store: + +- Identity graph store (`ec_store`) - EC identity graph, partner IDs, minimal consent metadata, and withdrawal tombstones + +Partners are configured statically in `[[ec.partners]]` and loaded into an in-memory registry at startup. There is no separate consent KV store. Consent is interpreted from live request cookies, headers, geolocation, and policy defaults. + +Create it: + +```bash +fastly kv-store create --name ec_identity_store +``` + +Configure in `trusted-server.toml`: + +```toml +[ec] +passphrase = "replace-with-32-plus-byte-random-secret"}]}},{ +ec_store = "ec_identity_store" +``` + +Verify stores exist: + +```bash +fastly kv-store list +``` + +Verify stores are linked to your active service version: + +```bash +fastly resource-link list --service-id --version +``` + +If EC sync returns `kv_unavailable` or identify responses are degraded, first check that the identity store is present and linked to the active version. Legacy partner/consent KV bindings can be removed once no deployment-specific tooling depends on them. + ## Next Steps - Return to [Getting Started](/guide/getting-started) to continue setup - See [Configuration](/guide/configuration) for detailed configuration options +- See [EC Setup Guide](/guide/ec-setup-guide) for end-to-end EC verification - See [Request Signing](/guide/request-signing) for setting up cryptographic signing diff --git a/docs/guide/first-party-proxy.md b/docs/guide/first-party-proxy.md index b978e35f..a2567c51 100644 --- a/docs/guide/first-party-proxy.md +++ b/docs/guide/first-party-proxy.md @@ -420,9 +420,9 @@ Configure proxy behavior in `trusted-server.toml`: ```toml [publisher] domain = "publisher.com" +cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" proxy_secret = "your-secure-random-secret" -cookie_domain = ".publisher.com" # For ts-ec cookies ``` ### Proxy Allowlist diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index ce4b328d..a4ed418c 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -72,5 +72,6 @@ fastly compute publish ## Next Steps - Learn about [Edge Cookies](/guide/edge-cookies) +- Follow the [EC Setup Guide](/guide/ec-setup-guide) - Understand [GDPR Compliance](/guide/gdpr-compliance) - Configure [Ad Serving](/guide/ad-serving) diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index fb8c99da..79576b8b 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -328,6 +328,16 @@ When the integration is enabled, the `IntegrationAttributeRewriter` removes any The NPM integration lives in `crates/js/lib/src/integrations/prebid/index.ts`. Tests typically assert that publisher references disappear and the deferred `tsjs-prebid.min.js` tag is present. +**5. Hybrid EID forwarding** + +For Prebid-routed auctions, Trusted Server now forwards identity using a hybrid model: + +- TSJS reads current-request EIDs from `pbjs.getUserIdsAsEids()` and includes them in the `/auction` payload. +- The edge resolves additional EIDs from the EC/KV identity graph. +- The auction handler merges and deduplicates both sets. +- The Prebid provider forwards the merged result to Prebid Server as `user.ext.eids`. +- The `ts-eids` cookie is still ingested after the response so future requests can benefit from those IDs even without fresh browser-side resolution. + Reusing these patterns makes it straightforward to convert additional legacy flows (for example, Next.js rewrites) into first-class integrations. ## Future Improvements diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index e9ba6fcb..9a9c24e7 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -223,6 +223,70 @@ The build script (`build-all.mjs`) validates that each adapter exists in `prebid Adding a new client-side bidder requires both a config change (`client_side_bidders`) **and** a rebuild with the adapter included in `TSJS_PREBID_ADAPTERS`. Without the adapter in the bundle, the bidder is silently dropped from both server-side and client-side auctions. ::: +## User ID Modules + +Prebid.js can expose publisher-configured User ID Module output via +`pbjs.getUserIdsAsEids()`. The TSJS Prebid shim reads those current-request +EIDs after auctions and forwards them to Trusted Server when they are available. + +Build-time configurable User ID submodule selection is not currently part of the +TSJS build pipeline. Do not rely on a `TSJS_PREBID_USER_IDS` environment +variable or generated `_user_ids.generated.ts` file for slim User ID builds. + +## Identity Forwarding + +Trusted Server uses a **hybrid EID forwarding model** for Prebid-routed auctions: + +1. **Current-request EIDs from Prebid.js** are read from `pbjs.getUserIdsAsEids()` in the browser and sent in the `/auction` request body. +2. **Server-side EIDs from the EC/KV identity graph** are resolved on the edge from the current EC ID. +3. Trusted Server **merges and deduplicates** both sets before calling Prebid Server. +4. The merged result is forwarded downstream as `user.ext.eids` in the OpenRTB request. +5. The `ts-eids` cookie is still ingested after the response so later requests can reuse the IDs even when the current auction does not provide them again. + +This means Prebid auctions get same-request transparency for browser-resolved IDs without giving up the durability of the server-managed EC identity graph. + +### Identity flow + +```mermaid +sequenceDiagram + participant B as Browser / Prebid.js + participant T as Trusted Server /auction + participant K as EC + KV identity graph + participant P as Prebid Server + + B->>B: User ID modules resolve EIDs + B->>T: POST /auction\n(adUnits + current-request eids) + T->>K: Resolve EC-backed partner IDs + K-->>T: KV-derived EIDs + T->>T: Merge + dedupe client + KV EIDs + T->>T: Apply consent gating + T->>P: OpenRTB request\nuser.ext.eids = merged set + P-->>T: OpenRTB bid response + T-->>B: Auction response + T->>K: Ingest ts-eids cookie for future requests +``` + +### Merge and deduplication rules + +- Client-request EIDs and KV-resolved EIDs are merged by `source` +- UIDs are deduplicated by `source + id` +- If the same UID appears in both places, it is sent only once downstream +- Distinct UIDs under the same source are preserved +- Consent gating is applied to the **merged** set before forwarding + +### What reaches Prebid Server + +The downstream Prebid Server request includes: + +- `user.id` when EC forwarding is allowed +- `user.ext.eids` containing the merged, deduplicated EID set +- forwarded browser cookies (subject to consent-forwarding mode) + +In practice, this gives operators both: + +- **same-request identity transparency** for Prebid User ID Module output, and +- **future-request continuity** through cookie ingestion and KV-backed partner resolution. + ## Endpoints ### GET /first-party/ad @@ -290,6 +354,7 @@ The `to_openrtb()` method in `PrebidAuctionProvider` builds OpenRTB requests: - Sets `tagid` from the slot ID - Adds site metadata with publisher domain, page URL, `site.ref` from the Referer header, and `site.publisher` from the domain - Injects EC ID in the user object +- Merges current-request browser EIDs with KV-resolved EIDs and forwards the deduplicated result as `user.ext.eids` - Forwards user consent string and sets the GDPR flag based on geo and consent presence - Translates the `Sec-GPC` header to a US Privacy string (`us_privacy`) - Extracts `DNT` and `Accept-Language` headers into device fields diff --git a/docs/guide/key-rotation.md b/docs/guide/key-rotation.md index d4467bc8..ec96d1ea 100644 --- a/docs/guide/key-rotation.md +++ b/docs/guide/key-rotation.md @@ -26,6 +26,10 @@ Key rotation is the process of generating new signing keys and transitioning fro - **Incident-based**: Immediately if compromise suspected - **Before major releases**: Ensure fresh keys for new deployments +## Edge Cookie HMAC Passphrase + +The Edge Cookie `ec.passphrase` is long-lived HMAC-SHA256 keying material used to derive visitor EC IDs. Use a high-entropy random value of at least 32 characters; shorter values are rejected at settings validation. Rotating this passphrase changes derived EC IDs and requires rebuilding or allowing expiry of the existing EC identity graph. + ## Prerequisites Before you can rotate keys, you need to set up the required Fastly stores and API credentials. @@ -240,14 +244,14 @@ You should see a JWKS response with your public keys. ### Using the Rotation Endpoint -**Endpoint**: `POST /admin/keys/rotate` +**Endpoint**: `POST /_ts/admin/keys/rotate` #### Automatic Key ID (Recommended) Let Trusted Server generate a date-based key ID: ```bash -curl -X POST https://your-domain/admin/keys/rotate \ +curl -X POST https://your-domain/_ts/admin/keys/rotate \ -H "Content-Type: application/json" \ -d '{}' ``` @@ -276,7 +280,7 @@ curl -X POST https://your-domain/admin/keys/rotate \ Specify a custom key identifier: ```bash -curl -X POST https://your-domain/admin/keys/rotate \ +curl -X POST https://your-domain/_ts/admin/keys/rotate \ -H "Content-Type: application/json" \ -d '{"kid": "production-2024-q1"}' ``` @@ -356,14 +360,14 @@ Deactivate old keys after: ### Deactivation Endpoint -**Endpoint**: `POST /admin/keys/deactivate` +**Endpoint**: `POST /_ts/admin/keys/deactivate` #### Deactivate (Keep in Storage) Remove from active rotation but keep in storage: ```bash -curl -X POST https://your-domain/admin/keys/deactivate \ +curl -X POST https://your-domain/_ts/admin/keys/deactivate \ -H "Content-Type: application/json" \ -d '{ "kid": "ts-2024-01-15", @@ -388,7 +392,7 @@ curl -X POST https://your-domain/admin/keys/deactivate \ Remove from storage completely: ```bash -curl -X POST https://your-domain/admin/keys/deactivate \ +curl -X POST https://your-domain/_ts/admin/keys/deactivate \ -H "Content-Type: application/json" \ -d '{ "kid": "ts-2024-01-15", @@ -476,14 +480,14 @@ Regular rotation on a fixed schedule: ```bash #!/bin/bash # Rotate signing keys -curl -X POST https://your-domain/admin/keys/rotate +curl -X POST https://your-domain/_ts/admin/keys/rotate # Wait 30 days grace period sleep $((30 * 24 * 60 * 60)) # Deactivate old key OLD_KEY=$(date -d '90 days ago' +ts-%Y-%m-%d) -curl -X POST https://your-domain/admin/keys/deactivate \ +curl -X POST https://your-domain/_ts/admin/keys/deactivate \ -d "{\"kid\": \"$OLD_KEY\", \"delete\": true}" ``` @@ -647,13 +651,13 @@ If a key is compromised: 1. **Immediate**: Rotate to new key ```bash -curl -X POST /admin/keys/rotate +curl -X POST /_ts/admin/keys/rotate ``` 2. **Urgent**: Deactivate compromised key ```bash -curl -X POST /admin/keys/deactivate \ +curl -X POST /_ts/admin/keys/deactivate \ -d '{"kid": "compromised-key", "delete": false}' ``` @@ -664,7 +668,7 @@ curl -X POST /admin/keys/deactivate \ 5. **Cleanup**: Delete compromised key after investigation ```bash -curl -X POST /admin/keys/deactivate \ +curl -X POST /_ts/admin/keys/deactivate \ -d '{"kid": "compromised-key", "delete": true}' ``` diff --git a/docs/guide/onboarding.md b/docs/guide/onboarding.md index 83203d35..5d4d5b32 100644 --- a/docs/guide/onboarding.md +++ b/docs/guide/onboarding.md @@ -40,7 +40,7 @@ Welcome to the Trusted Server project! This guide keeps internal onboarding note | `crates/trusted-server-adapter-fastly/src/main.rs` | Request routing entry point | | `crates/trusted-server-core/src/publisher.rs` | Publisher origin handling | | `crates/trusted-server-core/src/proxy.rs` | First-party proxy implementation | -| `crates/trusted-server-core/src/edge_cookie.rs` | EC ID generation | +| `crates/trusted-server-core/src/ec/` | EC identity subsystem | | `crates/trusted-server-core/src/integrations/registry.rs` | Integration module pattern | | `trusted-server.toml` | Application configuration | @@ -146,7 +146,7 @@ Use this checklist to track your onboarding progress: - [ ] Read through `main.rs` to understand request routing - [ ] Trace a request through `publisher.rs` and `proxy.rs` -- [ ] Understand EC ID generation in `edge_cookie.rs` +- [ ] Understand the EC identity subsystem in `ec/` - [ ] Review an existing integration (e.g., `prebid.rs`) ### Documentation & Contribution diff --git a/docs/guide/testing.md b/docs/guide/testing.md index cfa7ea51..fb12ac77 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -21,7 +21,7 @@ cargo test Tests are organized alongside source code in `#[cfg(test)]` modules: ```rust -// crates/trusted-server-core/src/edge_cookie.rs +// crates/trusted-server-core/src/ec/generation.rs #[cfg(test)] mod tests { use super::*; @@ -88,7 +88,7 @@ curl http://localhost:7676/.well-known/trusted-server.json ### EC ID Tests -From `crates/trusted-server-core/src/edge_cookie.rs`: +From `crates/trusted-server-core/src/ec/mod.rs`: ```rust #[test] diff --git a/docs/superpowers/plans/2026-03-31-pr9-wire-signing-to-store-primitives.md b/docs/superpowers/plans/2026-03-31-pr9-wire-signing-to-store-primitives.md new file mode 100644 index 00000000..c03de5fa --- /dev/null +++ b/docs/superpowers/plans/2026-03-31-pr9-wire-signing-to-store-primitives.md @@ -0,0 +1,1835 @@ +# PR 9: Wire Request-Signing to Platform Store Primitives + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove `api_client.rs` from `trusted-server-core`, move Fastly management API transport to the adapter as `management_api.rs`, and replace all direct `Fastly*Store` / `FastlyApiClient` usage in `request_signing/` with `RuntimeServices` store primitives. + +**Architecture:** Core `request_signing/` code calls platform-agnostic `services.config_store()` and `services.secret_store()` for all reads and writes. The Fastly adapter's `management_api.rs` absorbs the HTTP transport (calls to `api.fastly.com`) and backs the `put`/`delete`/`create` write methods in `FastlyPlatformConfigStore` and `FastlyPlatformSecretStore`. No signing-specific trait is introduced — adapters only implement store CRUD, and core owns all signing business logic. + +**Tech Stack:** Rust 2024 edition, `error-stack`, `derive_more::Display`, `fastly`, `ed25519-dalek`, `serde_json`, `urlencoding` + +--- + +## Background: What the Current Code Does + +Before touching anything, read these files to understand the current state: + +| File | Status | Notes | +| ------------------------------------------------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------ | +| `crates/trusted-server-core/src/storage/api_client.rs` | **Delete** | Contains `FastlyApiClient` — HTTP calls to `api.fastly.com`. Used only by `rotation.rs`. | +| `crates/trusted-server-core/src/request_signing/rotation.rs` | **Migrate** | Uses `FastlyConfigStore` (reads) + `FastlyApiClient` (writes). Main migration target. | +| `crates/trusted-server-core/src/request_signing/signing.rs` | **Migrate** | Uses `FastlyConfigStore` + `FastlySecretStore` in 3 places. | +| `crates/trusted-server-core/src/request_signing/endpoints.rs` | **Update** | `handle_verify_signature`, `handle_rotate_key`, `handle_deactivate_key` don't receive `&RuntimeServices`. | +| `crates/trusted-server-core/src/request_signing/jwks.rs` | Already migrated ✓ | Uses `RuntimeServices`. No changes needed. | +| `crates/trusted-server-adapter-fastly/src/platform.rs` | **Update** | `FastlyPlatformConfigStore::put/delete` and `FastlyPlatformSecretStore::create/delete` return `PlatformError::NotImplemented`. | +| `crates/trusted-server-adapter-fastly/src/main.rs` | **Update** | Three call sites pass handlers without `runtime_services`. | + +## File Map + +### Delete + +- `crates/trusted-server-core/src/storage/api_client.rs` + +### Modify (core) + +- `crates/trusted-server-core/src/storage/mod.rs` — remove `api_client` submodule + re-export +- `crates/trusted-server-core/src/platform/test_support.rs` — add `build_services_with_config_and_secret` +- `crates/trusted-server-core/src/request_signing/rotation.rs` — replace `FastlyConfigStore`/`FastlyApiClient` with `RuntimeServices` +- `crates/trusted-server-core/src/request_signing/signing.rs` — replace `FastlyConfigStore`/`FastlySecretStore` with `RuntimeServices` +- `crates/trusted-server-core/src/request_signing/endpoints.rs` — add `&RuntimeServices` to three handlers + +### Create (adapter) + +- `crates/trusted-server-adapter-fastly/src/management_api.rs` — Fastly management API transport (absorbs `api_client.rs` logic, returns `PlatformError`) + +### Modify (adapter) + +- `crates/trusted-server-adapter-fastly/src/platform.rs` — implement `put`/`delete` for config, `create`/`delete` for secrets +- `crates/trusted-server-adapter-fastly/src/main.rs` — pass `runtime_services` to three handlers + +--- + +## Tasks + +### Task 1: Add `build_services_with_config_and_secret` to `test_support.rs` + +**Why:** Tasks 4 and 5 need a `RuntimeServices` with both a custom config store AND a custom secret store. The current `build_services_with_config` only customises the config store. + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/test_support.rs` + +- [ ] **Step 1: Write a failing test that calls `build_services_with_config_and_secret`** + +Add to the `#[cfg(test)]` block at the bottom of `test_support.rs`: + +```rust +#[test] +fn build_services_with_config_and_secret_uses_provided_stores() { + // Arrange: noop stores + let services = build_services_with_config_and_secret(NoopConfigStore, NoopSecretStore); + + // Act: both stores return Unsupported (confirming the injected impls are active) + let config_result = services.config_store().get(&StoreName::from("s"), "k"); + let secret_result = services.secret_store().get_bytes(&StoreName::from("s"), "k"); + + assert!(config_result.is_err(), "should delegate to injected config store"); + assert!(secret_result.is_err(), "should delegate to injected secret store"); +} +``` + +- [ ] **Step 2: Run to confirm it fails to compile** + +```bash +cargo test --package trusted-server-core platform::test_support 2>&1 | head -20 +``` + +Expected: compile error — `build_services_with_config_and_secret` not found. + +- [ ] **Step 3: Add the function above the existing `build_services_with_config`** + +```rust +/// Build a [`RuntimeServices`] instance with a custom config store and a custom secret store. +/// +/// Use this when a test exercises code that reads from config AND secret stores, +/// such as `request_signing::signing` and `request_signing::rotation`. +pub(crate) fn build_services_with_config_and_secret( + config_store: impl PlatformConfigStore + 'static, + secret_store: impl PlatformSecretStore + 'static, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(config_store)) + .secret_store(Arc::new(secret_store)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: None, + tls_protocol: None, + tls_cipher: None, + }) + .build() +} +``` + +- [ ] **Step 4: Run the test to confirm it passes** + +```bash +cargo test --package trusted-server-core platform::test_support::tests::build_services_with_config_and_secret +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/platform/test_support.rs +git commit -m "Add build_services_with_config_and_secret to test_support" +``` + +--- + +### Task 2: Create `management_api.rs` in the adapter + +**Why:** Move the Fastly management API transport (currently in `api_client.rs` in core) to the adapter, where Fastly SDK usage is appropriate. Returns `PlatformError` instead of `TrustedServerError`. + +**Credential security note (from spec):** The Fastly API token is read from the `api-keys` secret store, key `api_key`. Log store IDs and operation names only — never the token or secret value. + +**Files:** + +- Create: `crates/trusted-server-adapter-fastly/src/management_api.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` — add `mod management_api;` + +- [ ] **Step 1: Write the new module** + +Create `crates/trusted-server-adapter-fastly/src/management_api.rs`: + +```rust +//! Fastly management API transport for store write operations. +//! +//! Provides [`FastlyManagementApiClient`], which wraps the Fastly REST +//! management API for write operations on config and secret stores. +//! Used by [`super::platform::FastlyPlatformConfigStore`] and +//! [`super::platform::FastlyPlatformSecretStore`] to back store write methods. +//! +//! # Credentials +//! +//! The Fastly API token is read from the `api-keys` secret store under the +//! `api_key` entry. The token must have config-store write and secret-store +//! write permissions only — no service-level admin or purge permissions. +//! +//! # Security +//! +//! Credential values are never logged. Log messages include store IDs and +//! operation names only. + +use std::io::Read; + +use error_stack::{Report, ResultExt}; +use fastly::{Request, Response}; +use http::StatusCode; +use trusted_server_core::backend::BackendConfig; +use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreName}; + +use crate::platform::FastlyPlatformSecretStore; + +const FASTLY_API_HOST: &str = "https://api.fastly.com"; +const API_KEYS_STORE: &str = "api-keys"; +const API_KEY_ENTRY: &str = "api_key"; + +pub(crate) fn build_config_item_payload(value: &str) -> String { + format!("item_value={}", urlencoding::encode(value)) +} + +/// HTTP client for Fastly management API write operations. +/// +/// Backs the `put`/`delete` methods of [`FastlyPlatformConfigStore`] and +/// the `create`/`delete` methods of [`FastlyPlatformSecretStore`]. +pub(crate) struct FastlyManagementApiClient { + api_key: Vec, + base_url: &'static str, + backend_name: String, +} + +impl FastlyManagementApiClient { + /// Initialize the client by reading the API token from the `api-keys` secret store. + /// + /// # Errors + /// + /// Returns [`PlatformError::Backend`] if the management API backend cannot + /// be registered, or [`PlatformError::SecretStore`] if the API key cannot + /// be read. + pub(crate) fn new() -> Result> { + let backend_name = BackendConfig::from_url(FASTLY_API_HOST, true) + .change_context(PlatformError::Backend) + .attach("failed to register Fastly management API backend")?; + + let api_key = FastlyPlatformSecretStore + .get_bytes(&StoreName::from(API_KEYS_STORE), API_KEY_ENTRY) + .change_context(PlatformError::SecretStore) + .attach("failed to read Fastly API key from secret store")?; + + log::debug!("FastlyManagementApiClient: initialized for management API operations"); + + Ok(Self { + api_key, + base_url: FASTLY_API_HOST, + backend_name, + }) + } + + fn make_request( + &self, + method: &str, + path: &str, + body: Option, + content_type: &str, + ) -> Result> { + let url = format!("{}{}", self.base_url, path); + let api_key_str = String::from_utf8_lossy(&self.api_key).to_string(); + + let mut request = match method { + "GET" => Request::get(&url), + "POST" => Request::post(&url), + "PUT" => Request::put(&url), + "DELETE" => Request::delete(&url), + _ => { + return Err(Report::new(PlatformError::ConfigStore) + .attach(format!("unsupported HTTP method: {}", method))) + } + }; + + request = request + .with_header("Fastly-Key", api_key_str) + .with_header("Accept", "application/json"); + + if let Some(body_content) = body { + request = request + .with_header("Content-Type", content_type) + .with_body(body_content); + } + + request.send(&self.backend_name).map_err(|e| { + Report::new(PlatformError::ConfigStore) + .attach(format!("management API request failed: {}", e)) + }) + } + + /// Update or create a config store item. + /// + /// # Errors + /// + /// Returns an error if the API request fails or returns a non-OK status. + pub(crate) fn update_config_item( + &self, + store_id: &str, + key: &str, + value: &str, + ) -> Result<(), Report> { + let path = format!("/resources/stores/config/{}/item/{}", store_id, key); + let payload = build_config_item_payload(value); + + let mut response = self.make_request( + "PUT", + &path, + Some(payload), + "application/x-www-form-urlencoded", + )?; + + let mut buf = String::new(); + response + .get_body_mut() + .read_to_string(&mut buf) + .change_context(PlatformError::ConfigStore)?; + + if response.get_status() == StatusCode::OK { + log::debug!( + "FastlyManagementApiClient: updated config key '{}' in store '{}'", + key, + store_id + ); + Ok(()) + } else { + Err(Report::new(PlatformError::ConfigStore).attach(format!( + "config item update failed with HTTP {} for key '{}' in store '{}'", + response.get_status(), + key, + store_id + ))) + } + } + + /// Delete a config store item. + /// + /// # Errors + /// + /// Returns an error if the API request fails or returns an unexpected status. + pub(crate) fn delete_config_item( + &self, + store_id: &str, + key: &str, + ) -> Result<(), Report> { + let path = format!("/resources/stores/config/{}/item/{}", store_id, key); + + let mut response = self.make_request("DELETE", &path, None, "application/json")?; + + let mut buf = String::new(); + response + .get_body_mut() + .read_to_string(&mut buf) + .change_context(PlatformError::ConfigStore)?; + + if response.get_status() == StatusCode::OK + || response.get_status() == StatusCode::NO_CONTENT + { + log::debug!( + "FastlyManagementApiClient: deleted config key '{}' from store '{}'", + key, + store_id + ); + Ok(()) + } else { + Err(Report::new(PlatformError::ConfigStore).attach(format!( + "config item delete failed with HTTP {} for key '{}' in store '{}'", + response.get_status(), + key, + store_id + ))) + } + } + + /// Create or overwrite a secret store entry. + /// + /// # Errors + /// + /// Returns an error if the API request fails or returns a non-OK status. + pub(crate) fn create_secret( + &self, + store_id: &str, + secret_name: &str, + secret_value: &str, + ) -> Result<(), Report> { + let path = format!("/resources/stores/secret/{}/secrets", store_id); + let payload = serde_json::json!({ + "name": secret_name, + "secret": secret_value + }); + + let mut response = + self.make_request("POST", &path, Some(payload.to_string()), "application/json")?; + + let mut buf = String::new(); + response + .get_body_mut() + .read_to_string(&mut buf) + .change_context(PlatformError::SecretStore)?; + + if response.get_status() == StatusCode::OK { + log::debug!( + "FastlyManagementApiClient: created secret '{}' in store '{}'", + secret_name, + store_id + ); + Ok(()) + } else { + Err(Report::new(PlatformError::SecretStore).attach(format!( + "secret create failed with HTTP {} for name '{}' in store '{}'", + response.get_status(), + secret_name, + store_id + ))) + } + } + + /// Delete a secret store entry. + /// + /// # Errors + /// + /// Returns an error if the API request fails or returns an unexpected status. + pub(crate) fn delete_secret( + &self, + store_id: &str, + secret_name: &str, + ) -> Result<(), Report> { + let path = format!( + "/resources/stores/secret/{}/secrets/{}", + store_id, secret_name + ); + + let mut response = self.make_request("DELETE", &path, None, "application/json")?; + + let mut buf = String::new(); + response + .get_body_mut() + .read_to_string(&mut buf) + .change_context(PlatformError::SecretStore)?; + + if response.get_status() == StatusCode::OK + || response.get_status() == StatusCode::NO_CONTENT + { + log::debug!( + "FastlyManagementApiClient: deleted secret '{}' from store '{}'", + secret_name, + store_id + ); + Ok(()) + } else { + Err(Report::new(PlatformError::SecretStore).attach(format!( + "secret delete failed with HTTP {} for name '{}' in store '{}'", + response.get_status(), + secret_name, + store_id + ))) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_config_item_payload_url_encodes_reserved_characters() { + let payload = + build_config_item_payload(r#"value with spaces + symbols &= {"kid":"a+b"}"#); + + assert_eq!( + payload, + "item_value=value%20with%20spaces%20%2B%20symbols%20%26%3D%20%7B%22kid%22%3A%22a%2Bb%22%7D", + "should URL-encode config item values in form payloads" + ); + } +} +``` + +- [ ] **Step 2: Add `mod management_api;` to `main.rs`** + +In `crates/trusted-server-adapter-fastly/src/main.rs`, add near the top (alongside the other `mod` declarations): + +```rust +mod management_api; +``` + +- [ ] **Step 3: Run the payload test** + +```bash +cargo test --package trusted-server-adapter-fastly management_api -- --nocapture +``` + +Expected: `build_config_item_payload_url_encodes_reserved_characters` passes. + +- [ ] **Step 4: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/management_api.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Add FastlyManagementApiClient to adapter" +``` + +--- + +### Task 3: Implement `FastlyPlatformConfigStore` write methods + +**Why:** Replace the `NotImplemented` stubs in `platform.rs` with real calls to `FastlyManagementApiClient`. The existing `NotImplemented` test for secret store (`fastly_platform_secret_store_create_returns_not_implemented`, `fastly_platform_secret_store_delete_returns_not_implemented`) must be deleted now that the real implementation lands. Check if there are equivalent config store tests to delete too. + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/platform.rs` + +- [ ] **Step 1: Delete `NotImplemented` tests for secret store writes** + +In `platform.rs` tests, find and delete these two tests (they assert the old stub behavior that no longer holds): + +- `fastly_platform_secret_store_create_returns_not_implemented` +- `fastly_platform_secret_store_delete_returns_not_implemented` + +There are no analogous `NotImplemented` tests for `FastlyPlatformConfigStore::put/delete` — only the secret store stubs have them. No config-store equivalent to search for. + +- [ ] **Step 2: Update `FastlyPlatformConfigStore::put` and `delete`** + +In `platform.rs`, replace: + +```rust +fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, +) -> Result<(), Report> { + Err(Report::new(PlatformError::NotImplemented)) +} + +fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::NotImplemented)) +} +``` + +With: + +```rust +fn put( + &self, + store_id: &StoreId, + key: &str, + value: &str, +) -> Result<(), Report> { + let client = crate::management_api::FastlyManagementApiClient::new()?; + client.update_config_item(store_id.as_ref(), key, value) +} + +fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + let client = crate::management_api::FastlyManagementApiClient::new()?; + client.delete_config_item(store_id.as_ref(), key) +} +``` + +- [ ] **Step 3: Update `FastlyPlatformSecretStore::create` and `delete`** + +Replace: + +```rust +fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, +) -> Result<(), Report> { + Err(Report::new(PlatformError::NotImplemented)) +} + +fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::NotImplemented)) +} +``` + +With: + +```rust +fn create( + &self, + store_id: &StoreId, + name: &str, + value: &str, +) -> Result<(), Report> { + let client = crate::management_api::FastlyManagementApiClient::new()?; + client.create_secret(store_id.as_ref(), name, value) +} + +fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { + let client = crate::management_api::FastlyManagementApiClient::new()?; + client.delete_secret(store_id.as_ref(), name) +} +``` + +- [ ] **Step 4: Verify adapter compiles and remaining tests pass** + +```bash +cargo test --package trusted-server-adapter-fastly -- --nocapture +``` + +Expected: all tests pass (the `NotImplemented` tests were deleted; remaining tests still pass). + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/platform.rs +git commit -m "Implement FastlyPlatformConfigStore and FastlyPlatformSecretStore write methods via management API" +``` + +--- + +### Task 4: Migrate `rotation.rs` to `RuntimeServices` + +**Why:** `KeyRotationManager` currently holds `FastlyConfigStore` (reads) and `FastlyApiClient` (writes) as fields. Replace both with `&RuntimeServices` passed to each method. + +**New design:** + +- Drop `config_store: FastlyConfigStore` and `api_client: FastlyApiClient` fields +- Keep `config_store_id: StoreId` and `secret_store_id: StoreId` (passed to write methods) +- `new()` is now infallible (no API key fetch at construction time) +- All `rotate_key`, `list_active_keys`, `deactivate_key`, `delete_key` accept `services: &RuntimeServices` +- Reads use `JWKS_STORE_NAME` (edge-visible name); writes use the stored `StoreId` values + +**Files:** + +- Modify: `crates/trusted-server-core/src/request_signing/rotation.rs` + +- [ ] **Step 1: Write failing tests that define the new API** + +Replace the `#[cfg(test)]` module in `rotation.rs` with: + +```rust +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Mutex; + + use error_stack::Report; + + use crate::platform::test_support::build_services_with_config_and_secret; + use crate::platform::{ + PlatformConfigStore, PlatformError, PlatformSecretStore, StoreId, StoreName, + }; + use crate::request_signing::Keypair; + + use super::*; + + // --------------------------------------------------------------------------- + // Spy stores: record put/create/delete calls, serve preset get values + // --------------------------------------------------------------------------- + + struct SpyConfigStore { + data: Mutex>, + puts: Mutex>, + deletes: Mutex>, + } + + impl SpyConfigStore { + fn new(initial: HashMap) -> Self { + Self { + data: Mutex::new(initial), + puts: Mutex::new(vec![]), + deletes: Mutex::new(vec![]), + } + } + } + + impl PlatformConfigStore for SpyConfigStore { + fn get(&self, _: &StoreName, key: &str) -> Result> { + self.data + .lock() + .expect("should lock data") + .get(key) + .cloned() + .ok_or_else(|| Report::new(PlatformError::ConfigStore)) + } + + fn put( + &self, + store_id: &StoreId, + key: &str, + value: &str, + ) -> Result<(), Report> { + self.puts.lock().expect("should lock puts").push(( + store_id.to_string(), + key.to_string(), + value.to_string(), + )); + self.data + .lock() + .expect("should lock data") + .insert(key.to_string(), value.to_string()); + Ok(()) + } + + fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + self.deletes + .lock() + .expect("should lock deletes") + .push((store_id.to_string(), key.to_string())); + self.data + .lock() + .expect("should lock data") + .remove(key); + Ok(()) + } + } + + struct SpySecretStore { + creates: Mutex>, + deletes: Mutex>, + } + + impl SpySecretStore { + fn new() -> Self { + Self { + creates: Mutex::new(vec![]), + deletes: Mutex::new(vec![]), + } + } + } + + impl PlatformSecretStore for SpySecretStore { + fn get_bytes(&self, _: &StoreName, _: &str) -> Result, Report> { + Err(Report::new(PlatformError::SecretStore)) + } + + fn create( + &self, + store_id: &StoreId, + name: &str, + value: &str, + ) -> Result<(), Report> { + self.creates.lock().expect("should lock creates").push(( + store_id.to_string(), + name.to_string(), + value.to_string(), + )); + Ok(()) + } + + fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { + self.deletes + .lock() + .expect("should lock deletes") + .push((store_id.to_string(), name.to_string())); + Ok(()) + } + } + + // --------------------------------------------------------------------------- + // Tests + // --------------------------------------------------------------------------- + + #[test] + fn generate_date_based_kid_has_correct_format() { + let kid = generate_date_based_kid(); + assert!(kid.starts_with("ts-"), "should start with 'ts-'"); + assert!(kid.len() >= 13, "should be at least 13 characters"); + let parts: Vec<&str> = kid.split('-').collect(); + assert_eq!(parts.len(), 4, "should have 4 dash-separated parts"); + assert_eq!(parts[0], "ts", "first part should be 'ts'"); + } + + #[test] + fn new_is_infallible_and_stores_ids() { + let manager = KeyRotationManager::new("cfg-store-123", "sec-store-456"); + assert_eq!( + manager.config_store_id.as_ref(), + "cfg-store-123", + "should store config_store_id" + ); + assert_eq!( + manager.secret_store_id.as_ref(), + "sec-store-456", + "should store secret_store_id" + ); + } + + #[test] + fn rotate_key_stores_private_key_via_secret_store_create() { + let config_store = SpyConfigStore::new(HashMap::new()); + let secret_store = SpySecretStore::new(); + let services = + build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.rotate_key(&services, Some("new-kid".to_string())); + + assert!(result.is_ok(), "should succeed when stores accept writes"); + let rotation = result.expect("should produce rotation result"); + assert_eq!(rotation.new_kid, "new-kid", "should use the provided kid"); + assert!( + rotation.active_kids.contains(&"new-kid".to_string()), + "should include new kid in active kids" + ); + } + + #[test] + fn deactivate_key_fails_when_only_one_key_remains() { + let mut data = HashMap::new(); + data.insert("active-kids".to_string(), "only-key".to_string()); + let config_store = SpyConfigStore::new(data); + let secret_store = SpySecretStore::new(); + let services = + build_services_with_config_and_secret(config_store, secret_store); + + let manager = KeyRotationManager::new("cfg-id", "sec-id"); + let result = manager.deactivate_key(&services, "only-key"); + + assert!( + result.is_err(), + "should fail to deactivate the last active key" + ); + } + + #[test] + fn key_rotation_result_structure_is_valid() { + let jwk = Keypair::generate().get_jwk("test-key".to_string()); + let result = KeyRotationResult { + new_kid: "ts-2024-01-01".to_string(), + previous_kid: Some("ts-2023-12-31".to_string()), + active_kids: vec![ + "ts-2023-12-31".to_string(), + "ts-2024-01-01".to_string(), + ], + jwk: jwk.clone(), + }; + + assert_eq!(result.new_kid, "ts-2024-01-01"); + assert_eq!(result.previous_kid, Some("ts-2023-12-31".to_string())); + assert_eq!(result.active_kids.len(), 2); + assert_eq!(result.jwk.prm.kid, Some("test-key".to_string())); + } +} +``` + +- [ ] **Step 2: Run tests to confirm they fail (expected compile error)** + +```bash +cargo test --package trusted-server-core request_signing::rotation 2>&1 | head -30 +``` + +Expected: compile error — `KeyRotationManager::new` still returns `Result`, and `rotate_key` doesn't take `services`. + +- [ ] **Step 3: Rewrite `rotation.rs`** + +Replace the entire file with the following (preserving `generate_date_based_kid` and `KeyRotationResult`): + +```rust +//! Key rotation management for request signing. +//! +//! This module provides functionality for rotating signing keys, managing key +//! lifecycle, and storing keys via platform store primitives through +//! [`RuntimeServices`]. + +use std::sync::LazyLock; + +use base64::{engine::general_purpose, Engine}; +use ed25519_dalek::SigningKey; +use error_stack::{Report, ResultExt}; +use jose_jwk::Jwk; + +use crate::error::TrustedServerError; +use crate::platform::{RuntimeServices, StoreId, StoreName}; +use crate::request_signing::JWKS_CONFIG_STORE_NAME; + +use super::Keypair; + +static JWKS_STORE_NAME: LazyLock = + LazyLock::new(|| StoreName::from(JWKS_CONFIG_STORE_NAME)); + +#[derive(Debug, Clone)] +pub struct KeyRotationResult { + pub new_kid: String, + pub previous_kid: Option, + pub active_kids: Vec, + pub jwk: Jwk, +} + +/// Manages signing key lifecycle using platform store primitives. +/// +/// Reads use the edge-visible store name ([`JWKS_CONFIG_STORE_NAME`]). +/// Writes use the management API store identifiers supplied at construction. +pub struct KeyRotationManager { + /// Management API store ID for config store writes. + config_store_id: StoreId, + /// Management API store ID for secret store writes. + secret_store_id: StoreId, +} + +impl KeyRotationManager { + /// Creates a new key rotation manager. + /// + /// The `config_store_id` and `secret_store_id` are platform management API + /// identifiers used for write operations. Edge reads use the store names + /// defined in [`JWKS_CONFIG_STORE_NAME`] and + /// [`crate::request_signing::SIGNING_SECRET_STORE_NAME`]. + #[must_use] + pub fn new( + config_store_id: impl Into, + secret_store_id: impl Into, + ) -> Self { + Self { + config_store_id: StoreId::from(config_store_id.into()), + secret_store_id: StoreId::from(secret_store_id.into()), + } + } + + /// Rotates the signing key by generating a new keypair and storing it. + /// + /// # Errors + /// + /// Returns an error if key storage or update operations fail. + pub fn rotate_key( + &self, + services: &RuntimeServices, + kid: Option, + ) -> Result> { + let new_kid = kid.unwrap_or_else(generate_date_based_kid); + + let keypair = Keypair::generate(); + let jwk = keypair.get_jwk(new_kid.clone()); + let previous_kid = services + .config_store() + .get(&JWKS_STORE_NAME, "current-kid") + .ok(); + + self.store_private_key(services, &new_kid, &keypair.signing_key)?; + self.store_public_jwk(services, &new_kid, &jwk)?; + + let active_kids = match &previous_kid { + Some(prev) if prev != &new_kid => vec![prev.clone(), new_kid.clone()], + _ => vec![new_kid.clone()], + }; + + self.update_current_kid(services, &new_kid)?; + self.update_active_kids(services, &active_kids)?; + + Ok(KeyRotationResult { + new_kid, + previous_kid, + active_kids, + jwk, + }) + } + + fn store_private_key( + &self, + services: &RuntimeServices, + kid: &str, + signing_key: &SigningKey, + ) -> Result<(), Report> { + let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); + + services + .secret_store() + .create(&self.secret_store_id, kid, &key_b64) + .change_context(TrustedServerError::Configuration { + message: format!("failed to store private key '{}'", kid), + }) + } + + fn store_public_jwk( + &self, + services: &RuntimeServices, + kid: &str, + jwk: &Jwk, + ) -> Result<(), Report> { + let jwk_json = serde_json::to_string(jwk).map_err(|e| { + Report::new(TrustedServerError::Configuration { + message: format!("failed to serialize JWK: {}", e), + }) + })?; + + services + .config_store() + .put(&self.config_store_id, kid, &jwk_json) + .change_context(TrustedServerError::Configuration { + message: format!("failed to store public JWK '{}'", kid), + }) + } + + fn update_current_kid( + &self, + services: &RuntimeServices, + kid: &str, + ) -> Result<(), Report> { + services + .config_store() + .put(&self.config_store_id, "current-kid", kid) + .change_context(TrustedServerError::Configuration { + message: "failed to update current-kid".into(), + }) + } + + fn update_active_kids( + &self, + services: &RuntimeServices, + active_kids: &[String], + ) -> Result<(), Report> { + let active_kids_str = active_kids.join(","); + + services + .config_store() + .put(&self.config_store_id, "active-kids", &active_kids_str) + .change_context(TrustedServerError::Configuration { + message: "failed to update active-kids".into(), + }) + } + + /// Lists all active key IDs. + /// + /// # Errors + /// + /// Returns an error if the active keys cannot be retrieved from the config store. + pub fn list_active_keys( + &self, + services: &RuntimeServices, + ) -> Result, Report> { + let active_kids_str = services + .config_store() + .get(&JWKS_STORE_NAME, "active-kids") + .change_context(TrustedServerError::Configuration { + message: "failed to read active-kids from config store".into(), + })?; + + Ok(active_kids_str + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect()) + } + + /// Deactivates a key by removing it from the active keys list. + /// + /// # Errors + /// + /// Returns an error if this would deactivate the last active key, or if the update fails. + pub fn deactivate_key( + &self, + services: &RuntimeServices, + kid: &str, + ) -> Result<(), Report> { + let mut active_kids = self.list_active_keys(services)?; + active_kids.retain(|k| k != kid); + + if active_kids.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "cannot deactivate the last active key".into(), + })); + } + + self.update_active_kids(services, &active_kids) + } + + /// Deletes a key by deactivating it and removing it from storage. + /// + /// # Errors + /// + /// Returns an error if deactivation fails or if the key cannot be deleted from storage. + pub fn delete_key( + &self, + services: &RuntimeServices, + kid: &str, + ) -> Result<(), Report> { + self.deactivate_key(services, kid)?; + + services + .config_store() + .delete(&self.config_store_id, kid) + .change_context(TrustedServerError::Configuration { + message: "failed to delete JWK from config store".into(), + })?; + + services + .secret_store() + .delete(&self.secret_store_id, kid) + .change_context(TrustedServerError::Configuration { + message: "failed to delete signing key from secret store".into(), + })?; + + Ok(()) + } +} + +#[must_use] +pub fn generate_date_based_kid() -> String { + use chrono::Utc; + format!("ts-{}", Utc::now().format("%Y-%m-%d")) +} +``` + +(Append the test module from Step 1 at the bottom.) + +- [ ] **Step 4: Run rotation tests** + +```bash +cargo test --package trusted-server-core request_signing::rotation -- --nocapture +``` + +Expected: all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/request_signing/rotation.rs +git commit -m "Migrate KeyRotationManager from FastlyApiClient to RuntimeServices store primitives" +``` + +--- + +### Task 5: Migrate `signing.rs` to `RuntimeServices` + +**Why:** Three items in `signing.rs` still construct `FastlyConfigStore`/`FastlySecretStore` directly. Replace all three with `RuntimeServices`. The existing viceroy-dependent tests are replaced with proper unit tests using stub stores. + +**Changed signatures:** + +- `get_current_key_id()` → `get_current_key_id(services: &RuntimeServices)` +- `RequestSigner::from_config()` → `RequestSigner::from_services(services: &RuntimeServices)` (rename to make the break explicit) +- `verify_signature(payload, sig, kid)` → `verify_signature(payload, sig, kid, services: &RuntimeServices)` + +**Files:** + +- Modify: `crates/trusted-server-core/src/request_signing/signing.rs` + +- [ ] **Step 1: Write failing tests for the new API** + +Replace the entire `#[cfg(test)]` module in `signing.rs` with the following (before updating the production code, so the tests fail to compile): + +```rust +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use error_stack::Report; + + use crate::platform::test_support::build_services_with_config_and_secret; + use crate::platform::{PlatformConfigStore, PlatformError, PlatformSecretStore, StoreId, StoreName}; + + use super::*; + + // --------------------------------------------------------------------------- + // Stub stores with preset data + // --------------------------------------------------------------------------- + + struct StubConfigStore(HashMap); + + impl PlatformConfigStore for StubConfigStore { + fn get(&self, _: &StoreName, key: &str) -> Result> { + self.0 + .get(key) + .cloned() + .ok_or_else(|| Report::new(PlatformError::ConfigStore)) + } + + fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct StubSecretStore(HashMap>); + + impl PlatformSecretStore for StubSecretStore { + fn get_bytes(&self, _: &StoreName, key: &str) -> Result, Report> { + self.0 + .get(key) + .cloned() + .ok_or_else(|| Report::new(PlatformError::SecretStore)) + } + + fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + /// Build `RuntimeServices` with a real Ed25519 keypair pre-loaded in the + /// stub stores. Returns the `kid` used so callers can reference it. + fn build_signing_services() -> crate::platform::RuntimeServices { + use base64::{engine::general_purpose, Engine}; + use ed25519_dalek::SigningKey; + use rand::rngs::OsRng; + + let signing_key = SigningKey::generate(&mut OsRng); + let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); + let verifying_key = signing_key.verifying_key(); + let x_b64 = general_purpose::URL_SAFE_NO_PAD.encode(verifying_key.as_bytes()); + let jwk_json = format!( + r#"{{"kty":"OKP","crv":"Ed25519","x":"{}","kid":"test-kid","alg":"EdDSA"}}"#, + x_b64 + ); + + let mut config_data = HashMap::new(); + config_data.insert("current-kid".to_string(), "test-kid".to_string()); + config_data.insert("test-kid".to_string(), jwk_json); + + let mut secret_data = HashMap::new(); + secret_data.insert("test-kid".to_string(), key_b64.into_bytes()); + + build_services_with_config_and_secret( + StubConfigStore(config_data), + StubSecretStore(secret_data), + ) + } + + // --------------------------------------------------------------------------- + // Tests + // --------------------------------------------------------------------------- + + #[test] + fn from_services_loads_kid_from_config_store() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + + assert_eq!(signer.kid, "test-kid", "should load kid from config store"); + } + + #[test] + fn sign_produces_non_empty_url_safe_base64_signature() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + + let signature = signer + .sign(b"these pretzels are making me thirsty") + .expect("should sign payload"); + + assert!(!signature.is_empty(), "should produce non-empty signature"); + assert!(signature.len() > 32, "should produce a full-length signature"); + } + + #[test] + fn sign_and_verify_roundtrip_succeeds() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + let payload = b"test payload for verification"; + + let signature = signer.sign(payload).expect("should sign payload"); + let verified = verify_signature(payload, &signature, &signer.kid, &services) + .expect("should attempt verification"); + + assert!(verified, "should verify a valid signature"); + } + + #[test] + fn verify_returns_false_for_wrong_payload() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + let signature = signer.sign(b"original").expect("should sign"); + + let verified = verify_signature(b"wrong payload", &signature, &signer.kid, &services) + .expect("should attempt verification"); + + assert!(!verified, "should not verify signature for wrong payload"); + } + + #[test] + fn verify_errors_for_unknown_kid() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + let signature = signer.sign(b"payload").expect("should sign"); + + let result = verify_signature(b"payload", &signature, "nonexistent-kid", &services); + + assert!(result.is_err(), "should error for unknown kid"); + } + + #[test] + fn verify_errors_for_malformed_signature() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + + let result = verify_signature(b"payload", "not-valid-base64!!!", &signer.kid, &services); + + assert!(result.is_err(), "should error for malformed signature"); + } + + #[test] + fn signing_params_build_payload_serializes_all_fields() { + let params = SigningParams { + request_id: "req-123".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + timestamp: 1706900000, + }; + + let payload = params.build_payload("kid-abc").expect("should build payload"); + let parsed: serde_json::Value = + serde_json::from_str(&payload).expect("should be valid JSON"); + + assert_eq!(parsed["version"], SIGNING_VERSION); + assert_eq!(parsed["kid"], "kid-abc"); + assert_eq!(parsed["host"], "example.com"); + assert_eq!(parsed["scheme"], "https"); + assert_eq!(parsed["id"], "req-123"); + assert_eq!(parsed["ts"], 1706900000); + } + + #[test] + fn signing_params_new_creates_recent_timestamp() { + let params = SigningParams::new( + "req-123".to_string(), + "example.com".to_string(), + "https".to_string(), + ); + + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("should get system time") + .as_millis() as u64; + + assert!( + params.timestamp <= now_ms, + "timestamp should not be in the future" + ); + assert!( + params.timestamp >= now_ms - 60_000, + "timestamp should be within the last minute" + ); + } + + #[test] + fn sign_request_enhanced_produces_verifiable_signature() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + let params = SigningParams::new( + "auction-123".to_string(), + "publisher.com".to_string(), + "https".to_string(), + ); + + let signature = signer.sign_request(¶ms).expect("should sign request"); + let payload = params.build_payload(&signer.kid).expect("should build payload"); + + let verified = + verify_signature(payload.as_bytes(), &signature, &signer.kid, &services) + .expect("should verify"); + + assert!(verified, "enhanced request signature should be verifiable"); + } + + #[test] + fn sign_request_different_hosts_produce_different_signatures() { + let services = build_signing_services(); + let signer = RequestSigner::from_services(&services) + .expect("should create signer from services"); + + let params1 = SigningParams { + request_id: "req-1".to_string(), + request_host: "host1.com".to_string(), + request_scheme: "https".to_string(), + timestamp: 1706900000, + }; + let params2 = SigningParams { + request_id: "req-1".to_string(), + request_host: "host2.com".to_string(), + request_scheme: "https".to_string(), + timestamp: 1706900000, + }; + + let sig1 = signer.sign_request(¶ms1).expect("should sign params1"); + let sig2 = signer.sign_request(¶ms2).expect("should sign params2"); + + assert_ne!( + sig1, sig2, + "different hosts should produce different signatures" + ); + } +} +``` + +- [ ] **Step 2: Run to confirm compile failure** + +```bash +cargo test --package trusted-server-core request_signing::signing 2>&1 | head -20 +``` + +Expected: compile error — `from_services`, `verify_signature` with 4 args not found. + +- [ ] **Step 3: Rewrite `signing.rs` production code** + +Replace the imports, `LazyLock` statics, and function bodies. Key changes: + +**Imports — replace:** + +```rust +use crate::storage::{FastlyConfigStore, FastlySecretStore}; +``` + +**With:** + +```rust +use std::sync::LazyLock; + +use crate::platform::{RuntimeServices, StoreName}; +``` + +**Add after imports:** + +```rust +static JWKS_STORE_NAME: LazyLock = + LazyLock::new(|| StoreName::from(JWKS_CONFIG_STORE_NAME)); + +static SIGNING_STORE_NAME: LazyLock = + LazyLock::new(|| StoreName::from(SIGNING_SECRET_STORE_NAME)); +``` + +**Replace `get_current_key_id`:** + +```rust +pub fn get_current_key_id( + services: &RuntimeServices, +) -> Result> { + services + .config_store() + .get(&JWKS_STORE_NAME, "current-kid") + .change_context(TrustedServerError::Configuration { + message: "failed to read current-kid from config store".into(), + }) +} +``` + +**Replace `RequestSigner::from_config` with `from_services`:** + +```rust +pub fn from_services( + services: &RuntimeServices, +) -> Result> { + let key_id = services + .config_store() + .get(&JWKS_STORE_NAME, "current-kid") + .change_context(TrustedServerError::Configuration { + message: "failed to get current-kid".into(), + })?; + + let key_bytes = services + .secret_store() + .get_bytes(&SIGNING_STORE_NAME, &key_id) + .change_context(TrustedServerError::Configuration { + message: format!("failed to get signing key for kid: {}", key_id), + })?; + + let signing_key = parse_ed25519_signing_key(key_bytes)?; + + Ok(Self { + key: signing_key, + kid: key_id, + }) +} +``` + +**Replace `verify_signature` — add `services: &RuntimeServices` parameter and replace `FastlyConfigStore::new(...)` with `services.config_store().get(&JWKS_STORE_NAME, kid)`.** + +Full new signature: + +```rust +pub fn verify_signature( + payload: &[u8], + signature_b64: &str, + kid: &str, + services: &RuntimeServices, +) -> Result> { + let jwk_json = services + .config_store() + .get(&JWKS_STORE_NAME, kid) + .change_context(TrustedServerError::Configuration { + message: format!("failed to get JWK for kid: {}", kid), + })?; + // ... rest of verification logic unchanged ... +} +``` + +- [ ] **Step 4: Run signing tests** + +```bash +cargo test --package trusted-server-core request_signing::signing -- --nocapture +``` + +Expected: all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/request_signing/signing.rs +git commit -m "Migrate signing.rs from FastlyConfigStore/FastlySecretStore to RuntimeServices" +``` + +--- + +### Task 6: Update `endpoints.rs` to accept `&RuntimeServices` + +**Why:** Three handlers don't receive `&RuntimeServices`: `handle_verify_signature`, `handle_rotate_key`, `handle_deactivate_key`. They need it to pass to `verify_signature`, `KeyRotationManager` methods, and (for verify) `RequestSigner::from_services`. + +**Note:** `fastly::{Request, Response}` and `fastly::mime` remain — type migration is Phase 2 (PR 12). + +**Files:** + +- Modify: `crates/trusted-server-core/src/request_signing/endpoints.rs` + +- [ ] **Step 1: Update `handle_verify_signature` signature and body** + +Change: + +```rust +pub fn handle_verify_signature( + _settings: &Settings, + mut req: Request, +) -> Result> { +``` + +To: + +```rust +pub fn handle_verify_signature( + _settings: &Settings, + services: &RuntimeServices, + mut req: Request, +) -> Result> { +``` + +Update the `verify_signature` call: + +```rust +let verification_result = signing::verify_signature( + verify_req.payload.as_bytes(), + &verify_req.signature, + &verify_req.kid, + services, +); +``` + +- [ ] **Step 2: Update `handle_rotate_key` signature and body** + +Change signature to add `services: &RuntimeServices` as second parameter. Update the `KeyRotationManager` usage: + +```rust +// Before: +let manager = KeyRotationManager::new(config_store_id, secret_store_id).change_context(...)?; +match manager.rotate_key(rotate_req.kid) { ... } +manager.list_active_keys().unwrap_or_else(...) + +// After: +let manager = KeyRotationManager::new(config_store_id, secret_store_id); +match manager.rotate_key(services, rotate_req.kid) { ... } +manager.list_active_keys(services).unwrap_or_else(...) +``` + +Remove the `.change_context(...)` on `KeyRotationManager::new(...)` — it's now infallible. + +- [ ] **Step 3: Update `handle_deactivate_key` signature and body** + +Same pattern: add `services: &RuntimeServices`, update all `manager.*` calls to pass `services`: + +- `manager.delete_key(&deactivate_req.kid)` → `manager.delete_key(services, &deactivate_req.kid)` +- `manager.deactivate_key(&deactivate_req.kid)` → `manager.deactivate_key(services, &deactivate_req.kid)` +- `manager.list_active_keys()` → `manager.list_active_keys(services)` + +Remove the `.change_context(...)` on `KeyRotationManager::new(...)`. + +- [ ] **Step 4: Update `endpoints.rs` tests** + +The tests in `endpoints.rs` that call `handle_verify_signature`, `handle_rotate_key`, `handle_deactivate_key` must be updated to pass a `&RuntimeServices`. Use `noop_services()` (from `test_support`) for rotation/deactivation tests (they test error paths that don't reach the stores). For `test_handle_verify_signature_valid` and `test_handle_verify_signature_invalid`, build a `RuntimeServices` with actual key material using `build_signing_services` (inline the helper or import logic). + +Also update `RequestSigner::from_config()` calls in test helpers to `RequestSigner::from_services(&services)`. + +**Add this helper to the `#[cfg(test)]` block at the top of `endpoints.rs` tests** (it cannot be imported from `signing.rs` because that function lives inside a `#[cfg(test)]` private module): + +```rust +/// Build `RuntimeServices` pre-loaded with a real Ed25519 keypair for +/// testing signature creation and verification in endpoint handlers. +fn build_signing_services_for_test() -> crate::platform::RuntimeServices { + use std::collections::HashMap; + + use base64::{engine::general_purpose, Engine}; + use ed25519_dalek::SigningKey; + use error_stack::Report; + use rand::rngs::OsRng; + + use crate::platform::test_support::build_services_with_config_and_secret; + use crate::platform::{ + PlatformConfigStore, PlatformError, PlatformSecretStore, StoreId, StoreName, + }; + + struct MapConfigStore(HashMap); + impl PlatformConfigStore for MapConfigStore { + fn get(&self, _: &StoreName, key: &str) -> Result> { + self.0.get(key).cloned().ok_or_else(|| Report::new(PlatformError::ConfigStore)) + } + fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct MapSecretStore(HashMap>); + impl PlatformSecretStore for MapSecretStore { + fn get_bytes(&self, _: &StoreName, key: &str) -> Result, Report> { + self.0.get(key).cloned().ok_or_else(|| Report::new(PlatformError::SecretStore)) + } + fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + let signing_key = SigningKey::generate(&mut OsRng); + let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); + let x_b64 = general_purpose::URL_SAFE_NO_PAD.encode(signing_key.verifying_key().as_bytes()); + let jwk_json = format!( + r#"{{"kty":"OKP","crv":"Ed25519","x":"{}","kid":"test-kid","alg":"EdDSA"}}"#, + x_b64 + ); + + let mut cfg = HashMap::new(); + cfg.insert("current-kid".to_string(), "test-kid".to_string()); + cfg.insert("test-kid".to_string(), jwk_json); + + let mut sec = HashMap::new(); + sec.insert("test-kid".to_string(), key_b64.into_bytes()); + + build_services_with_config_and_secret(MapConfigStore(cfg), MapSecretStore(sec)) +} +``` + +Pattern for verify tests: + +```rust +// In test_handle_verify_signature_valid: +let services = build_signing_services_for_test(); +let signer = crate::request_signing::RequestSigner::from_services(&services) + .expect("should create signer from services"); +// ... build req as before ... +let mut resp = handle_verify_signature(&settings, &services, req) + .expect("should handle verification request"); +``` + +For rotation/deactivation tests, `noop_services()` is fine — these tests use the `match result { Ok => log, Err => log }` pattern and do not assert against store state. The `noop_services()` causes `KeyRotationManager` methods to fail at the store read/write level, which is the expected behavior in a test environment without real stores: + +```rust +let services = noop_services(); +let result = handle_rotate_key(&settings, &services, req); +// existing match-and-log pattern works unchanged +``` + +- [ ] **Step 5: Run endpoints tests** + +```bash +cargo test --package trusted-server-core request_signing::endpoints -- --nocapture +``` + +Expected: all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/request_signing/endpoints.rs +git commit -m "Add RuntimeServices parameter to handle_verify_signature, handle_rotate_key, handle_deactivate_key" +``` + +--- + +### Task 7: Update `main.rs` to pass `runtime_services` to updated handlers + +**Why:** The adapter `main.rs` calls the three handlers without `runtime_services`. Add it. `runtime_services` is already in scope in all call sites. + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` + +- [ ] **Step 1: Update the three handler call sites** + +Find (approximate lines, verify exact lines before editing): + +```rust +(Method::POST, "/verify-signature") => handle_verify_signature(settings, req), +(Method::POST, "/admin/keys/rotate") => handle_rotate_key(settings, req), +(Method::POST, "/admin/keys/deactivate") => handle_deactivate_key(settings, req), +``` + +Replace with: + +```rust +(Method::POST, "/verify-signature") => { + handle_verify_signature(settings, runtime_services, req) +} +(Method::POST, "/admin/keys/rotate") => { + handle_rotate_key(settings, runtime_services, req) +} +(Method::POST, "/admin/keys/deactivate") => { + handle_deactivate_key(settings, runtime_services, req) +} +``` + +- [ ] **Step 2: Verify the full workspace compiles** + +```bash +cargo check --workspace +``` + +Expected: no errors. + +- [ ] **Step 3: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Pass runtime_services to signing endpoint handlers in main.rs" +``` + +--- + +### Task 8: Delete `api_client.rs` and clean up `storage/mod.rs` + +**Why:** `api_client.rs` is now fully superseded by `management_api.rs` in the adapter. No core code references `FastlyApiClient` anymore (verified by rotation.rs migration). + +**Files:** + +- Delete: `crates/trusted-server-core/src/storage/api_client.rs` +- Modify: `crates/trusted-server-core/src/storage/mod.rs` + +- [ ] **Step 1: Verify zero legacy storage imports remain in `request_signing/`** + +```bash +grep -r "FastlyApiClient\|FastlyConfigStore\|FastlySecretStore" crates/trusted-server-core/src/request_signing/ +``` + +Expected: no output. If any matches appear, fix those call sites before continuing. + +Also verify no remaining references to `FastlyApiClient` anywhere in core: + +```bash +grep -r "FastlyApiClient" crates/trusted-server-core/src/ +``` + +Expected: no output. + +- [ ] **Step 2: Delete `api_client.rs`** + +```bash +rm crates/trusted-server-core/src/storage/api_client.rs +``` + +- [ ] **Step 3: Update `storage/mod.rs`** + +Remove the `api_client` module declaration and re-export. Change from: + +```rust +//! Legacy Fastly-backed store types. +//! +//! These types predate the [`crate::platform`] abstraction and will be removed +//! once all call sites have migrated to the platform traits. New code should +//! use [`crate::platform::PlatformConfigStore`], +//! [`crate::platform::PlatformSecretStore`], and the management write methods +//! via [`crate::platform::RuntimeServices`]. + +pub(crate) mod api_client; +pub(crate) mod config_store; +pub(crate) mod secret_store; + +pub use api_client::FastlyApiClient; +pub use config_store::FastlyConfigStore; +pub use secret_store::FastlySecretStore; +``` + +To: + +```rust +//! Legacy Fastly-backed store types. +//! +//! These types predate the [`crate::platform`] abstraction and will be removed +//! once all call sites have migrated to the platform traits. New code should +//! use [`crate::platform::PlatformConfigStore`] and +//! [`crate::platform::PlatformSecretStore`] via [`crate::platform::RuntimeServices`]. + +pub(crate) mod config_store; +pub(crate) mod secret_store; + +pub use config_store::FastlyConfigStore; +pub use secret_store::FastlySecretStore; +``` + +- [ ] **Step 4: Run the full workspace test suite** + +```bash +cargo test --workspace +``` + +Expected: all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/storage/mod.rs +git rm crates/trusted-server-core/src/storage/api_client.rs +git commit -m "Delete storage/api_client.rs from core; remove FastlyApiClient" +``` + +--- + +### Task 9: Run CI gates + +- [ ] **Step 1: Format check** + +```bash +cargo fmt --all -- --check +``` + +If it fails, fix with `cargo fmt --all` and re-run. + +- [ ] **Step 2: Clippy** + +```bash +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +Fix any lints before proceeding. + +- [ ] **Step 3: Full test suite** + +```bash +cargo test --workspace +``` + +Expected: all tests pass. + +- [ ] **Step 4: Commit any lint/format fixes** + +```bash +git add -A +git commit -m "Fix clippy lints and formatting" +``` + +Only create this commit if there are actual changes. + +--- + +## Acceptance Checklist + +Verify all of the following before considering PR 9 complete: + +- [ ] `crates/trusted-server-core/src/storage/api_client.rs` no longer exists +- [ ] `crates/trusted-server-adapter-fastly/src/management_api.rs` exists +- [ ] `grep -r "FastlyApiClient\|from crate::storage::api" crates/trusted-server-core/src/request_signing/` returns no matches +- [ ] `grep -r "FastlyConfigStore\|FastlySecretStore" crates/trusted-server-core/src/request_signing/` returns no matches +- [ ] `cargo test --workspace` passes +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` passes +- [ ] `cargo fmt --all -- --check` passes +- [ ] `handle_verify_signature`, `handle_rotate_key`, `handle_deactivate_key` in `endpoints.rs` all accept `&RuntimeServices` +- [ ] `FastlyPlatformConfigStore::put/delete` and `FastlyPlatformSecretStore::create/delete` in `platform.rs` no longer return `PlatformError::NotImplemented` diff --git a/docs/superpowers/plans/2026-04-02-pr10-logging-initialization.md b/docs/superpowers/plans/2026-04-02-pr10-logging-initialization.md new file mode 100644 index 00000000..88a39b55 --- /dev/null +++ b/docs/superpowers/plans/2026-04-02-pr10-logging-initialization.md @@ -0,0 +1,289 @@ +# PR 10 Logging Initialization Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Keep logging backend initialization adapter-owned by extracting Fastly logging setup into an adapter-local module and removing `log-fastly` from `trusted-server-core`. + +**Architecture:** `trusted-server-core` continues to emit logs only through `log` macros and has no platform logging backend dependency. `trusted-server-adapter-fastly` owns Fastly-specific logger initialization behind a local `logging.rs` module, and `main.rs` just calls into that adapter-local entrypoint. + +**Tech Stack:** Rust 2024 edition conventions, `log`, `log-fastly`, `fern`, `chrono` + +--- + +## File Structure + +- Create: `crates/trusted-server-adapter-fastly/src/logging.rs` + - Own Fastly-specific logger setup and any small formatting helpers needed for unit testing. +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` + - Stop carrying logger implementation details directly; import the adapter-local module and call `logging::init_logger()`. +- Modify: `crates/trusted-server-core/Cargo.toml` + - Remove `log-fastly` from core dependencies. +- Modify: `Cargo.lock` + - Lockfile update after dependency removal. + +The plan intentionally avoids any core logging trait or shared abstraction. Future adapters can mirror the same adapter-local module shape without forcing a premature common interface. + +## Tasks + +### Task 1: Extract Fastly logger helper and initializer into an adapter-local module + +**Files:** + +- Create: `crates/trusted-server-adapter-fastly/src/logging.rs` + +- [ ] **Step 1: Write a failing unit test for a non-allocating formatting helper** + +Create `crates/trusted-server-adapter-fastly/src/logging.rs` with a test-first skeleton. Add a helper test for the target-label extraction logic without trying to install a global logger: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn target_label_uses_last_target_segment() { + assert_eq!( + target_label("trusted_server_adapter_fastly::proxy"), + "proxy", + "should use the final target segment" + ); + } +} +``` + +Also add a production skeleton so the file compiles but the test fails: + +```rust +fn target_label(target: &str) -> &str { + target +} +``` + +- [ ] **Step 2: Run the adapter test to verify it fails** + +Run: + +```bash +cargo test --package trusted-server-adapter-fastly logging -- --nocapture +``` + +Expected: FAIL because `target_label()` returns the full target instead of the final segment. + +- [ ] **Step 3: Implement the minimal helper and adapter logger initializer** + +Replace the skeleton with the real adapter-local module: + +```rust +use chrono::{SecondsFormat, Utc}; +use log_fastly::Logger; + +fn target_label(target: &str) -> &str { + match target.rsplit_once("::") { + Some((head, "")) => head, + Some((_, last)) => last, + None => target, + } +} + +pub(crate) fn init_logger() { + let logger = Logger::builder() + .default_endpoint("tslog") + .echo_stdout(true) + .max_level(log::LevelFilter::Info) + .build() + .expect("should build Logger"); + + fern::Dispatch::new() + .format(|out, message, record| { + out.finish(format_args!( + "{} {} [{}] {}", + Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + record.level(), + target_label(record.target()), + message + )); + }) + .chain(Box::new(logger) as Box) + .apply() + .expect("should initialize logger"); +} +``` + +Keep the logic semantically equivalent to the current `main.rs` formatting and avoid introducing a new heap allocation on each log call. + +- [ ] **Step 4: Run the adapter test to verify it passes** + +Run: + +```bash +cargo test --package trusted-server-adapter-fastly logging -- --nocapture +``` + +Expected: PASS. + +- [ ] **Step 5: Commit the extracted logging module** + +```bash +git add crates/trusted-server-adapter-fastly/src/logging.rs +git commit -m "Extract Fastly logging initialization into adapter module" +``` + +--- + +### Task 2: Wire `main.rs` to the adapter-local logging module + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` + +- [ ] **Step 1: Write a failing compile-time integration step for the new module wiring** + +Update `main.rs` to reference `logging::init_logger()` before the module exists in the file: + +```rust +mod logging; + +#[fastly::main] +fn main(req: Request) -> Result { + logging::init_logger(); + // ... +} +``` + +Delete the old inline `init_logger()` function and remove imports that only it used: + +- `use log_fastly::Logger;` +- any `chrono`/`fern` imports that are no longer needed in `main.rs` + +- [ ] **Step 2: Run the adapter package tests to verify the extraction is wired correctly** + +Run: + +```bash +cargo test --package trusted-server-adapter-fastly -- --nocapture +``` + +Expected: PASS. If compilation fails, fix the module imports and remaining references in `main.rs`. + +- [ ] **Step 3: Commit the adapter wiring cleanup** + +```bash +git add crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Wire Fastly main.rs to adapter-local logging module" +``` + +--- + +### Task 3: Remove `log-fastly` from core + +**Files:** + +- Modify: `crates/trusted-server-core/Cargo.toml` +- Modify: `Cargo.lock` + +- [ ] **Step 1: Verify core does not reference `log-fastly` directly** + +Run: + +```bash +rg -n "log_fastly|Logger::builder|Logger::from_env" crates/trusted-server-core +``` + +Expected: no matches. + +- [ ] **Step 2: Remove `log-fastly` from core dependencies** + +In `crates/trusted-server-core/Cargo.toml`, delete: + +```toml +log-fastly = { workspace = true } +``` + +Do not remove `log = { workspace = true }`. + +- [ ] **Step 3: Update the lockfile** + +Run: + +```bash +cargo test --package trusted-server-core --lib --no-run +``` + +Expected: `Cargo.lock` updates only as needed for the dependency graph while core still compiles. + +- [ ] **Step 4: Confirm `log-fastly` remains adapter-only** + +Run: + +```bash +rg -n "log-fastly" crates +``` + +Expected: match only in `crates/trusted-server-adapter-fastly/Cargo.toml`. + +- [ ] **Step 5: Commit the dependency cleanup** + +```bash +git add crates/trusted-server-core/Cargo.toml Cargo.lock +git commit -m "Remove log-fastly from trusted-server-core" +``` + +--- + +### Task 4: Run project verification gates + +**Files:** + +- Verify the whole workspace after the logging extraction and dependency cleanup + +- [ ] **Step 1: Format check** + +Run: + +```bash +cargo fmt --all -- --check +``` + +Expected: PASS. If it fails, run `cargo fmt --all` and re-run the check. + +- [ ] **Step 2: Clippy** + +Run: + +```bash +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +Expected: PASS. + +- [ ] **Step 3: Full workspace tests** + +Run: + +```bash +cargo test --workspace +``` + +Expected: PASS. + +- [ ] **Step 4: Commit any formatting fallout** + +Only if `cargo fmt --all` changed files: + +```bash +git add -A +git commit -m "Fix formatting after logging extraction" +``` + +--- + +## Acceptance Checklist + +- [ ] `crates/trusted-server-adapter-fastly/src/logging.rs` exists +- [ ] `crates/trusted-server-adapter-fastly/src/main.rs` no longer contains the inline Fastly logger implementation +- [ ] `crates/trusted-server-core/Cargo.toml` no longer depends on `log-fastly` +- [ ] `rg -n "log-fastly" crates` reports only the Fastly adapter crate +- [ ] `trusted-server-core` still uses `log` macros and compiles without a Fastly-specific logging backend dependency +- [ ] `cargo fmt --all -- --check` passes +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` passes +- [ ] `cargo test --workspace` passes diff --git a/docs/superpowers/plans/2026-04-15-sourcepoint-gpp-consent.md b/docs/superpowers/plans/2026-04-15-sourcepoint-gpp-consent.md new file mode 100644 index 00000000..8c9de843 --- /dev/null +++ b/docs/superpowers/plans/2026-04-15-sourcepoint-gpp-consent.md @@ -0,0 +1,695 @@ +# Sourcepoint GPP Consent for Edge Cookie Generation — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Enable EC generation for sites using Sourcepoint by mirroring localStorage consent into cookies (client) and recognizing GPP US `sale_opt_out` as a consent signal (server). + +**Architecture:** New JS-only `sourcepoint` integration auto-discovers `_sp_user_consent_*` in localStorage and writes `__gpp` / `__gpp_sid` cookies. Server-side, `GppConsent` gains a `us_sale_opt_out: Option` field extracted from any GPP US section (IDs 7–23). `allows_ec_creation()` checks this field between the existing TCF and `us_privacy` branches. + +**Tech Stack:** TypeScript (Vitest, jsdom), Rust (iab_gpp crate for GPP section decoding) + +**Spec:** `docs/superpowers/specs/2026-04-15-sourcepoint-gpp-consent-design.md` + +--- + +## File Map + +| File | Action | Responsibility | +|---|---|---| +| `crates/trusted-server-core/src/consent/types.rs` | Modify | Add `us_sale_opt_out: Option` to `GppConsent` | +| `crates/trusted-server-core/src/consent/gpp.rs` | Modify | Decode US sections, extract `sale_opt_out` | +| `crates/trusted-server-core/src/consent/mod.rs` | Modify | Add GPP US branch in `allows_ec_creation()`, tests | +| `crates/js/lib/src/integrations/sourcepoint/index.ts` | Create | localStorage auto-discovery, cookie mirroring | +| `crates/js/lib/test/integrations/sourcepoint/index.test.ts` | Create | Vitest tests for cookie mirroring | + +--- + +## Task 1: Add `us_sale_opt_out` field to `GppConsent` + +**Files:** +- Modify: `crates/trusted-server-core/src/consent/types.rs:297-305` + +- [ ] **Step 1: Add the field** + +In `crates/trusted-server-core/src/consent/types.rs`, add `us_sale_opt_out` to `GppConsent`: + +```rust +/// Decoded GPP (Global Privacy Platform) consent data. +/// +/// Wraps the `iab_gpp` crate's decoded output with our domain types. +#[derive(Debug, Clone)] +pub struct GppConsent { + /// GPP header version. + pub version: u8, + /// Active section IDs present in the GPP string. + pub section_ids: Vec, + /// Decoded EU TCF v2.2 section (if present in GPP, section ID 2). + pub eu_tcf: Option, + /// Whether the user opted out of sale of personal information via a US GPP + /// section (IDs 7–23). + /// + /// - `Some(true)` — a US section is present and `sale_opt_out == OptedOut` + /// - `Some(false)` — a US section is present and user did not opt out + /// - `None` — no US section exists in the GPP string + pub us_sale_opt_out: Option, +} +``` + +- [ ] **Step 2: Fix compilation — update all `GppConsent` construction sites** + +There are existing places that construct `GppConsent`. Each needs the new field. Search for them: + +In `crates/trusted-server-core/src/consent/gpp.rs` (~line 74), update `decode_gpp_string`: + +```rust + Ok(GppConsent { + version: 1, + section_ids, + eu_tcf, + us_sale_opt_out: None, // placeholder — Task 2 fills this in + }) +``` + +In `crates/trusted-server-core/src/consent/mod.rs`, find every test that constructs `GppConsent` (search for `GppConsent {`). Add `us_sale_opt_out: None` to each. There are instances around lines 720, 883, and 965: + +```rust + gpp: Some(GppConsent { + version: 1, + section_ids: vec![2], + eu_tcf: Some(...), + us_sale_opt_out: None, + }), +``` + +- [ ] **Step 3: Verify compilation** + +Run: `cargo check --workspace` +Expected: compiles with no errors. + +- [ ] **Step 4: Run tests to confirm nothing broke** + +Run: `cargo test --workspace` +Expected: all existing tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/consent/types.rs \ + crates/trusted-server-core/src/consent/gpp.rs \ + crates/trusted-server-core/src/consent/mod.rs +git commit -m "Add us_sale_opt_out field to GppConsent" +``` + +--- + +## Task 2: Decode US sale opt-out from GPP sections + +**Files:** +- Modify: `crates/trusted-server-core/src/consent/gpp.rs` + +- [ ] **Step 1: Write the failing test for US sale opt-out extraction** + +Add to the `#[cfg(test)] mod tests` block in `crates/trusted-server-core/src/consent/gpp.rs`: + +```rust + // A GPP string with UsNat section (section ID 7). + // Header "DBABLA" encodes: version=1, section IDs=[7] (UsNat). + // The section string encodes a UsNat v1 core with sale_opt_out=DidNotOptOut (2). + #[test] + fn decodes_us_sale_opt_out_not_opted_out() { + // Build a real GPP string with UsNat section using iab_gpp parsing. + // "DBABLA~BVQqAAAAAgA.QA" is the example from the issue (Sourcepoint payload). + let result = decode_gpp_string("DBABLA~BVQqAAAAAgA.QA"); + match &result { + Ok(gpp) => { + assert_eq!( + gpp.us_sale_opt_out, + Some(false), + "should extract sale_opt_out=false from UsNat section" + ); + } + Err(e) => { + // If the specific GPP string doesn't parse, test with section ID presence. + // The important thing is that the decode_us_sale_opt_out function is wired up. + panic!("GPP decode failed: {e}"); + } + } + } + + #[test] + fn no_us_section_returns_none() { + // GPP_TCF_AND_USP has section IDs [2, 6] — no US sections (7–23). + let result = decode_gpp_string(GPP_TCF_AND_USP).expect("should decode GPP"); + assert_eq!( + result.us_sale_opt_out, None, + "should return None when no US section (7-23) is present" + ); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --workspace -p trusted-server-core -- consent::gpp::tests::decodes_us_sale_opt_out` +Expected: FAIL — `us_sale_opt_out` is hardcoded to `None`. + +- [ ] **Step 3: Implement `decode_us_sale_opt_out`** + +In `crates/trusted-server-core/src/consent/gpp.rs`, add after `decode_tcf_from_gpp`: + +```rust +/// GPP section IDs that represent US state/national privacy sections. +/// +/// Range 7–23 per the GPP v1 specification: +/// 7=UsNat, 8=UsCa, 9=UsVa, 10=UsCo, 11=UsUt, 12=UsCt, 13=UsFl, +/// 14=UsMt, 15=UsOr, 16=UsTx, 17=UsDe, 18=UsIa, 19=UsNe, 20=UsNh, +/// 21=UsNj, 22=UsTn, 23=UsMn. +const US_SECTION_ID_RANGE: std::ops::RangeInclusive = 7..=23; + +/// Extracts the `sale_opt_out` signal from the first US section in a parsed +/// GPP string. +/// +/// Iterates through section IDs looking for any in the US range (7–23). +/// For the first match, decodes the section and extracts `sale_opt_out`. +/// +/// Returns `Some(true)` if the user opted out of sale, `Some(false)` if they +/// did not, or `None` if no US section is present. +fn decode_us_sale_opt_out(parsed: &iab_gpp::v1::GPPString) -> Option { + use iab_gpp::sections::us_common::OptOut; + use iab_gpp::sections::Section; + + let us_section_id = parsed + .section_ids() + .find(|id| US_SECTION_ID_RANGE.contains(&(**id as u16)))?; + + match parsed.decode_section(*us_section_id) { + Ok(section) => { + let sale_opt_out = match §ion { + Section::UsNat(s) => match &s.core { + iab_gpp::sections::usnat::Core::V1(c) => &c.sale_opt_out, + iab_gpp::sections::usnat::Core::V2(c) => &c.sale_opt_out, + }, + Section::UsCa(s) => &s.core.sale_opt_out, + Section::UsVa(s) => &s.core.sale_opt_out, + Section::UsCo(s) => &s.core.sale_opt_out, + Section::UsUt(s) => &s.core.sale_opt_out, + Section::UsCt(s) => &s.core.sale_opt_out, + Section::UsFl(s) => &s.core.sale_opt_out, + Section::UsMt(s) => &s.core.sale_opt_out, + Section::UsOr(s) => &s.core.sale_opt_out, + Section::UsTx(s) => &s.core.sale_opt_out, + Section::UsDe(s) => &s.core.sale_opt_out, + Section::UsIa(s) => &s.core.sale_opt_out, + Section::UsNe(s) => &s.core.sale_opt_out, + Section::UsNh(s) => &s.core.sale_opt_out, + Section::UsNj(s) => &s.core.sale_opt_out, + Section::UsTn(s) => &s.core.sale_opt_out, + Section::UsMn(s) => &s.core.sale_opt_out, + // Non-US sections — should not reach here given the ID filter. + _ => return None, + }; + Some(*sale_opt_out == OptOut::OptedOut) + } + Err(e) => { + log::warn!("Failed to decode US GPP section {us_section_id}: {e}"); + None + } + } +} +``` + +- [ ] **Step 4: Wire it into `decode_gpp_string`** + +In the same file, replace the placeholder in `decode_gpp_string`: + +```rust + let us_sale_opt_out = decode_us_sale_opt_out(&parsed); + + Ok(GppConsent { + version: 1, + section_ids, + eu_tcf, + us_sale_opt_out, + }) +``` + +- [ ] **Step 5: Run tests** + +Run: `cargo test --workspace -p trusted-server-core -- consent::gpp::tests` +Expected: all GPP tests pass, including the two new ones. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/consent/gpp.rs +git commit -m "Decode US sale opt-out from GPP sections" +``` + +--- + +## Task 3: Add GPP US branch to `allows_ec_creation()` + +**Files:** +- Modify: `crates/trusted-server-core/src/consent/mod.rs` + +- [ ] **Step 1: Write failing tests** + +Add to the `#[cfg(test)] mod tests` block in `crates/trusted-server-core/src/consent/mod.rs`: + +```rust + #[test] + fn ec_allowed_us_state_gpp_no_sale_opt_out() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("TN".to_owned()), + gpp: Some(GppConsent { + version: 1, + section_ids: vec![7], + eu_tcf: None, + us_sale_opt_out: Some(false), + }), + ..ConsentContext::default() + }; + assert!( + allows_ec_creation(&ctx), + "US state + GPP US sale_opt_out=false should allow EC" + ); + } + + #[test] + fn ec_blocked_us_state_gpp_sale_opted_out() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("TN".to_owned()), + gpp: Some(GppConsent { + version: 1, + section_ids: vec![7], + eu_tcf: None, + us_sale_opt_out: Some(true), + }), + ..ConsentContext::default() + }; + assert!( + !allows_ec_creation(&ctx), + "US state + GPP US sale_opt_out=true should block EC" + ); + } + + #[test] + fn ec_blocked_us_state_gpc_overrides_gpp_us() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("TN".to_owned()), + gpc: true, + gpp: Some(GppConsent { + version: 1, + section_ids: vec![7], + eu_tcf: None, + us_sale_opt_out: Some(false), + }), + ..ConsentContext::default() + }; + assert!( + !allows_ec_creation(&ctx), + "GPC should block EC even when GPP US says no opt-out" + ); + } + + #[test] + fn ec_us_state_tcf_takes_priority_over_gpp_us() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("TN".to_owned()), + tcf: Some(make_tcf_with_storage(true)), + gpp: Some(GppConsent { + version: 1, + section_ids: vec![7], + eu_tcf: None, + us_sale_opt_out: Some(true), + }), + ..ConsentContext::default() + }; + assert!( + allows_ec_creation(&ctx), + "TCF consent should take priority over GPP US opt-out" + ); + } + + #[test] + fn ec_us_state_gpp_us_takes_priority_over_us_privacy() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("TN".to_owned()), + gpp: Some(GppConsent { + version: 1, + section_ids: vec![7], + eu_tcf: None, + us_sale_opt_out: Some(false), + }), + us_privacy: Some(UsPrivacy { + version: 1, + notice_given: PrivacyFlag::Yes, + opt_out_sale: PrivacyFlag::Yes, + lspa_covered: PrivacyFlag::NotApplicable, + }), + ..ConsentContext::default() + }; + assert!( + allows_ec_creation(&ctx), + "GPP US should take priority over us_privacy opt-out" + ); + } + + #[test] + fn ec_us_state_gpp_no_us_section_falls_through_to_us_privacy() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpp: Some(GppConsent { + version: 1, + section_ids: vec![2], + eu_tcf: None, + us_sale_opt_out: None, + }), + us_privacy: Some(UsPrivacy { + version: 1, + notice_given: PrivacyFlag::Yes, + opt_out_sale: PrivacyFlag::No, + lspa_covered: PrivacyFlag::NotApplicable, + }), + ..ConsentContext::default() + }; + assert!( + allows_ec_creation(&ctx), + "GPP without US section should fall through to us_privacy" + ); + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test --workspace -p trusted-server-core -- consent::tests::ec_allowed_us_state_gpp` +Expected: FAIL — the GPP US branch doesn't exist yet, so `ec_allowed_us_state_gpp_no_sale_opt_out` fails (falls through to fail-closed). + +- [ ] **Step 3: Add the GPP US branch to `allows_ec_creation()`** + +In `crates/trusted-server-core/src/consent/mod.rs`, update `allows_ec_creation()`. The `UsState` arm currently reads: + +```rust + jurisdiction::Jurisdiction::UsState(_) => { + if ctx.gpc { + return false; + } + if let Some(tcf) = effective_tcf(ctx) { + return tcf.has_storage_consent(); + } + if let Some(usp) = &ctx.us_privacy { + return usp.opt_out_sale != PrivacyFlag::Yes; + } + false + } +``` + +Insert the GPP US check between TCF and us_privacy: + +```rust + jurisdiction::Jurisdiction::UsState(_) => { + if ctx.gpc { + return false; + } + if let Some(tcf) = effective_tcf(ctx) { + return tcf.has_storage_consent(); + } + // Check GPP US section for sale opt-out. + if let Some(gpp) = &ctx.gpp { + if let Some(opted_out) = gpp.us_sale_opt_out { + return !opted_out; + } + } + if let Some(usp) = &ctx.us_privacy { + return usp.opt_out_sale != PrivacyFlag::Yes; + } + false + } +``` + +- [ ] **Step 4: Run all tests** + +Run: `cargo test --workspace` +Expected: all tests pass, including the six new EC gating tests. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/consent/mod.rs +git commit -m "Recognize GPP US sale opt-out in EC consent gating" +``` + +--- + +## Task 4: Create Sourcepoint JS integration + +**Files:** +- Create: `crates/js/lib/src/integrations/sourcepoint/index.ts` + +- [ ] **Step 1: Write the test file first** + +Create `crates/js/lib/test/integrations/sourcepoint/index.test.ts`: + +```typescript +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { mirrorSourcepointConsent } from '../../../src/integrations/sourcepoint'; + +describe('integrations/sourcepoint', () => { + beforeEach(() => { + // Clear cookies and localStorage before each test. + document.cookie.split(';').forEach((c) => { + const name = c.split('=')[0].trim(); + if (name) document.cookie = `${name}=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/`; + }); + localStorage.clear(); + }); + + afterEach(() => { + localStorage.clear(); + }); + + it('mirrors __gpp and __gpp_sid from _sp_user_consent_* localStorage', () => { + const payload = { + gppData: { + gppString: 'DBABLA~BVQqAAAAAgA.QA', + applicableSections: [7], + }, + }; + localStorage.setItem('_sp_user_consent_36026', JSON.stringify(payload)); + + const result = mirrorSourcepointConsent(); + + expect(result).toBe(true); + expect(document.cookie).toContain('__gpp=DBABLA~BVQqAAAAAgA.QA'); + expect(document.cookie).toContain('__gpp_sid=7'); + }); + + it('handles multiple applicable sections', () => { + const payload = { + gppData: { + gppString: 'DBABLA~BVQqAAAAAgA.QA', + applicableSections: [7, 8], + }, + }; + localStorage.setItem('_sp_user_consent_99999', JSON.stringify(payload)); + + mirrorSourcepointConsent(); + + expect(document.cookie).toContain('__gpp_sid=7,8'); + }); + + it('returns false when no _sp_user_consent_* key exists', () => { + localStorage.setItem('unrelated_key', 'value'); + + const result = mirrorSourcepointConsent(); + + expect(result).toBe(false); + expect(document.cookie).not.toContain('__gpp='); + expect(document.cookie).not.toContain('__gpp_sid='); + }); + + it('returns false for malformed JSON in localStorage', () => { + localStorage.setItem('_sp_user_consent_12345', 'not-json!!!'); + + const result = mirrorSourcepointConsent(); + + expect(result).toBe(false); + expect(document.cookie).not.toContain('__gpp='); + }); + + it('returns false when gppData is missing from payload', () => { + localStorage.setItem('_sp_user_consent_12345', JSON.stringify({ otherField: true })); + + const result = mirrorSourcepointConsent(); + + expect(result).toBe(false); + expect(document.cookie).not.toContain('__gpp='); + }); + + it('returns false when gppString is empty', () => { + const payload = { + gppData: { + gppString: '', + applicableSections: [7], + }, + }; + localStorage.setItem('_sp_user_consent_12345', JSON.stringify(payload)); + + const result = mirrorSourcepointConsent(); + + expect(result).toBe(false); + expect(document.cookie).not.toContain('__gpp='); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd crates/js/lib && npx vitest run test/integrations/sourcepoint/index.test.ts` +Expected: FAIL — module `../../../src/integrations/sourcepoint` does not exist. + +- [ ] **Step 3: Implement the integration** + +Create `crates/js/lib/src/integrations/sourcepoint/index.ts`: + +```typescript +import { log } from '../../core/log'; + +const SP_CONSENT_PREFIX = '_sp_user_consent_'; + +interface SourcepointGppData { + gppString: string; + applicableSections: number[]; +} + +interface SourcepointConsentPayload { + gppData?: SourcepointGppData; +} + +function findSourcepointConsent(): SourcepointConsentPayload | null { + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i); + if (!key?.startsWith(SP_CONSENT_PREFIX)) continue; + + const raw = localStorage.getItem(key); + if (!raw) continue; + + try { + return JSON.parse(raw) as SourcepointConsentPayload; + } catch { + log.debug('sourcepoint: failed to parse localStorage value', { key }); + return null; + } + } + return null; +} + +function writeCookie(name: string, value: string): void { + document.cookie = `${name}=${encodeURIComponent(value)}; path=/; SameSite=Lax`; +} + +/// Reads Sourcepoint consent from localStorage and mirrors it into +/// `__gpp` and `__gpp_sid` cookies for Trusted Server to read. +/// +/// Returns `true` if cookies were written, `false` otherwise. +export function mirrorSourcepointConsent(): boolean { + if (typeof localStorage === 'undefined' || typeof document === 'undefined') { + return false; + } + + const payload = findSourcepointConsent(); + if (!payload?.gppData) { + log.debug('sourcepoint: no GPP data found in localStorage'); + return false; + } + + const { gppString, applicableSections } = payload.gppData; + if (!gppString) { + log.debug('sourcepoint: gppString is empty'); + return false; + } + + writeCookie('__gpp', gppString); + + if (Array.isArray(applicableSections) && applicableSections.length > 0) { + writeCookie('__gpp_sid', applicableSections.join(',')); + } + + log.info('sourcepoint: mirrored GPP consent to cookies', { + gppLength: gppString.length, + sections: applicableSections, + }); + + return true; +} + +if (typeof window !== 'undefined') { + mirrorSourcepointConsent(); +} + +export default mirrorSourcepointConsent; +``` + +- [ ] **Step 4: Run tests** + +Run: `cd crates/js/lib && npx vitest run test/integrations/sourcepoint/index.test.ts` +Expected: all 6 tests pass. + +- [ ] **Step 5: Run the full JS test suite** + +Run: `cd crates/js/lib && npx vitest run` +Expected: all tests pass (existing + new). + +- [ ] **Step 6: Format** + +Run: `cd crates/js/lib && npm run format` +Expected: no formatting issues. + +- [ ] **Step 7: Commit** + +```bash +git add crates/js/lib/src/integrations/sourcepoint/index.ts \ + crates/js/lib/test/integrations/sourcepoint/index.test.ts +git commit -m "Add Sourcepoint JS integration for GPP consent cookie mirroring" +``` + +--- + +## Task 5: Final verification + +**Files:** None (verification only) + +- [ ] **Step 1: Build the JS bundles** + +Run: `cd crates/js/lib && node build-all.mjs` +Expected: builds successfully, `dist/tsjs-sourcepoint.js` appears in the output. + +- [ ] **Step 2: Full Rust build** + +Run: `cargo build --workspace` +Expected: compiles with no errors. + +- [ ] **Step 3: Full Rust test suite** + +Run: `cargo test --workspace` +Expected: all tests pass. + +- [ ] **Step 4: Clippy** + +Run: `cargo clippy --workspace --all-targets --all-features -- -D warnings` +Expected: no warnings. + +- [ ] **Step 5: Rust format check** + +Run: `cargo fmt --all -- --check` +Expected: no formatting issues. + +- [ ] **Step 6: Full JS test suite** + +Run: `cd crates/js/lib && npx vitest run` +Expected: all tests pass. + +- [ ] **Step 7: JS format check** + +Run: `cd crates/js/lib && npm run format` +Expected: no formatting issues. diff --git a/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md b/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md index 5203e5ff..e9bf5d16 100644 --- a/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md +++ b/docs/superpowers/specs/2026-03-11-production-readiness-report-design.md @@ -51,10 +51,10 @@ match config. ### C-2: Admin endpoints unprotected unless handler regex covers them -`/admin/keys/rotate` and `/admin/keys/deactivate` are always routed. The +`/_ts/admin/keys/rotate` and `/_ts/admin/keys/deactivate` are always routed. The `enforce_basic_auth` gate only triggers for paths that match a configured `handlers[].path` regex. The default config (`^/secure`) does not cover -`/admin/*`. An operator who doesn't add an explicit admin handler has +`/_ts/admin/*`. An operator who doesn't add an explicit admin handler has **publicly-accessible key rotation/deletion endpoints**. **Refs:** @@ -64,7 +64,7 @@ match config. - `settings.rs:381` -- `handlers` parsing - `trusted-server.toml:1` -- default handler only covers `^/secure` -**Recommendation:** Either hard-require auth for `/admin/*` paths regardless of +**Recommendation:** Either hard-require auth for `/_ts/admin/*` paths regardless of handler config, or validate at startup that an admin handler exists. --- diff --git a/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md b/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md index 0ae8c906..c48a947d 100644 --- a/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md +++ b/docs/superpowers/specs/2026-03-19-edgezero-migration-design.md @@ -46,7 +46,7 @@ These decisions are finalized and reflected in this plan: 2. **Migrate all integrations** including GPT and Google Tag Manager as first-class scope. 3. **Admin key routes must be supported on all adapters** — - `/admin/keys/rotate` and `/admin/keys/deactivate` are required on Fastly, + `/_ts/admin/keys/rotate` and `/_ts/admin/keys/deactivate` are required on Fastly, Axum, and Cloudflare (no disabled-route mode). 4. **Temporary Fastly compatibility adapter is required** — `compat.rs` lives in trusted-server during migration (created in PR 11, deleted in PR 15), @@ -1357,7 +1357,7 @@ Changes: - Local development without Viceroy - Mock stores for local KV/config/secret - Implement required admin key routes - (`/admin/keys/rotate`, `/admin/keys/deactivate`) — core signing logic + (`/_ts/admin/keys/rotate`, `/_ts/admin/keys/deactivate`) — core signing logic composes the Axum store primitives (local config/secret providers) - Add `.env.dev` or local config file for Axum-specific **non-secret** settings only (listen address, mock store paths, log level). @@ -1387,7 +1387,7 @@ Changes: - Construct `RuntimeServices` with Cloudflare-backed trait implementations - Wrangler configuration - Implement required admin key routes - (`/admin/keys/rotate`, `/admin/keys/deactivate`) — core signing logic + (`/_ts/admin/keys/rotate`, `/_ts/admin/keys/deactivate`) — core signing logic composes the Cloudflare store primitives (Workers API bindings) - Add `wrangler.toml` with bindings for KV, secrets, and config - Add integration tests: route smoke tests, admin key route tests, @@ -1421,7 +1421,7 @@ Changes: - Route parity validation for all routes currently in `crates/trusted-server-adapter-fastly/src/main.rs` (`/static/tsjs=*`, `/.well-known/trusted-server.json`, - `/verify-signature`, `/admin/keys/rotate`, `/admin/keys/deactivate`, + `/verify-signature`, `/_ts/admin/keys/rotate`, `/_ts/admin/keys/deactivate`, `/auction`, `/first-party/*`, integration routes, and publisher fallback) - Cross-adapter behavior parity tests (Fastly vs Axum vs Cloudflare) for: response status/body, required headers, cookie behavior, and request-signing diff --git a/docs/superpowers/specs/2026-03-24-ssc-prd-design.md b/docs/superpowers/specs/2026-03-24-ssc-prd-design.md index 7f88ef15..7ed0a6cc 100644 --- a/docs/superpowers/specs/2026-03-24-ssc-prd-design.md +++ b/docs/superpowers/specs/2026-03-24-ssc-prd-design.md @@ -68,8 +68,8 @@ Today, regular cookies don't suffice for publisher and partner needs. Additional - Implement real-time consent withdrawal: delete cookie and KV entry when consent is revoked - Build a server-side identity graph in Fastly KV Store that accumulates resolved partner IDs over time - Provide three KV write paths: real-time pixel sync redirects, S2S batch push from partners, and TS-initiated S2S pull from partner resolution endpoints -- Expose two bidstream integration modes: header decoration (`/identify`) and full auction orchestration (`/auction`) -- Expose a publisher-authenticated `/admin/partners/register` endpoint for partner provisioning without direct KV access +- Expose two bidstream integration modes: header decoration (`/_ts/api/v1/identify`) and full auction orchestration (`/auction`) +- Expose a publisher-authenticated `/_ts/admin/v1/partners/register` endpoint for partner provisioning without direct KV access ### Non-Goals @@ -116,9 +116,9 @@ TS Lite is a runtime configuration of the existing Trusted Server binary. It is | `GET /first-party/proxy-rebuild` | Enabled | Disabled | | HTML injection pipeline | Enabled | Disabled | | GTM integration | Enabled | Disabled | -| `GET /sync` | Disabled | **Enabled** | -| `GET /identify` | Disabled | **Enabled** | -| `POST /api/v1/sync` | Disabled | **Enabled** | +| `GET /_ts/api/v1/sync` | Disabled | **Enabled** | +| `GET /_ts/api/v1/identify` | Disabled | **Enabled** | +| `POST /_ts/api/v1/batch-sync` | Disabled | **Enabled** | | `GET /.well-known/trusted-server.json` | Enabled | Enabled | When a disabled route is requested, TS returns `404` with the header `X-ts-error: feature-disabled`. @@ -337,19 +337,19 @@ The existing `counter_store` and `opid_store` settings (currently defined but un The EC cookie is deterministic (derived from IP + publisher salt) and lives in the browser. It does not depend on KV Store availability. KV Store holds identity enrichment only — resolved partner UIDs accumulated over time. The degraded behavior policy follows from this: **EC always works; enrichment degrades gracefully.** -| Operation | KV unavailable or error | Rationale | -| --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| EC cookie creation | Set the cookie. Skip the KV entry creation silently. Log the failure at `warn` level. | The cookie is the identity anchor — it does not require KV. The KV entry will be created on the next request once KV recovers. | -| EC cookie refresh (existing user) | Refresh the cookie. Skip the KV `last_seen` update silently. Log at `warn`. | Same as above — the cookie continues working. Stale `last_seen` is acceptable. | -| `/sync` KV write | Redirect to `return` with `ts_synced=0&ts_reason=write_failed`. | The browser redirect must not be blocked by KV availability. This case is already specified in Section 9.4. | -| `/identify` KV read | Return `200` with `ec` hash (from cookie) and `degraded: true`. Set `uids: {}` and `eids: []`. | The EC hash is still valid and useful for attribution and analytics. Empty uids signal that enrichment is unavailable, not that the user has no synced partners. `degraded: true` lets callers distinguish transient KV failure from a genuinely unenriched user. | -| S2S batch write (`/api/v1/sync`) | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | The request was valid; the failure is infrastructure. Partners should retry the batch. | -| S2S pull sync write (async) | Discard the resolved uid. Log at `warn`. Retry will occur on the next qualifying request per the `pull_sync_ttl_sec` window. | Async path — no user-facing impact. | -| Consent withdrawal KV delete | Expire the cookie immediately. Log the KV delete failure at `error` level. Retry the KV delete on the next request for this user. | Cookie deletion is the primary enforcement mechanism. KV delete failure must not block or delay the cookie expiry. | +| Operation | KV unavailable or error | Rationale | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| EC cookie creation | Set the cookie. Skip the KV entry creation silently. Log the failure at `warn` level. | The cookie is the identity anchor — it does not require KV. The KV entry will be created on the next request once KV recovers. | +| EC cookie refresh (existing user) | Refresh the cookie. Skip the KV `last_seen` update silently. Log at `warn`. | Same as above — the cookie continues working. Stale `last_seen` is acceptable. | +| `/_ts/api/v1/sync` KV write | Redirect to `return` with `ts_synced=0&ts_reason=write_failed`. | The browser redirect must not be blocked by KV availability. This case is already specified in Section 9.4. | +| `/_ts/api/v1/identify` KV read | Return `200` with `ec` hash (from cookie) and `degraded: true`. Set `uids: {}` and `eids: []`. | The EC hash is still valid and useful for attribution and analytics. Empty uids signal that enrichment is unavailable, not that the user has no synced partners. `degraded: true` lets callers distinguish transient KV failure from a genuinely unenriched user. | +| S2S batch write (`/_ts/api/v1/sync`) | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | The request was valid; the failure is infrastructure. Partners should retry the batch. | +| S2S pull sync write (async) | Discard the resolved uid. Log at `warn`. Retry will occur on the next qualifying request per the `pull_sync_ttl_sec` window. | Async path — no user-facing impact. | +| Consent withdrawal KV delete | Expire the cookie immediately. Log the KV delete failure at `error` level. Retry the KV delete on the next request for this user. | Cookie deletion is the primary enforcement mechanism. KV delete failure must not block or delay the cookie expiry. | -**`degraded: true` in `/identify` responses** +**`degraded: true` in `/_ts/api/v1/identify` responses** -When a KV read fails, the `/identify` response includes `"degraded": true` in the JSON body alongside an empty `uids` and `eids`. The `ec` field is still populated from the cookie. Callers should proceed with identity-only targeting (EC hash) and omit partner UID parameters from downstream requests. +When a KV read fails, the `/_ts/api/v1/identify` response includes `"degraded": true` in the JSON body alongside an empty `uids` and `eids`. The `ec` field is still populated from the cookie. Callers should proceed with identity-only targeting (EC hash) and omit partner UID parameters from downstream requests. ```json { @@ -461,7 +461,7 @@ This is the primary real-time write path for building the identity graph from ex ### 9.2 Endpoint ``` -GET /sync +GET /_ts/api/v1/sync ``` ### 9.3 Parameters @@ -505,7 +505,7 @@ Partners should treat `ts_synced=0` as a signal that the mapping was not stored. **Acceptance criteria:** -- [ ] `GET /sync?partner=ssp_x&uid=abc&return=https://sync.ssp.com/ack` returns a redirect to the `return` URL within 50ms (excluding KV write time) +- [ ] `GET /_ts/api/v1/sync?partner=ssp_x&uid=abc&return=https://sync.ssp.com/ack` returns a redirect to the `return` URL within 50ms (excluding KV write time) - [ ] KV entry for the EC hash contains `ids.ssp_x.uid = "abc"` after a successful sync; response redirects to `return` with `ts_synced=1` - [ ] If no `ts-ec` cookie is present, redirects to `return` with `ts_synced=0&ts_reason=no_ec`; no KV write performed - [ ] If consent is absent or invalid, redirects to `return` with `ts_synced=0&ts_reason=no_consent`; no KV write performed @@ -524,17 +524,17 @@ The S2S batch sync API allows partners to push ID mappings to Trusted Server in ### 10.2 Endpoint ``` -POST /api/v1/sync +POST /_ts/api/v1/batch-sync ``` ### 10.3 Authentication -Partners authenticate with a rotatable API key. Key rotation must not require redeploying the binary. Partner provisioning is handled via the `/admin/partners/register` endpoint (see Section 15, Open Questions). +Partners authenticate with a rotatable API key. Key rotation must not require redeploying the binary. Partner provisioning is handled via the `/_ts/admin/v1/partners/register` endpoint (see Section 15, Open Questions). ### 10.4 Request ``` -POST /api/v1/sync +POST /_ts/api/v1/batch-sync Content-Type: application/json Authorization: Bearer @@ -593,7 +593,7 @@ Before writing a mapping, Trusted Server checks the KV metadata for the given EC **Acceptance criteria:** -- [ ] `POST /api/v1/sync` with a valid Bearer token and a batch of up to 1000 mappings returns a response within 5 seconds +- [ ] `POST /_ts/api/v1/batch-sync` with a valid Bearer token and a batch of up to 1000 mappings returns a response within 5 seconds - [ ] Accepted mappings are written to the corresponding KV identity graph entries within 1 second - [ ] Mappings for unknown `ec_hash` values are rejected with `ec_hash_not_found` - [ ] Mappings for users with withdrawn consent are rejected with `consent_withdrawn` @@ -721,22 +721,22 @@ The following fields are added to the partner record schema (Section 13.3): Trusted Server exposes two modes for injecting EC identity into the bidstream. Publishers choose the mode that fits their existing ad stack. -### 12.2 Mode A: Identity resolution (`/identify`) +### 12.2 Mode A: Identity resolution (`/_ts/api/v1/identify`) -Trusted Server exposes `/identify` as a standalone identity resolution endpoint for callers that need EC identity and resolved partner UIDs outside of TS's own auction orchestration. TS builds the OpenRTB request in Mode B — `/identify` is not part of that path. It serves three distinct use cases: +Trusted Server exposes `/_ts/api/v1/identify` as a standalone identity resolution endpoint for callers that need EC identity and resolved partner UIDs outside of TS's own auction orchestration. TS builds the OpenRTB request in Mode B — `/_ts/api/v1/identify` is not part of that path. It serves three distinct use cases: **Use case 1 — Attribution and analytics** Any server-side or browser-side system that needs to tag an event, impression, or conversion with the user's EC hash. Examples: analytics pipelines, attribution platforms, reporting dashboards. **Use case 2 — Publisher ad server outbid context** -After TS's auction completes and winners are delivered to the publisher's ad server endpoint, the publisher's ad server may need EC identity and resolved partner UIDs to evaluate whether to accept the programmatic winner or outbid with a direct-sold placement. For this use case, TS includes the EC identity in the winner notification payload directly (see Section 12.3) — a separate `/identify` call is only needed if the publisher's ad server receives the winner through a path that does not carry TS headers. +After TS's auction completes and winners are delivered to the publisher's ad server endpoint, the publisher's ad server may need EC identity and resolved partner UIDs to evaluate whether to accept the programmatic winner or outbid with a direct-sold placement. For this use case, TS includes the EC identity in the winner notification payload directly (see Section 12.3) — a separate `/_ts/api/v1/identify` call is only needed if the publisher's ad server receives the winner through a path that does not carry TS headers. **Use case 3 — Client-side wrappers for non-TS SSPs** -Some SSPs run client-side header bidding wrappers (e.g., Amazon TAM, certain Index Exchange configurations) that do not participate in TS's server-side auction orchestration. A Prebid.js module or custom wrapper script calls `/identify` from the browser to obtain the EC hash and resolved partner UIDs, then injects those values into bid requests sent to those SSPs. This ensures non-TS demand sources bid with the same identity enrichment as TS-orchestrated bids, enabling a fair comparison at winner selection. +Some SSPs run client-side header bidding wrappers (e.g., Amazon TAM, certain Index Exchange configurations) that do not participate in TS's server-side auction orchestration. A Prebid.js module or custom wrapper script calls `/_ts/api/v1/identify` from the browser to obtain the EC hash and resolved partner UIDs, then injects those values into bid requests sent to those SSPs. This ensures non-TS demand sources bid with the same identity enrichment as TS-orchestrated bids, enabling a fair comparison at winner selection. -> **Prerequisite for use case 3:** For a non-TS SSP to receive a useful UID from `/identify`, that SSP must already be a registered partner in `partner_store` and must have a resolved uid in the KV identity graph for this user (via pixel sync, S2S batch, or S2S pull). Without a prior sync, `/identify` returns no uid for that partner. +> **Prerequisite for use case 3:** For a non-TS SSP to receive a useful UID from `/_ts/api/v1/identify`, that SSP must already be a registered partner in `partner_store` and must have a resolved uid in the KV identity graph for this user (via pixel sync, S2S batch, or S2S pull). Without a prior sync, `/_ts/api/v1/identify` returns no uid for that partner. -**Endpoint:** `GET /identify` +**Endpoint:** `GET /_ts/api/v1/identify` **When to call:** Once per auction event — not per-pageview. For use case 3, call before sending bid requests to non-TS SSPs. @@ -744,7 +744,7 @@ Some SSPs run client-side header bidding wrappers (e.g., Amazon TAM, certain Ind **Pattern 1 — Browser-direct (recommended for use cases 1 and 3)** -A script on the publisher's page calls `/identify` via `fetch()`. Because `ec.publisher.com` is same-site with the publisher's domain, the browser sends the `ts-ec` cookie and consent cookies automatically. No forwarding required. +A script on the publisher's page calls `/_ts/api/v1/identify` via `fetch()`. Because `ec.publisher.com` is same-site with the publisher's domain, the browser sends the `ts-ec` cookie and consent cookies automatically. No forwarding required. ```js const identity = await fetch('https://ec.publisher.com/identify').then((r) => @@ -773,7 +773,7 @@ A server-side caller must forward the following from the original browser reques #### Cookie and consent handling -`/identify` follows the EC retrieval priority from Section 6.4. It does not generate a new EC — if no EC is present, the response body contains `consent: denied` and empty identity fields. Consent is evaluated per Section 7.1. `/identify` never sets or modifies cookies. +`/_ts/api/v1/identify` follows the EC retrieval priority from Section 6.4. It does not generate a new EC — if no EC is present, the response body contains `consent: denied` and empty identity fields. Consent is evaluated per Section 7.1. `/_ts/api/v1/identify` never sets or modifies cookies. #### Response @@ -850,7 +850,7 @@ Trusted Server owns the full auction path in Mode B. TS builds the OpenRTB reque **EC context in winner notification to publisher's ad server:** -When TS delivers auction winners to the publisher's ad server endpoint, the response includes EC identity so the publisher's ad server has full context for its outbid decision without needing to call `/identify` separately: +When TS delivers auction winners to the publisher's ad server endpoint, the response includes EC identity so the publisher's ad server has full context for its outbid decision without needing to call `/_ts/api/v1/identify` separately: | Header | Value | | ----------------- | ------------------------------------------------------------ | @@ -895,21 +895,21 @@ Each partner registered in `partner_store` declares: ### 12.6 User stories -**As a publisher using Mode A for analytics/attribution**, I want to call `/identify` from a browser script so that I can tag events and impressions with the user's EC hash and resolved partner UIDs using URL parameters. +**As a publisher using Mode A for analytics/attribution**, I want to call `/_ts/api/v1/identify` from a browser script so that I can tag events and impressions with the user's EC hash and resolved partner UIDs using URL parameters. **Acceptance criteria:** -- [ ] `GET /identify` returns `200` with a valid JSON body within 30ms when EC is present and consent is valid +- [ ] `GET /_ts/api/v1/identify` returns `200` with a valid JSON body within 30ms when EC is present and consent is valid - [ ] `uids` object contains one key per partner with `bidstream_enabled: true` and a resolved UID; partners with no resolved UID are omitted - [ ] If consent is denied, response is `403 Forbidden` with body `{"consent": "denied"}` - [ ] If no EC is present, response is `204 No Content` with no body - [ ] Response headers `X-ts-ec`, `X-ts-eids`, `X-ts-`, and `X-ts-ec-consent` are present on `200` responses as supplementary signals -**As a publisher using a client-side wrapper for non-TS SSPs**, I want to call `/identify` from my Prebid.js configuration so that SSPs outside TS's auction receive the same identity enrichment as TS-orchestrated bids, enabling a fair winner comparison. +**As a publisher using a client-side wrapper for non-TS SSPs**, I want to call `/_ts/api/v1/identify` from my Prebid.js configuration so that SSPs outside TS's auction receive the same identity enrichment as TS-orchestrated bids, enabling a fair winner comparison. **Acceptance criteria:** -- [ ] `GET /identify` called from the browser returns resolved UIDs for all registered partners with a KV entry for this user +- [ ] `GET /_ts/api/v1/identify` called from the browser returns resolved UIDs for all registered partners with a KV entry for this user - [ ] A partner with no KV entry for this user is omitted from `uids` — no empty or null entries - [ ] Response is available within 30ms so it does not block Prebid.js auction timeout @@ -933,7 +933,7 @@ The following capabilities must be configurable without redeploying the binary: - **Publisher passphrase** — the HMAC key used for EC hash generation; same value across all of the publisher's domains; shared with trusted partners to form an identity-federated consortium - **Identity graph store** — the KV store backing the EC hash → identity graph - **Partner registry store** — the KV store backing partner configuration and API key validation -- **Partner records** — each partner's allowed sync domains, bidstream settings, pull sync configuration, and API credentials; managed via `/admin/partners/register` without redeployment +- **Partner records** — each partner's allowed sync domains, bidstream settings, pull sync configuration, and API credentials; managed via `/_ts/admin/v1/partners/register` without redeployment The exact configuration format (TOML keys, KV schema, JSON field names) is an engineering decision and will be documented in the technical design doc. @@ -945,11 +945,11 @@ The following documentation changes are required alongside the EC feature: - **Rename SyntheticID → Edge Cookie** across the entire `docs/` GitHub Pages site. The underlying concept is the same but the product name changes. - **New integration guides**, one per customer type: - - Publisher (full TS): enabling EC in `trusted-server.toml`, partner onboarding via `/admin/partners/register` + - Publisher (full TS): enabling EC in `trusted-server.toml`, partner onboarding via `/_ts/admin/v1/partners/register` - SSP: pixel sync integration guide, sync pixel URL format, callback handling, optional pull resolution endpoint - DSP: S2S batch API reference, authentication, conflict resolution behavior, optional pull resolution endpoint - Identity Provider: registering as a partner, `source_domain` and `openrtb_atype` configuration, sync patterns -- **API reference** for the four new endpoints: `GET /sync`, `GET /identify`, `POST /api/v1/sync`, and the partner-side pull resolution contract +- **API reference** for the four new endpoints: `GET /_ts/api/v1/sync`, `GET /_ts/api/v1/identify`, `POST /_ts/api/v1/batch-sync`, and the partner-side pull resolution contract - **Pull sync integration guide**: partner requirements for exposing a resolution endpoint, authentication, expected response shape, rate limit behavior - **Consent enforcement guide**: how TCF and GPP signals are read, precedence rules, what happens on withdrawal @@ -957,10 +957,10 @@ The following documentation changes are required alongside the EC feature: ## 15. Open Questions -| # | Question | Owner | Status | -| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | --------------------------------------------------------------------------- | -| 1 | Partner provisioning: TS will expose a `/admin/partners/register` endpoint authenticated at the publisher level (bearer token issued per publisher Fastly service), so publishers can onboard SSP/DSP partners without touching KV directly. Engineering to define the exact auth mechanism. | Engineering | **Resolved** — `/admin/partners/register` endpoint, publisher-authenticated | -| 2 | Should TS Lite expose a `GET /health` endpoint so partners can programmatically verify their service is running and their partner config is active in KV? | Product | **N/A** — TS Lite deferred (see Section 5) | +| # | Question | Owner | Status | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | --------------------------------------------------------------------------------------- | +| 1 | Partner provisioning: TS will expose a `/_ts/admin/v1/partners/register` endpoint authenticated at the publisher level, so publishers can onboard SSP/DSP partners without touching KV directly. | Engineering | **Resolved** — `/_ts/admin/v1/partners/register` endpoint protected by admin basic auth | +| 2 | Should TS Lite expose a `GET /health` endpoint so partners can programmatically verify their service is running and their partner config is active in KV? | Product | **N/A** — TS Lite deferred (see Section 5) | --- @@ -970,7 +970,7 @@ The following documentation changes are required alongside the EC feature: | ------------------------------- | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | | EC match rate (returning users) | >90% within 30 days | Fastly real-time logs: ratio of requests with existing `ts-ec` cookie vs. new EC generations | | Consent enforcement accuracy | 0 ECs created for opted-out EU/UK users | Log audit: verify no `ts-ec` `Set-Cookie` in responses where consent signal is absent | -| KV sync latency (pixel sync) | p99 <75ms end-to-end | Fastly log timing on `/sync` endpoint | +| KV sync latency (pixel sync) | p99 <75ms end-to-end | Fastly log timing on `/_ts/api/v1/sync` endpoint | | S2S batch API throughput | >500 mappings/sec sustained | Load test prior to partner onboarding | | S2S pull sync resolution rate | >30% of pull calls return a non-null uid within 60 days of first partner go-live | Fastly log: pull call outcomes per partner | | Identity graph fill rate | >50% of EC hashes with at least 1 resolved partner ID within 60 days of partner go-live | KV scan sample | diff --git a/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md b/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md index ad47c65d..f4e7a217 100644 --- a/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md +++ b/docs/superpowers/specs/2026-03-24-ssc-technical-spec-design.md @@ -3,7 +3,13 @@ **Status:** Draft **Author:** Engineering **PRD reference:** `docs/internal/ssc-prd.md` -**Last updated:** 2026-03-18 +**Last updated:** 2026-04-14 + +> **Supersession note (issue #666):** Sections in this historical design spec +> that describe a separate `consent_store` or consent KV fallback are obsolete. +> Current runtime behavior interprets live consent from request cookies, headers, +> geolocation, and policy defaults. `ec_identity_store` is the only KV-backed EC +> lifecycle store and holds identity graph state plus withdrawal tombstones. --- @@ -16,12 +22,13 @@ 5. [Cookie and Header Handling](#5-cookie-and-header-handling) 6. [Consent Enforcement](#6-consent-enforcement) 7. [KV Store Identity Graph](#7-kv-store-identity-graph) -8. [Pixel Sync Endpoint (`GET /sync`)](#8-pixel-sync-endpoint-get-sync) -9. [S2S Batch Sync API (`POST /api/v1/sync`)](#9-s2s-batch-sync-api-post-apiv1sync) + 7A. [Device Signals and Bot Gate](#7a-device-signals-and-bot-gate) +8. [Prebid EID Cookie Ingestion](#8-prebid-eid-cookie-ingestion) +9. [S2S Batch Sync API (`POST /_ts/api/v1/batch-sync`)](#9-s2s-batch-sync-api-post-apiv1sync) 10. [S2S Pull Sync (TS-Initiated)](#10-s2s-pull-sync-ts-initiated) -11. [Identity Resolution Endpoint (`GET /identify`)](#11-identity-resolution-endpoint-get-identify) +11. [Identity Resolution Endpoint (`GET /_ts/api/v1/identify`)](#11-identity-resolution-endpoint-get-identify) 12. [Bidstream Decoration (`/auction` Mode B)](#12-bidstream-decoration-auction-mode-b) -13. [Partner Registry and Admin Endpoint](#13-partner-registry-and-admin-endpoint) +13. [Partner Registry (Config-Based)](#13-partner-registry-config-based) 14. [Configuration](#14-configuration) 15. [Constants and Header Names](#15-constants-and-header-names) 16. [Error Handling](#16-error-handling) @@ -39,12 +46,12 @@ EC is the full replacement for SyntheticID. The PRD explicitly states backward c **Prerequisites (must be merged before this epic begins):** -- **SyntheticID → Edge Cookie rename** — [PR #479](https://github.com/IABTechLab/trusted-server/pull/479) renames SyntheticID to Edge Cookie (EC) across all code paths: `synthetic.rs` → `edge_cookie.rs`, `COOKIE_SYNTHETIC_ID` → `COOKIE_EC_ID`, `X-Synthetic-*` → `X-ts-ec`/`X-ts-ec-fresh` headers, `settings.synthetic` → `settings.edge_cookie`, and simplifies EC generation to IP-only HMAC-SHA256 (removing Handlebars templating). It also renames `ConsentPipelineInput.synthetic_id` to `ec_id`, updates consent KV helper parameters/docs, and handles consent-store key migration (old SyntheticID keys orphaned, TTL expiry cleans them up). **This PR must be merged before implementation of this spec begins.** The spec assumes a codebase where SyntheticID no longer exists. Verify before starting: +- **SyntheticID removal** — [PR #479](https://github.com/IABTechLab/trusted-server/pull/479) removes SyntheticID from all active code paths: `get_or_generate_synthetic_id()`, `COOKIE_SYNTHETIC_ID`, `X-Synthetic-*` headers, `synthetic.rs` module, `settings.synthetic` config, and all SyntheticID generation/cookie code from `publisher.rs`, `endpoints.rs`, and `registry.rs`. **This PR must be merged before implementation of this spec begins.** The spec assumes a codebase where SyntheticID no longer exists. Verify before starting: - `grep -r 'synthetic_id' crates/` returns no hits outside test fixtures - `grep -r 'X-Synthetic' crates/` returns no hits - `trusted-server.toml` has no `[synthetic]` section - - `ConsentPipelineInput` uses `ec_id`, not `synthetic_id` -- **Consent implementation** — The consent pipeline (`build_consent_context()`, `ConsentContext`, `allows_ec_creation()`, TCF/GPP/US-Privacy decoding) is implemented and available as a stable interface before this epic. PR `#380` merged to `main`. EC calls `allows_ec_creation()` directly — no new gating functions are introduced. Note: EC changes the _phase order_ relative to the old SyntheticID flow — consent is evaluated before EC generation, so first-visit consent KV persistence is deferred to the second request (see §6.1.1 for full analysis). + - `ConsentPipelineInput` uses `identity_key`, not `synthetic_id` +- **Consent implementation** — The consent pipeline (`build_consent_context()`, `ConsentContext`, `allows_ec_creation()`, TCF/GPP/US-Privacy decoding) is implemented and available as a stable interface before this epic. PR `#380` merged to `main`. EC calls `allows_ec_creation()` directly — no new gating functions are introduced. Consent is evaluated from live request cookies, headers, geolocation, and policy defaults before EC generation. **Deferred from this spec (not in scope):** @@ -66,8 +73,21 @@ Browser Request │ extract GeoInfo → enforce auth → route_request │ └──────────┬──────────────────────────────────────┘ │ -Two-phase model (matches existing codebase pattern): - +Phase 0 — bot gate (pure in-memory, no KV I/O): + ┌─────────────────────────────────────────────────┐ + │ derive_device_signals(req) │ + │ - UA → is_mobile, platform_class │ + │ - req.get_tls_ja4() → ja4_class (Section 1) │ + │ - req.get_client_h2_fingerprint() → h2_fp_hash │ + │ - (ja4_class, h2_fp_hash) → known_browser │ + │ │ + │ !looks_like_browser()? │ + │ → suppress KV graph (None), skip ec_finalize, │ + │ skip pull sync. Request still proxied to │ + │ origin — bot receives valid HTML but leaves │ + │ no trace in the identity graph. │ + └──────┬────────────────────────────────────────────┘ + │ Phase 1 — pre-routing (like `GeoInfo::from_request()`): ┌─────────────────────────────────────────┐ │ EcContext::read_from_request() │ @@ -75,6 +95,9 @@ Phase 1 — pre-routing (like `GeoInfo::from_request()`): │ - build_consent_context() → ConsentContext │ │ - allows_ec_creation(consent) │ │ No generation. No cookie writes. │ + │ │ + │ ec_context.set_device_signals(signals) │ + │ (passed through to KvEntry on creation) │ └──────┬──────────────────────────────────┘ │ Phase 2 — inside organic handlers only: @@ -84,20 +107,20 @@ Phase 2 — inside organic handlers only: handle_publisher_request() integration_registry.handle_proxy() calls ec_context.generate_if_needed() calls ec_context.generate_if_needed() -EC route handlers (GET /sync, GET /identify, POST /auction, -POST /api/v1/sync, POST /admin/*) NEVER call generate_if_needed(). -`/identify`, `/auction`, `POST /api/v1/sync`, and `POST /admin/*` -use `EcContext` in read-only form. `GET /sync` is the one exception: -it never bootstraps an EC, but it may replace `ec_context.consent` -with a locally-decoded fallback consent context for that request only -when the optional `consent` query param is the sole available signal. +EC route handlers (GET /_ts/api/v1/identify, POST /auction, +POST /_ts/api/v1/batch-sync) NEVER call generate_if_needed(). +`/_ts/api/v1/identify`, `/auction`, and `POST /_ts/api/v1/batch-sync` +use `EcContext` in read-only form. /auction reads EC identity but never bootstraps it — the publisher page-load path generates the EC before any auction request arrives. ec_finalize_response() — after every handler: - - consent withdrawn + cookie present? → clear_ec_on_response() + tombstone - - returning-user mismatch? → set_ec_on_response() [reconcile cookie to header EC] - - ec_generated == true? → set_ec_on_response() [new cookie only] + - !allows_ec_creation(&consent)? → strip EC response headers + - explicit withdrawal + cookie present? → also expire the cookie and write tombstones + - returning user with consent? → set x-ts-ec header only (no cookie/KV TTL refresh) + - ec_generated == true? → set EC cookie + x-ts-ec header + - Prebid EID ingestion: reads `ts-eids` cookie, matches source domains + via PartnerRegistry, writes changed partner UIDs to KV (same UID = no write) ``` EC state flows through an `EcContext` struct created once per request and passed through handlers. @@ -112,23 +135,28 @@ New files in `crates/trusted-server-core/src/`: crates/trusted-server-core/src/ ec/ mod.rs — EcContext, pub re-exports - identity.rs — EC generation (HMAC-SHA256, IP normalization) - cookie.rs — create_ec_cookie(), delete_ec_cookie(), set_ec_on_response() - finalize.rs — ec_finalize_response() (cookie write/delete, last_seen, tombstone) - kv.rs — KvIdentityGraph, read/write/delete identity entries - partner.rs — PartnerRecord, PartnerStore, load_partner() - sync_pixel.rs — handle_sync() handler - sync_batch.rs — handle_batch_sync() handler + generation.rs — EC generation (HMAC-SHA256, IP normalization) + cookies.rs — set_ec_cookie(), expire_ec_cookie() + consent.rs — EC consent gating helpers + device.rs — DeviceSignals derivation, UA/JA4/H2 parsing, known browser allowlist + eids.rs — OpenRTB EID construction helpers + finalize.rs — ec_finalize_response() (cookie write/delete, tombstone, EID ingestion) + kv.rs — KvIdentityGraph, read/write/delete identity entries, cluster evaluation + kv_types.rs — KvEntry, KvGeo, KvConsent, KvPubProperties, KvNetwork, KvDevice, KvMetadata + partner.rs — Partner validation helpers (ID format, API key hashing) + registry.rs — PartnerRegistry (in-memory, config-based, O(1) indexes) + rate_limiter.rs — RateLimiter trait and Fastly ERL implementation + prebid_eids.rs — ingest_prebid_eids() — ts-eids cookie parsing and KV sync + batch_sync.rs — handle_batch_sync() handler pull_sync.rs — PullSyncDispatcher, dispatch_background() identify.rs — handle_identify() handler - admin.rs — handle_register_partner() handler ``` Existing files modified: | File | Change | | -------------------------------------------------- | ----------------------------------------------------- | -| `crates/trusted-server-core/src/settings.rs` | Add `EdgeCookie` settings struct | +| `crates/trusted-server-core/src/settings.rs` | Add `Ec` and `EcPartner` settings structs | | `crates/trusted-server-core/src/constants.rs` | Add EC header/cookie name constants | | `crates/trusted-server-core/src/error.rs` | Add `EdgeCookie` error variant | | `crates/trusted-server-core/src/auction/` | Inject EC into `user.id`, `user.eids`, `user.consent` | @@ -138,7 +166,7 @@ Existing files modified: ## 4. EC Identity Generation -### 4.1 Module: `ec/identity.rs` +### 4.1 Module: `ec/generation.rs` The EC generation mirrors the SyntheticID approach (`synthetic.rs`) but strips volatile inputs. @@ -161,9 +189,12 @@ pub fn generate_ec(passphrase: &str, ip: IpAddr) -> Result String; -/// Extracts the stable 64-character hex prefix from a full EC value. +/// Extracts the stable 64-character hex prefix from a full EC ID. /// -/// The prefix is used as the KV store key. The `.suffix` is discarded. +/// This is primarily used for logging and debugging. Both the EC identity +/// EC identity KV operations use the **full EC ID** (including the +/// `.suffix`) as the key, not just this prefix. The suffix provides uniqueness +/// for users behind the same NAT/proxy infrastructure. /// /// Returns `None` if the value is not in `{64-hex}.{6-alnum}` format. pub fn ec_hash(ec_value: &str) -> Option<&str>; @@ -179,7 +210,7 @@ pub fn ec_hash(ec_value: &str) -> Option<&str>; **Output format:** `{64-char lowercase hex}.{6-char random alphanumeric}` -The random suffix is generated with `fastly::rand` (same approach as SyntheticID). Once set in a cookie the full value is preserved; only the hash prefix is used as the KV key. +The random suffix is generated with `fastly::rand` (same approach as SyntheticID). Once set in a cookie, the full value (hash + suffix) is preserved and used as the KV store key for the EC identity graph. The suffix provides uniqueness for users behind the same NAT/proxy who share the same IP-derived hash. **IPv6 /64 prefix:** Split on `:`, take first 4 groups, join with `:`. Example: `2001:db8:85a3:0000:0000:8a2e:0370:7334` → `2001:db8:85a3:0`. @@ -203,14 +234,16 @@ When both header and cookie are present, the **header wins** as `ec_value` (used - `ec_value` = header value (authoritative for handler reads) - `cookie_ec_value` = cookie value (tracked separately for withdrawal) -On consent **withdrawal** (`!allows_ec_creation && cookie_was_present`): +On **explicit consent withdrawal** (`has_explicit_ec_withdrawal(&consent) && cookie_was_present`): - Delete the browser cookie (always, based on `cookie_was_present`) - Tombstone the **cookie-derived** hash: `kv.write_withdrawal_tombstone(ec_hash(cookie_ec_value))` - If the header-derived hash differs, also tombstone it: `kv.write_withdrawal_tombstone(ec_hash(ec_value))` - This matches the existing SyntheticID behavior where revocation targets the cookie value (`publisher.rs:515`), not the header value. -On **non-withdrawal** paths (last_seen, handler reads): use `ec_value` (header-derived) as the active identity. When `cookie_ec_value` is set (mismatch), `ec_finalize_response()` overwrites the browser cookie with the header-derived `ec_value` via `set_ec_on_response()`. This reconciles the browser identity to match the publisher-forwarded identity and prevents persistent oscillation between two ECs on subsequent requests. +If `allows_ec_creation(&consent)` is `false` but there is **no explicit withdrawal signal** (for example, unknown jurisdiction or missing/undecodable consent in a regulated regime), the response strips EC-related headers only. It does **not** delete the cookie or write tombstones. + +On **non-withdrawal** paths (handler reads and response headers): use `ec_value` (header-derived) as the active identity. Returning-user responses set `x-ts-ec` for the active identity but do not refresh or repair the browser cookie. Cookie writes are reserved for newly generated ECs; cookie deletion is reserved for explicit consent withdrawal. **Validation:** Both the header and cookie values are validated independently via `ec_hash()` (`{64-hex}.{6-alnum}` format). If the header is present but malformed, it is discarded and the cookie value is used instead (if valid). A malformed header must not suppress a valid cookie — bad forwarding infrastructure should not break returning-user identity. `cookie_was_present` is set based on the raw cookie existing, regardless of validity — an invalid cookie value is still a cookie that needs to be cleared on withdrawal. @@ -220,18 +253,16 @@ Generation (step 3 above becoming a new EC) happens only inside organic handlers ```rust /// Per-request Edge Cookie state. Constructed pre-routing once per request. -/// Organic handlers call `generate_if_needed()` to mint new ECs. `/sync` is the -/// one EC route that may replace `consent` with a locally-decoded fallback for -/// the remainder of that request only. +/// Organic handlers call `generate_if_needed()` to mint new ECs. pub struct EcContext { - /// Full EC value (`hash.suffix`), if present on request or generated this request. + /// Full EC ID (`{64-hex}.{6-alnum}`), if present on request or generated this request. pub ec_value: Option, /// Whether the `ts-ec` **cookie** was present on the inbound request. /// This is the only field that gates consent-withdrawal cookie deletion — /// the PRD's delete branch is conditioned on the cookie, not on X-ts-ec header. pub cookie_was_present: bool, /// The cookie's EC value, if different from `ec_value` (header won priority). - /// Used only for withdrawal: tombstone targets the cookie-derived hash to match + /// Used only for withdrawal: tombstone targets the cookie-derived EC ID to match /// existing SyntheticID revocation behavior (`publisher.rs:515`). /// `None` when cookie absent or cookie == header value. pub cookie_ec_value: Option, @@ -249,6 +280,11 @@ pub struct EcContext { /// Stored here so pull sync can use it after `req` has been consumed by routing. /// `None` only if Fastly's `get_client_ip_addr()` returns `None`. pub client_ip: Option, + /// Device signals derived from TLS/H2/UA in the adapter layer. + /// Set via `set_device_signals()` after `read_from_request()` returns. + /// Converted to `KvDevice` and stored on new entries in `generate_if_needed()`. + /// `None` when the adapter does not provide signals (e.g., test environments). + pub device_signals: Option, } impl EcContext { @@ -256,16 +292,9 @@ impl EcContext { /// Does not write to the **EC identity KV store**. Called pre-routing, like /// `GeoInfo::from_request()` in the current `main.rs`. /// - /// Calls `build_consent_context()` with the EC hash (when present) passed - /// via `ConsentPipelineInput.ec_id` (renamed from `synthetic_id` - /// in PR #479). - /// - /// When an EC hash is available (returning user), this enables the consent - /// pipeline's KV fallback (read) and KV persistence (write to the - /// **consent** KV store). On a first visit (no EC cookie), `ec_hash` is - /// `None` and no consent KV interaction occurs; consent is evaluated purely - /// from request cookies/headers. This means consent is not persisted to - /// consent KV until the user's second request. See §6.1.1. + /// Calls `build_consent_context()` with request-local cookies, headers, + /// settings, and geo data. There is no separate consent KV fallback; live + /// consent is interpreted from the current request. pub fn read_from_request( req: &Request, settings: &Settings, @@ -292,19 +321,30 @@ impl EcContext { kv: &KvIdentityGraph, ); + /// Sets device signals derived from the adapter layer (TLS/H2/UA). + /// Must be called before `generate_if_needed()` so new entries include `KvDevice`. + pub fn set_device_signals(&mut self, signals: DeviceSignals); + + /// Returns the device signals, if set. + pub fn device_signals(&self) -> Option<&DeviceSignals>; + /// Returns the stable 64-char hex prefix, or `None` if no EC. + /// + /// Note: This extracts only the prefix for display/logging purposes. All KV + /// operations use the full EC ID (via `ec_value()`), not just this hash. pub fn ec_hash(&self) -> Option<&str>; } ``` -**`ec_finalize_response()` behavior** (signature: `ec_finalize_response(settings, geo, ec_context, kv, response)`): +**`ec_finalize_response()` behavior** (signature: `ec_finalize_response(settings, ec_context, kv, registry, eids_cookie, response)`): -1. If `!allows_ec_creation(&consent) && cookie_was_present`: call `clear_ec_on_response()` (deletes cookie **and** strips any handler-built `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, and `X-ts-` response headers) and write withdrawal tombstones for each valid known EC hash (cookie-derived and, when different, header-derived). This runs on **every route** — consent withdrawal is always real-time enforced. Keyed on `cookie_was_present`, not `ec_was_present`, because only a cookie-held EC can be deleted by the browser. When the cookie is malformed and there is no valid header-derived hash, no tombstone is written. -2. If `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)`: call `kv.update_last_seen()` (debounced). If `cookie_ec_value.is_some()`, also call `set_ec_on_response()` to reconcile the browser cookie to the authoritative header-derived EC. -3. If `ec_generated == true`: call `set_ec_on_response()` — sets `Set-Cookie` and `X-ts-ec`. KV create already happened inside `generate_if_needed()`; `ec_finalize_response()` does NOT write KV beyond tombstones and `last_seen`. -4. Handler-built response headers (`X-ts-ec`, `X-ts-eids` set directly by `/identify`) are preserved on non-withdrawal paths only. +1. If `!allows_ec_creation(&consent)`: call `clear_ec_headers_on_response()` to strip any handler-built `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, and `X-ts-` response headers. This runs on **every route**, including fail-closed cases where consent cannot be verified. +2. If `has_explicit_ec_withdrawal(&consent) && cookie_was_present`: additionally expire the cookie and write withdrawal tombstones for each valid known EC ID (cookie-derived and, when different, header-derived). Keyed on `cookie_was_present`, not `ec_was_present`, because only a cookie-held EC can be deleted by the browser. When the cookie is malformed and there is no valid header-derived EC ID, no tombstone is written. +3. If `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)`: ingest Prebid EIDs from the `ts-eids` cookie if present (see section 8) and set the `x-ts-ec` response header only. Ordinary returning-user requests do not refresh the EC cookie and do not write KV solely to extend TTL. +4. If `ec_generated == true`: set `Set-Cookie` and `X-ts-ec`. KV create already happened inside `generate_if_needed()`; `ec_finalize_response()` does NOT write KV beyond explicit-withdrawal tombstones and Prebid EID ingestion. Also ingest Prebid EIDs from the `ts-eids` cookie if present. +5. Handler-built response headers (`X-ts-ec` set directly by `/_ts/api/v1/identify`) are preserved only when consent currently allows EC. -**Note on `kv_degraded`:** Not on `EcContext` — `read_from_request()` does not read KV. Handlers track degraded state locally. `/identify` returns `degraded: true` in the JSON body on KV read failure; the auction handler treats a failed read as `eids: []`. +**Note on `kv_degraded`:** Not on `EcContext` — `read_from_request()` does not read KV. Handlers track degraded state locally. `/_ts/api/v1/identify` returns `degraded: true` in the JSON body on KV read failure; the auction handler treats a failed read as `eids: []`. ```` @@ -324,7 +364,7 @@ impl EcContext { | Max-Age | `31536000` (1 year) | | HttpOnly | No | -### 5.2 Module: `ec/cookie.rs` +### 5.2 Module: `ec/cookies.rs` The `cookie_domain` parameter passed to all functions below is computed as `format!(".{}", settings.publisher.domain)`. Do **not** use @@ -341,24 +381,52 @@ pub fn create_ec_cookie(ec_value: &str, cookie_domain: &str) -> String; pub fn delete_ec_cookie(cookie_domain: &str) -> String; // Sets Max-Age=0 with same Domain/Path/Secure/SameSite attributes. +/// Sets only the `X-ts-ec` response header on a response. +pub fn set_ec_header_on_response(response: &mut Response, ec_value: &str); + /// Sets the EC cookie and `X-ts-ec` response header on a response. -pub fn set_ec_on_response(response: &mut Response, ec_value: &str, cookie_domain: &str); +pub fn set_ec_cookie_and_header_on_response(response: &mut Response, ec_value: &str, cookie_domain: &str); /// Removes the EC cookie and strips all EC-related response headers: /// `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, -/// and any `X-ts-` headers. Called on consent withdrawal to -/// prevent leaking EC identity in handler-built headers. +/// and any `X-ts-` headers. Called on explicit consent +/// withdrawal to prevent leaking EC identity in handler-built headers. pub fn clear_ec_on_response(response: &mut Response, cookie_domain: &str); ```` ### 5.3 Response header -`X-ts-ec: {ec_hash.suffix}` is set by `set_ec_on_response()`, which is called by `ec_finalize_response()` in two cases: (1) `ec_generated == true` (new EC minted this request), or (2) `cookie_ec_value.is_some()` (header/cookie mismatch reconciliation — overwrites cookie to match header). It is also set explicitly by `/identify` and `/auction` handlers on their own response paths when an EC is present. It is **not** set on ordinary returning-user requests where the cookie already matches the header (or no header is present). +`X-ts-ec: {64-hex}.{6-alnum}` is set when an EC is available for the response. In current behavior, returning users (`ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)`) receive the header only; newly generated ECs (`ec_generated == true`) receive both the header and `Set-Cookie`. `/_ts/api/v1/identify` and `/auction` also set EC-related headers on their response paths. This header is added to `INTERNAL_HEADERS` in `constants.rs` so it is stripped before proxying to downstream backends, consistent with existing `X-ts-*` handling. ### 5.4 Per-request EC lifecycle +**Phase 0 — bot gate** (always runs, all routes — pure in-memory, no KV I/O): + +``` +derive_device_signals(req) + ua = req.get_header_str("user-agent") + ja4 = req.get_tls_ja4() // Fastly SDK — full JA4 hash + h2_fp = req.get_client_h2_fingerprint() // Fastly SDK — raw H2 SETTINGS string + + DeviceSignals::derive(ua, ja4, h2_fp) + is_mobile = parse_is_mobile(ua) // 0=desktop, 1=mobile, 2=unknown + ja4_class = extract_ja4_section1(ja4) // split on '_', take [0] + platform_class = parse_platform_class(ua) // mac/windows/ios/android/linux/None + h2_fp_hash = sha256(h2_fp)[..6].hex() // 12 hex chars + known_browser = evaluate_known_browser(ja4_class, h2_fp_hash) // allowlist match + + is_real_browser = looks_like_browser() // ja4_class.is_some() && platform_class.is_some() + + if !is_real_browser: + log::debug("Bot gate: blocking EC operations") + kv_graph = None // suppress all KV operations + // ec_finalize_response() will be skipped + // pull sync will be skipped + // request still proxied to origin normally +``` + **Phase 1 — pre-routing** (always runs, all routes): ``` @@ -371,12 +439,13 @@ EcContext::read_from_request() If neither valid: ec_value = None ec_was_present = ec_value.is_some() cookie_was_present = ts-ec cookie raw key exists (regardless of validity) - ec_hash = ec_value.as_deref().and_then(ec_hash) // None on first visit or malformed - build_consent_context(jar, req, config, geo, ec_hash) → consent: ConsentContext - // ec_hash is the identity key for consent KV (renamed from synthetic_id in PR #479). - // When ec_hash is Some: consent KV fallback read + consent KV write (to consent store, not EC store). - // When ec_hash is None (first visit): no consent KV interaction — cookies/headers only. + ec_id = ec_value.as_deref() // None on first visit or malformed + build_consent_context(jar, req, config, geo, ec_id) → consent: ConsentContext + // Consent is interpreted from request-local cookies, headers, settings, and geo. + // No separate consent KV fallback or persistence runs in the EC lifecycle. ec_generated = false + + ec_context.set_device_signals(device_signals) // for KvDevice on creation ``` **Phase 2 — inside organic handlers only** (`handle_publisher_request`, `handle_proxy`): @@ -389,41 +458,42 @@ ec_context.generate_if_needed(settings, &kv) // best-effort — never 500s → generate_ec(passphrase, ip) → ec_value = Some(new_ec) → ec_generated = true - → kv.create_or_revive(ec_hash, &entry) (best-effort, log warn if fails) + → kv.create_or_revive(new_ec, &entry) (best-effort, log warn if fails) // create_or_revive overwrites a tombstone (ok=false) on re-consent // no-ops if a live entry (ok=true) already exists ``` -**`ec_finalize_response(settings, geo, ec_context, &kv, response)` — always runs, all routes:** +**`ec_finalize_response(settings, geo, ec_context, &kv, response)` — runs only when `is_real_browser == true`:** ``` - ├── !allows_ec_creation(&consent) && cookie_was_present? - │ → clear_ec_on_response() (delete cookie + strip ALL EC headers from response) - │ → // Tombstone all known valid EC hashes. May be 0, 1, or 2 hashes. - │ if let Some(cookie_hash) = cookie_ec_value.and_then(|v| ec_hash(&v)): - │ kv.write_withdrawal_tombstone(cookie_hash) // cookie-derived hash - │ if let Some(header_hash) = ec_value.and_then(|v| ec_hash(&v)): - │ if Some(header_hash) != cookie_hash: - │ kv.write_withdrawal_tombstone(header_hash) // header-derived hash (if different) - │ // When cookie is malformed and no valid header exists: no tombstone written. - │ // Cookie deletion is still the authoritative enforcement mechanism. - │ // Tombstone fails? log error, do NOT block — no retry possible on browser path. + // Bot gate: when !looks_like_browser(), this entire block is skipped. + // The response is proxied to origin without any cookie writes or KV operations. + + ├── !allows_ec_creation(&consent)? + │ → clear_ec_headers_on_response() (strip ALL EC headers from response) + │ → has_explicit_ec_withdrawal(&consent) && cookie_was_present? + │ → expire_ec_cookie() + │ → // Tombstone all known valid EC IDs. May be 0, 1, or 2 IDs. + │ if let Some(cookie_ec_id) = cookie_ec_value.filter(|v| is_valid_ec_id(v)): + │ kv.write_withdrawal_tombstone(cookie_ec_id) // cookie-derived EC ID + │ if let Some(header_ec_id) = ec_value.filter(|v| is_valid_ec_id(v)): + │ if Some(header_ec_id) != cookie_ec_id: + │ kv.write_withdrawal_tombstone(header_ec_id) // header-derived EC ID (if different) + │ // When cookie is malformed and no valid header exists: no tombstone written. + │ // Cookie deletion is still the authoritative enforcement mechanism. + │ // Tombstone fails? log error, do NOT block — no retry possible on browser path. + │ → return │ ├── ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)? - │ → kv.update_last_seen(ec_hash, now()) (returning user — debounced at 300s) - │ → if cookie_ec_value.is_some(): - │ // Header and cookie disagree — reconcile by overwriting cookie with header value. - │ // Prevents persistent split identity where user oscillates between two ECs - │ // depending on whether the forwarded header is present on subsequent requests. - │ set_ec_on_response() (Set-Cookie with ec_value, the header-derived identity) + │ → set_ec_header_on_response() (returning user — no cookie/KV TTL refresh) │ └── ec_generated == true? - → set_ec_on_response() (Set-Cookie + X-ts-ec on response) + → set_ec_cookie_and_header_on_response() (Set-Cookie + X-ts-ec on response) ``` -EC route handlers (`GET /sync`, `GET /identify`, `POST /api/v1/sync`, `POST /admin/*`) never call `generate_if_needed()`. `ec_finalize_response()` will still delete the cookie on those routes if consent is withdrawn — that is intentional. +EC route handlers (`GET /_ts/api/v1/identify`, `POST /_ts/api/v1/batch-sync`) never call `generate_if_needed()`. `ec_finalize_response()` will still delete the cookie on those routes if consent is explicitly withdrawn — that is intentional. -**Cookie write rule:** `Set-Cookie` is written in exactly two cases: (1) `ec_generated == true` (first-time generation), or (2) `cookie_ec_value.is_some()` (header/cookie mismatch — reconcile cookie to match the header-derived identity). There is no cookie refresh or Max-Age reset on ordinary returning users where cookie already matches. The PRD defers a blanket refresh-on-every-request strategy to a future iteration. +**Cookie write rule:** `Set-Cookie` is written for newly generated ECs and consent-withdrawal deletion only. Ordinary returning requests set `x-ts-ec` but do not refresh the cookie `Max-Age`. --- @@ -442,41 +512,51 @@ Consent decoding shipped in `#380` (already merged). This spec treats the follow ### 6.1.1 EC consent gating EC reuses the existing `allows_ec_creation(&ConsentContext) -> bool` function -from the consent module (`consent/mod.rs`). No parallel gating function is -introduced — EC calls `allows_ec_creation()` directly for all consent decisions -(EC generation, withdrawal detection, sync gating). +from the consent module (`consent/mod.rs`) for EC generation, header emission, +and other "may this request use ECs right now?" decisions. -There is no EC-specific consent gate and no behavior change to -`allows_ec_creation()` in this spec. Shared consent-policy semantics stay in -the consent module; EC only consumes that existing decision. - -**Consent pipeline integration:** +Explicit withdrawal semantics use a separate +`has_explicit_ec_withdrawal(&ConsentContext) -> bool` helper. This narrower +signal distinguishes authoritative opt-outs from fail-closed cases where EC use +must be blocked for the current request but an already-issued EC must not be +revoked (for example, unknown jurisdiction or missing/undecodable consent in a +regulated regime). -`EcContext::read_from_request()` calls `build_consent_context()` with the EC hash as the identity key, passed via `ConsentPipelineInput.ec_id` (renamed from `synthetic_id` in PR #479). The consent pipeline's KV persistence and fallback behavior works with EC hashes: +There is no new consent source or KV lookup in this spec. Shared +consent-policy semantics stay in the consent module; EC consumes the existing +request-local decision plus the explicit-withdrawal helper. -- **Returning user** (EC cookie present → `ec_hash` is `Some`): consent KV fallback read is available when consent cookies are absent; consent KV write persists cookie-sourced consent for future requests. Note: `build_consent_context()` calls `try_kv_write()` internally, so phase 1 writes to the **consent** KV store (not the EC identity store). -- **First visit** (no EC cookie → `ec_hash` is `None`): no consent KV interaction. Consent is evaluated purely from request cookies/headers. The gap: consent is not persisted to consent KV on the first request. This is accepted — in regulated jurisdictions (GDPR, US state), consent cookies/headers must be present for `allows_ec_creation()` to return `true`, so there is always a signal to persist on the next request. In non-regulated jurisdictions, `allows_ec_creation()` returns `true` without consent signals, so there is nothing to persist anyway. Consent KV persistence begins on the second request when the EC cookie is present. +**Consent pipeline integration:** -**Consent store keying:** Old consent KV entries under SyntheticID keys become orphaned after PR #479 ships. New entries are keyed by EC hash. Orphaned entries expire via TTL — no explicit migration is performed. +`EcContext::read_from_request()` calls `build_consent_context()` with request-local cookies, headers, settings, and geo data. Current runtime behavior does not use a separate consent KV store or consent KV fallback. Consent is interpreted from live request signals on every request; the EC identity store only keeps the minimal `KvEntry.consent` snapshot and withdrawal tombstones for S2S enforcement. -**Rollout impact:** At cutover, returning users who relied on consent KV fallback (consent cookies absent, consent loaded from KV under SyntheticID key) will lose that fallback until a new EC-keyed consent entry is written on a subsequent request where consent cookies are present. This is a one-time window: once the EC cookie is set and a request with consent cookies arrives, the consent KV entry is written under the EC hash and fallback works again. The window duration depends on how quickly users return with consent cookies. This is accepted — consent cookies are the primary signal; KV fallback is a secondary mechanism for when cookies are blocked or absent. +All downstream EC logic uses `allows_ec_creation(&self.consent)` for creation/forwarding decisions and `has_explicit_ec_withdrawal(&self.consent)` for cookie-expiry/tombstone decisions. No consent decoding or KV-backed gating logic is added in this epic. -All downstream EC logic calls `allows_ec_creation(&self.consent)`. No consent decoding or gating logic is added in this epic. +### 6.2 Consent withdrawal — explicit delete path -### 6.2 Consent withdrawal — KV delete +When `allows_ec_creation(&consent)` returns `false`, Trusted Server **always** +strips EC-related response headers for that request. This covers both explicit +revocation and fail-closed cases. -When `allows_ec_creation(&consent)` returns `false` for a user whose **`ts-ec` cookie** is present (`cookie_was_present == true`). A user identified only by the `X-ts-ec` request header is not subject to cookie deletion — there is no cookie to expire. +Cookie expiry and tombstone writes happen only when +`has_explicit_ec_withdrawal(&consent)` returns `true` **and** the request +carried a **`ts-ec` cookie** (`cookie_was_present == true`). A user identified +only by the `X-ts-ec` request header is not subject to cookie deletion or +`tombstoning` on this path — there is no browser cookie to revoke. -1. Issue `Set-Cookie: ts-ec=; Max-Age=0; ...` and strip all EC response headers (synchronous — must not fail silently). This always happens when `cookie_was_present == true`. -2. Write tombstone for each valid EC hash available (`cookie_ec_value` and/or `ec_value`). When neither is valid (malformed cookie, no header), **no tombstone is written** — cookie deletion alone is the enforcement mechanism. When at least one valid hash exists: `kv.write_withdrawal_tombstone(hash)` sets `consent.ok = false`, clears partner IDs, TTL 24h — approximately 25ms per write. +1. Strip all EC response headers (synchronous — must not fail silently) whenever `!allows_ec_creation(&consent)`. +2. If `has_explicit_ec_withdrawal(&consent) && cookie_was_present == true`, issue `Set-Cookie: ts-ec=; Max-Age=0; ...`. +3. In that same explicit-withdrawal + cookie-present case, write a tombstone for each valid EC ID available (`cookie_ec_value` and/or `ec_value`). When neither is valid (malformed cookie, no header), **no tombstone is written** — cookie deletion alone is the browser-side enforcement mechanism. When at least one valid EC ID exists: `kv.write_withdrawal_tombstone(ec_id)` sets `consent.ok = false`, clears partner IDs, TTL 24h — approximately 25ms per write. -The tombstone write runs in the request path (not async) to ensure real-time enforcement. Using a tombstone rather than a hard delete preserves the `consent_withdrawn` signal for batch sync clients for 24 hours — otherwise batch sync cannot distinguish consent withdrawal from an EC that never existed. +The tombstone write runs in the request path (not async) to ensure real-time enforcement for authoritative withdrawals. Using a tombstone rather than a hard delete preserves the `consent_withdrawn` signal for batch sync clients for 24 hours — otherwise batch sync cannot distinguish consent withdrawal from an EC that never existed. If the tombstone write fails: -- Log at `error` level with EC hash -- Do not block the response — cookie deletion is the primary enforcement mechanism -- **No retry is possible on the browser path.** Once the cookie is deleted, subsequent browser requests carry no EC value (`ec_hash()` returns `None`), so there is no hash to tombstone. A failed tombstone means batch sync clients may see `ec_hash_not_found` (after TTL expiry) rather than `consent_withdrawn` — this is accepted degradation. The cookie deletion remains the authoritative enforcement mechanism. +- Log at `error` level with EC ID +- Do not block the response — cookie deletion is the primary enforcement mechanism on explicit-withdrawal paths +- **No retry is possible on the browser path.** Once the cookie is deleted, subsequent browser requests carry no EC value (`ec_value` returns `None`), so there is no EC ID to tombstone. A failed tombstone means batch sync clients may see `ec_id_not_found` (after TTL expiry) rather than `consent_withdrawn` — this is accepted degradation. + +Fail-closed / unverifiable-consent cases keep the cookie intact and do not write tombstones; they only suppress EC use on that request. --- @@ -484,37 +564,56 @@ If the tombstone write fails: ### 7.1 Module: `ec/kv.rs` -Two KV stores are used. Their names are configured in `trusted-server.toml`: +One KV store is used for the identity graph. Its name is configured in `trusted-server.toml`: -| Store | TOML key | Purpose | -| ---------------- | ------------------ | ---------------------------------- | -| Identity graph | `ec.ec_store` | EC hash → identity JSON | -| Partner registry | `ec.partner_store` | Partner ID → config + API key hash | +| Store | TOML key | Purpose | +| -------------- | ------------- | --------------------- | +| Identity graph | `ec.ec_store` | EC ID → identity JSON | + +Partners are defined in config (`[[ec.partners]]` in TOML) and loaded into an in-memory `PartnerRegistry` at startup. There is no KV-backed partner store. ### 7.2 Identity graph schema -**KV key:** 64-character hex hash (the stable prefix from `ec_value`, without `.suffix`). +**KV key:** Full EC ID in `{64-char hex}.{6-char alphanumeric}` format. The random suffix is intentionally included to provide uniqueness for users behind the same NAT/proxy infrastructure who would otherwise share identical IP-derived hash prefixes. **KV value (JSON, max ~5KB):** ```json { "v": 1, - "created": 1741824000, - "last_seen": 1741910400, + "created": 1775162556, "consent": { "tcf": "CP...", "gpp": "DBA...", "ok": true, - "updated": 1741910400 + "updated": 1775162556 }, "geo": { "country": "US", - "region": "CA" + "region": "TN", + "asn": 7922, + "dma": 659 + }, + "device": { + "is_mobile": 0, + "ja4_class": "t13d1516h2", + "platform_class": "mac", + "h2_fp_hash": "a3f9d21c8b04", + "known_browser": true + }, + "pub_properties": { + "origin_domain": "autoblog.com", + "seen_domains": { + "autoblog.com": { "visits": 1 } + } + }, + "network": { + "cluster_size": 2 }, "ids": { - "ssp_x": { "uid": "abc123", "synced": 1741824000 }, - "liveramp": { "uid": "LR_xyz", "synced": 1741890000 } + "id5": { "uid": "ID5*qe8VHv..." }, + "trade_desk": { "uid": "226fb4b3-..." }, + "liveramp_ats": { "uid": "Ag2z1TDA..." } } } ``` @@ -522,12 +621,19 @@ Two KV stores are used. Their names are configured in `trusted-server.toml`: **KV metadata (max 2048 bytes, readable without streaming body):** ```json -{ "ok": true, "country": "US", "v": 1 } +{ + "ok": true, + "country": "US", + "v": 1, + "cluster_size": 2, + "is_mobile": 0, + "known_browser": true +} ``` -The `ok` field in metadata is a **historical consent record for S2S consumers only** — it is set to `false` by `write_withdrawal_tombstone()` so that batch sync clients (`POST /api/v1/sync`) can return `consent_withdrawn` rather than `ec_hash_not_found` during the 24-hour tombstone TTL. +The `ok` field in metadata is a **historical consent record for S2S consumers only** — it is set to `false` by `write_withdrawal_tombstone()` so that batch sync clients (`POST /_ts/api/v1/batch-sync`) can return `consent_withdrawn` rather than `ec_id_not_found` during the 24-hour tombstone TTL. -**`consent.ok` is NOT used to make the withdrawal decision on the main request path.** Consent withdrawal is determined entirely from `allows_ec_creation(&ec_context.consent)` on the current request. When withdrawal is detected, the cookie is deleted and `write_withdrawal_tombstone()` is called in-path (setting `ok = false`, 24h TTL — see §6.2). Engineers must not add a KV read to the consent withdrawal hot path based on this field. +**`consent.ok` is NOT used to make the withdrawal decision on the main request path.** Withdrawal enforcement is driven by current request-local consent: `allows_ec_creation(&ec_context.consent)` decides whether EC use and EC response headers are allowed on this request, and `has_explicit_ec_withdrawal(&ec_context.consent)` decides whether to expire the cookie and call `write_withdrawal_tombstone()` in-path (setting `ok = false`, 24h TTL — see §6.2). Engineers must not add a KV read to the consent withdrawal hot path based on this field. **Rust types:** @@ -535,9 +641,18 @@ The `ok` field in metadata is a **historical consent record for S2S consumers on pub struct KvEntry { pub v: u8, pub created: u64, - pub last_seen: u64, pub consent: KvConsent, pub geo: KvGeo, + /// Creation-time publisher property metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pub_properties: Option, + /// Device class signals. Written once on creation — never updated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device: Option, + /// Network cluster disambiguation. Written only by /identify. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub ids: HashMap, } @@ -550,24 +665,86 @@ pub struct KvConsent { pub struct KvGeo { pub country: String, + #[serde(default, skip_serializing_if = "Option::is_none")] pub region: Option, + /// Autonomous System Number (e.g. 7922 = Comcast). + /// Primary signal for distinguishing home ISP vs. corporate VPN. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asn: Option, + /// DMA/metro code (e.g. 807 = San Francisco). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dma: Option, } pub struct KvPartnerId { pub uid: String, - pub synced: u64, +} + +/// Publisher property metadata captured when an EC entry is created. +pub struct KvPubProperties { + /// Apex domain where this EC entry was first created. + pub origin_domain: String, + /// Per-domain visit history, keyed by apex domain. + /// Capped at 50 entries; new domains silently dropped at cap. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub seen_domains: HashMap, +} + +pub struct KvDomainVisit { + /// Legacy visit count retained for schema compatibility. + pub visits: u32, +} + +/// Coarse device signals derived from TLS handshake and UA. +/// Written once on creation — never updated after. +pub struct KvDevice { + /// 0 = desktop, 1 = mobile, 2 = unknown (non-standard client). + pub is_mobile: u8, + /// JA4 Section 1 only (e.g. "t13d1516h2" = Chrome). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ja4_class: Option, + /// Coarse OS family: "mac", "windows", "ios", "android", "linux". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform_class: Option, + /// SHA256 prefix (12 hex chars) of H2 SETTINGS fingerprint. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub h2_fp_hash: Option, + /// true = known browser, false = known bot, None = unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub known_browser: Option, +} + +/// Network cluster disambiguation data. +/// Written only by /identify — too expensive for organic hot path. +pub struct KvNetwork { + /// Number of distinct EC suffixes sharing this hash prefix. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cluster_size: Option, } pub struct KvMetadata { pub ok: bool, pub country: String, pub v: u8, + /// Mirrors KvNetwork::cluster_size. None = not yet evaluated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cluster_size: Option, + /// Mirrors KvDevice::is_mobile. Enables propagation gating without body read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub is_mobile: Option, + /// Mirrors KvDevice::known_browser. Buyer-facing quality signal. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub known_browser: Option, } ``` +All new fields use `Option` types or `serde(default)`, so existing entries +deserialize without error. No schema version bump is needed — v1 has not +shipped yet. + ### 7.3 TTL -All KV writes use `time_to_live_sec = 31536000` (1 year), matching the cookie `Max-Age`. +New live entries use `time_to_live_sec = 31536000` (1 year), matching the initial cookie `Max-Age`. Ordinary returning-user page views do not refresh the EC cookie and do not write the KV entry solely to extend TTL. Real data mutations (for example, a changed partner UID or first cluster-size evaluation) still write the live entry with the live-entry TTL. Withdrawal tombstones use a 24-hour TTL. ### 7.4 Conflict resolution — atomic read-modify-write @@ -584,22 +761,34 @@ impl KvIdentityGraph { pub fn new(store_name: impl Into) -> Self; /// Reads the full entry, returning the generation marker for CAS writes. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn get( &self, - ec_hash: &str, + ec_id: &str, ) -> Result, Report>; /// Reads only the metadata fields (consent flag, country). + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn get_metadata( &self, - ec_hash: &str, + ec_id: &str, ) -> Result, Report>; /// Creates a new entry. Returns `Ok(())` if successful, `Err` if the key /// already exists (concurrent create) or on KV error. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn create( &self, - ec_hash: &str, + ec_id: &str, entry: &KvEntry, ) -> Result<(), Report>; @@ -615,9 +804,13 @@ impl KvIdentityGraph { /// Called by `generate_if_needed()` instead of `create()`. This ensures that /// re-consent recovery is immediate — a user who withdraws and then re-consents /// within the 24-hour tombstone window gets a fresh identity entry without delay. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn create_or_revive( &self, - ec_hash: &str, + ec_id: &str, entry: &KvEntry, ) -> Result<(), Report>; @@ -627,214 +820,491 @@ impl KvIdentityGraph { /// /// If the key does not exist, creates a minimal live entry first: /// `consent.ok = true`, `consent.tcf = None`, `consent.gpp = None`, - /// `created = synced`, `last_seen = synced`, `geo.country = "ZZ"`, - /// `geo.region = None`, and `ids = { partner_id: ... }`. + /// `created = now`, `geo.country = "ZZ"`, `geo.region = None`, + /// and `ids = { partner_id: ... }`. /// /// This recovery path is intentional: it materializes the graph later when /// the initial best-effort `create_or_revive()` on EC generation failed. /// Batch sync still performs its explicit existence/tombstone check before - /// calling this method, so `POST /api/v1/sync` retains its `ec_hash_not_found` + /// calling this method, so `POST /_ts/api/v1/batch-sync` retains its `ec_id_not_found` /// contract. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn upsert_partner_id( &self, - ec_hash: &str, + ec_id: &str, partner_id: &str, uid: &str, - synced: u64, ) -> Result<(), Report>; - /// Updates `last_seen` timestamp, but only if the stored value is more than - /// 300 seconds older than `timestamp`. This debounce prevents KV write - /// thrashing under bursty traffic — Fastly KV enforces a 1 write/sec limit - /// per key. Callers should log `warn` on failure and continue. - pub fn update_last_seen( + /// Upserts a partner ID only when the KV entry already exists. Used by + /// S2S batch sync. Returns `Unchanged` when the existing UID matches, + /// avoiding a KV write. Different UIDs overwrite the stored value; mapping + /// timestamps are not used for ordering because they are no longer stored + /// in the EC identity entry. + pub fn upsert_partner_id_if_exists( &self, - ec_hash: &str, - timestamp: u64, - ) -> Result<(), Report>; + ec_id: &str, + partner_id: &str, + uid: &str, + ) -> Result>; + + /// Counts the number of KV keys sharing a hash prefix via the list API. + /// Uses a single-page list with `limit(100)`. Returns the count, or + /// `None` if the list exceeds 100 keys (clearly a large network). + pub fn count_hash_prefix_keys( + &self, + hash_prefix: &str, + ) -> Result, Report>; + + /// Evaluates the network cluster size for an EC entry. + /// + /// Returns a stored `cluster_size` without a list call when present. If + /// missing, calls `count_hash_prefix_keys()` and writes the result to + /// `entry.network` via CAS. Returns the cluster size for inclusion in + /// the `/_ts/api/v1/identify` response. + pub fn evaluate_cluster( + &self, + ec_id: &str, + entry: &KvEntry, + generation: u64, + ) -> Result, Report>; /// Writes a withdrawal tombstone for consent enforcement. /// /// Instead of hard-deleting the KV entry, this overwrites it with /// `consent.ok = false`, clears all partner IDs, and sets a 24-hour TTL. - /// The tombstone allows batch sync clients (`POST /api/v1/sync`) to return - /// `consent_withdrawn` rather than `ec_hash_not_found` for the tombstone TTL. + /// The tombstone allows batch sync clients (`POST /_ts/api/v1/batch-sync`) to return + /// `consent_withdrawn` rather than `ec_id_not_found` for the tombstone TTL. /// /// After the 24-hour TTL expires, the entry is gone. Any subsequent `get()` - /// returns `None` (`ec_hash_not_found`) — the distinction is time-bounded. + /// returns `None` (`ec_id_not_found`) — the distinction is time-bounded. /// /// Caller must handle `Err` by logging at `error` level; the cookie deletion /// in `ec_finalize_response()` is the primary enforcement mechanism. + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. pub fn write_withdrawal_tombstone( &self, - ec_hash: &str, + ec_id: &str, ) -> Result<(), Report>; /// Hard-deletes the entry. Used only for data deletion requests (IAB deletion /// framework — deferred). For consent withdrawal, use `write_withdrawal_tombstone()`. - pub fn delete(&self, ec_hash: &str) -> Result<(), Report>; + /// + /// # Arguments + /// + /// * `ec_id` — The full EC ID (`{64-hex}.{6-alnum}`) used as the KV key. + pub fn delete(&self, ec_id: &str) -> Result<(), Report>; } ``` -`MAX_CAS_RETRIES = 3`. If all retries fail on a generation conflict, return `Err` — callers handle per-endpoint policy (§8.3 step 7 for pixel sync, §9.4 for batch sync). +`MAX_CAS_RETRIES = 5`. If all retries fail on a generation conflict, return `Err` — callers handle per-endpoint policy (§9.4 for batch sync, §8.4 for Prebid EID ingestion). ### 7.5 KV degraded behavior | Operation | KV unavailable | Action | | ---------------------------------- | -------------- | ---------------------------------------------------------------------------------------------- | | EC cookie creation | KV error | Set cookie. Skip KV create. Log `warn`. | -| `/sync` KV write | KV error | Redirect with `ts_synced=0&ts_reason=write_failed`. | -| `/identify` KV read | KV error | Return `200` with `ec` set, `degraded: true`, empty `uids`/`eids`. | -| `POST /api/v1/sync` | KV error | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | +| Prebid EID ingestion KV write | KV error | Skip write. Log `warn`. Retry on next qualifying request. | +| `/_ts/api/v1/identify` KV read | KV error | Return `200` with `ec` set, `degraded: true`, empty `uid`/`eid`. | +| `POST /_ts/api/v1/batch-sync` | KV error | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | | Pull sync KV write | KV error | Discard uid. Log `warn`. Retry on next qualifying request. | | Consent withdrawal tombstone write | KV error | Delete cookie (primary enforcement). Log `error`. Next request: no cookie → no EC regenerated. | --- -## 8. Pixel Sync Endpoint (`GET /sync`) +## 7A. Device Signals and Bot Gate + +### 7A.1 Overview + +Device signals provide coarse, non-PII browser classification derived from +the TLS handshake and User-Agent header at the Fastly edge. They serve two +purposes: + +1. **Bot gate** — block all KV identity operations for unrecognized clients + (bots, scrapers, non-standard HTTP clients). The request is still proxied + to the publisher origin normally — the bot receives valid HTML but leaves + no trace in the identity graph. +2. **Device class record** — store a write-once `KvDevice` on each EC entry + for future cross-browser propagation decisions and buyer-facing device + quality scoring. -### 8.1 Module: `ec/sync_pixel.rs` +All signal derivation is pure in-memory computation — no KV I/O. It runs on +every request before EC context creation. + +### 7A.2 Signal derivation + +No Client Hints are used — JA4 and UA platform parsing provide equivalent or +superior signal for every browser including Safari and Firefox, which do not +send Client Hints. + +**`is_mobile`** — derived in priority order: + +| Condition | Value | +| ---------------------------------------------- | -------------------------------------------------------------------------- | +| UA contains `iPhone`, `iPad`, or `Android` | `1` — confirmed mobile | +| UA contains `Macintosh`, `Windows`, or `Linux` | `0` — confirmed desktop | +| Neither pattern matches | `2` — genuinely unknown (rare; typically bots or heavily hardened clients) | + +Note: `is_mobile: 2` in practice signals a non-standard client rather than +Safari, since Safari always produces a recognizable UA platform string. + +**`platform_class`** — coarse OS family parsed from UA (checked in order): + +| UA segment | `platform_class` | +| ---------------------------------- | ---------------- | +| `iPhone` or `iPad` | `ios` | +| `Android` (checked before `Linux`) | `android` | +| `Macintosh` | `mac` | +| `Windows NT` | `windows` | +| `Linux` (non-Android) | `linux` | +| No match | `None` | + +**`ja4_class`** — Section 1 of the JA4 fingerprint only (e.g. `t13d1516h2`). +Available via `req.get_tls_ja4()` in the Fastly Compute Rust SDK. The full +JA4 format is `section1_section2_section3` separated by underscores; we split +on `_` and take `[0]`. Section 1 identifies browser family (cipher count, +extension count, ALPN) without uniquely fingerprinting a device. The full JA4 +is never stored. + +**`h2_fp_hash`** — first 12 hex characters of SHA256 of the raw HTTP/2 +SETTINGS fingerprint string, available via `req.get_client_h2_fingerprint()`. +Used alongside `ja4_class` to confirm browser family and detect bots. + +**`known_browser`** — set `true` when `ja4_class` + `h2_fp_hash` match a +known legitimate browser pattern from the allowlist below. `None` when +unknown. Both signals must be present for a match — if either is `None`, +returns `None`. + +### 7A.3 Known browser fingerprint allowlist + +Empirically derived from Fastly Compute production responses (2026-04-03): + +| Browser | `ja4_class` | `h2_fp` raw string | `known_browser` | +| ----------------------------------- | ------------ | -------------------------------- | --------------- | +| Chrome/Mac (v146) | `t13d1516h2` | `1:65536;2:0;4:6291456;6:262144` | `true` | +| Safari/Mac (v26) + Safari/iOS (v26) | `t13d2013h2` | `2:0;3:100;4:2097152` | `true` | +| Firefox/Mac (v149) | `t13d1717h2` | `1:65536;2:0;4:131072;5:16384` | `true` | + +Safari Mac and Safari iOS share identical TLS/H2 stacks — distinguished only +by `platform_class` (`mac` vs `ios`) and `is_mobile` (`0` vs `1`). + +This allowlist will expand as new browser versions are observed in production. +Entries not matching any allowlist row get `known_browser: None` (not `false`) +unless they match a confirmed bot pattern. + +The allowlist comparison works by hashing the known raw H2 SETTINGS strings +at evaluation time and comparing against the request's `h2_fp_hash`. The list +is small (3 entries) so the cost is negligible. + +### 7A.4 Bot gate behavior + +The bot gate checks for **signal presence** rather than matching against a +hardcoded fingerprint allowlist. Real browsers always produce a valid TLS +fingerprint (`ja4_class`) and a recognizable UA platform string +(`platform_class`). Raw HTTP clients (curl, Python requests, Go net/http, +headless scrapers) typically lack one or both. + +The gate uses `DeviceSignals::looks_like_browser()`: ```rust -pub async fn handle_sync( - settings: &Settings, - kv: &KvIdentityGraph, - partner_store: &PartnerStore, - req: &Request, - ec_context: &mut EcContext, -) -> Result>; +pub fn looks_like_browser(&self) -> bool { + self.ja4_class.is_some() && self.platform_class.is_some() +} +``` + +| Condition | EC operations | Example | +| ------------------------------------------------ | ------------- | -------------------------------- | +| `ja4_class` present AND `platform_class` present | **Allowed** | Any real browser on any OS | +| `ja4_class` missing OR `platform_class` missing | **Blocked** | curl, Python requests, Googlebot | + +`known_browser` (the fingerprint allowlist match) is still computed and stored +on `KvDevice` for analytics and future buyer-facing quality scoring, but it +does **not** gate identity operations. This avoids blocking legitimate browsers +whose JA4/H2 fingerprints are not yet in the allowlist. + +**Implementation in the Fastly adapter:** + +1. After `GeoInfo::from_request()`, call `derive_device_signals(req)` which + reads `User-Agent`, `req.get_tls_ja4()`, and + `req.get_client_h2_fingerprint()`. +2. If `!looks_like_browser()`: + - `kv_graph` is set to `None` (suppresses all KV reads and writes) + - `ec_finalize_response()` is skipped (no cookie set/deleted) + - Pull sync is skipped + - The request proceeds through normal routing — organic requests are + proxied to publisher origin, API endpoints respond normally (but + without EC identity data) +3. If `looks_like_browser()`: proceed normally. Device signals are set + on `EcContext` via `set_device_signals()` so they flow through to + `KvEntry` creation. + +**Current bot response:** the request is served normally (proxied to origin) +without any KV operations or cookie writes. The bot receives a valid HTML +response but leaves no trace in the identity graph. + +### 7A.5 `DeviceSignals` struct + +```rust +/// Device signals derived from a single request. +/// Computed in the Fastly adapter from raw TLS/H2/UA data. +pub struct DeviceSignals { + pub is_mobile: u8, + pub ja4_class: Option, + pub platform_class: Option, + pub h2_fp_hash: Option, + pub known_browser: Option, +} + +impl DeviceSignals { + /// Derives all device signals from raw request data. + pub fn derive(ua: &str, ja4: Option<&str>, h2_fp: Option<&str>) -> Self; + + /// Returns true when ja4_class and platform_class are both present. + /// Used by the bot gate — see §7A.4. + pub fn looks_like_browser(&self) -> bool; + + /// Converts to KvDevice for KV storage. + pub fn to_kv_device(&self) -> KvDevice; +} ``` -### 8.2 Query parameters +### 7A.6 `KvDevice` write policy + +`KvDevice` is written to `KvEntry.device` only during `generate_if_needed()` +(new EC creation). It is never updated after creation — device signals are a +first-seen record of how this EC entry was established. + +Existing entries (created before device signals were implemented) will have +`device: None`. Downstream consumers must handle `None` as "pre-device-signals +entry" rather than "unknown device." + +### 7A.7 Publisher property metadata (`KvPubProperties`) + +`KvPubProperties` records the publisher domain where the EC entry was created. +Earlier drafts treated `seen_domains` as mutable domain history, but the current +implementation avoids recurring organic-request KV writes. New entries seed only +the creation domain and runtime requests do not append domains or increment +visit counts. The `seen_domains`/`visits` shape remains for compatibility with +legacy records. + +```rust +pub struct KvPubProperties { + pub origin_domain: String, + pub seen_domains: HashMap, +} + +pub struct KvDomainVisit { + pub visits: u32, +} +``` + +**Written:** on `KvEntry::new()` / `create_or_revive()` for the creation domain +only. Ordinary returning-user requests do not update this structure. + +**Cap:** legacy `seen_domains` maps are capped at 50 entries +(`MAX_SEEN_DOMAINS`) during validation so old or malformed records cannot grow +unbounded. -| Parameter | Required | Description | -| --------- | -------- | ---------------------------------------------------------------------------- | -| `partner` | Yes | Partner ID — must exist in `partner_store` | -| `uid` | Yes | Partner's user ID for this user | -| `return` | Yes | Redirect-back URL (must match partner's `allowed_return_domains`) | -| `consent` | No | Fallback TCF/GPP string if `ec_context.consent.is_empty()` after pre-routing | +### 7A.8 Network cluster disambiguation (`KvNetwork`) -### 8.3 Flow +Tracks how many distinct EC entries share the same hash prefix. A high count +indicates a shared network (corporate VPN, campus); a low count indicates an +individual or household. +```rust +pub struct KvNetwork { + pub cluster_size: Option, +} ``` -1. Parse query params. Missing required params → 400. - -2. Require a valid cookie-held EC. - If `cookie_was_present == false` OR `ec_context.ec_hash().is_none()` - (cookie missing or malformed) → redirect to - {return}?ts_synced=0&ts_reason=no_ec - -3. Look up partner record in partner_store. - Not found → 400. - -4. Validate return URL host against partner.allowed_return_domains. - - Exact hostname match only — no suffix or wildcard. - - Mismatch → 400. - -5. Evaluate consent. Use `ec_context.consent` (built pre-routing via - `build_consent_context()`). The optional `consent` query param is a **fallback - only** — used solely when `ec_context.consent.is_empty()` returns `true`. - This is the actual contract from the consent module. It is broader than - “no cookies or headers on the wire”: if consent KV fallback, decoded objects, - GPP section IDs, AC string, raw US privacy, or GPC already populated the - context, `is_empty()` is `false` and the query param is ignored entirely. - - When the fallback applies: decode the query param into a **locally-built** - `ConsentContext` (same TCF/GPP/USP decoders, same jurisdiction inputs), then - assign that value into `ec_context.consent` for the remainder of this request. - This makes the sync write decision and `ec_finalize_response()` use the same - effective consent view, avoiding a same-request “write partner ID, then - withdraw EC” conflict. Do NOT re-call `build_consent_context()` — that would - trigger `try_kv_write()` and persist the query-param consent to the consent KV - store, which is not intended. The decoded fallback applies only to this `/sync` - request; it is not written to the consent KV store and does not change any - future request unless the client sends real consent cookies/headers again. - - `!allows_ec_creation(...)` → redirect to {return}?ts_synced=0&ts_reason=no_consent - -6. Check anti-stuffing rate limit (sync_rate_limit per EC hash per partner per hour). - Exceeded → `429 Too Many Requests` (no redirect — the `return` URL is never called). - -7. kv.upsert_partner_id(ec_hash, partner_id, uid, now()) - If the root KV entry is missing (e.g. initial `create_or_revive()` failed on - the organic page load), `upsert_partner_id()` creates a minimal live entry and - then writes `ids[partner_id]`. This is the recovery path for best-effort EC - creation misses. - KV write failure → redirect to {return}?ts_synced=0&ts_reason=write_failed - -8. Success → redirect to {return}?ts_synced=1 + +**Written:** only by the `/_ts/api/v1/identify` endpoint, never on the organic proxy path. +The prefix-match list API call required to compute `cluster_size` is too +expensive for the hot path. + +**Evaluation:** `evaluate_cluster()` on `KvIdentityGraph`: + +- Returns the stored `cluster_size` without a prefix-list call when present +- If `cluster_size` is missing, calls `count_hash_prefix_keys()` with `limit(100)` — a single list-page call +- Writes the computed result to `entry.network` via best-effort CAS +- `cluster_recheck_secs` is retained only as a legacy compatibility setting because no cluster-check timestamp is stored in the EC identity entry + +**Threshold guidance:** + +| Cluster size | Likely scenario | +| ------------ | ----------------------------------------- | +| 1–3 | Individual / household | +| 4–10 | Small shared space (family, small office) | +| 11–50 | Medium office, hotel, coworking | +| 50+ | Corporate VPN, university, campus | + +**Default trust threshold:** entries with `cluster_size <= 10` are treated as +individual users for identity resolution purposes. Configurable per publisher +via `trusted-server.toml`: + +```toml +[ec] +cluster_trust_threshold = 10 # default +# cluster_recheck_secs is legacy compatibility; cluster_size is computed once per entry ``` -`ts_synced` values: +### 7A.9 Geo extensions (`KvGeo`) + +`KvGeo` is extended with two non-PII network signals available from Fastly's +`geo_lookup()` on the client IP: -| Value | Meaning | -| ------------------------------------ | ----------------------------- | -| `ts_synced=1` | KV write succeeded | -| `ts_synced=0&ts_reason=no_ec` | No valid EC cookie present | -| `ts_synced=0&ts_reason=no_consent` | Consent absent or denied | -| `ts_synced=0&ts_reason=write_failed` | KV write failed after retries | +- **`asn: Option`** — Autonomous System Number (e.g. `7922` = Comcast). + Primary signal for distinguishing home ISP vs. corporate VPN. Populated from + `GeoInfo::asn` which reads `fastly::geo::Geo::as_number()`. A value of `0` + from the Fastly API is mapped to `None`. +- **`dma: Option`** — DMA/metro code (e.g. `807` = San Francisco). + Market-level targeting signal; not personal data. Populated from + `GeoInfo::metro_code` when non-zero. -Rate limit exceeded returns `429 Too Many Requests` directly — the partner's `return` URL is not called in this case. +Both fields are written on initial `KvEntry::new()` from `GeoInfo`. Never +updated after creation — geo is a first-seen signal, not a real-time one. -### 8.4 Return URL construction +### 7A.10 IP address storage policy -Append `ts_synced` (and optional `ts_reason`) to the `return` URL: +Raw IP addresses are personal data under GDPR (CJEU _Breyer v. Germany_, 2016) +and must not be stored in KV entries. The EC hash already derives from the IP +without persisting it. -- If the URL already has a query string, append `&ts_synced=...` -- If not, append `?ts_synced=...` +Permitted IP-derived signals (written at creation time): -Do not modify any other query parameters on the `return` URL. +- `geo.country` — ISO 3166-1 alpha-2 +- `geo.region` — ISO 3166-2 subdivision +- `geo.asn` — ASN number (network identifier, not personal data) +- `geo.dma` — DMA/metro code (market identifier, not personal data) -### 8.5 Security +### 7A.11 Privacy rationale -- `return` URL validated by exact hostname match against `partner.allowed_return_domains`. No subdomain wildcard matching. -- No HMAC signature required on inbound sync request. -- Rate limit: `partner.sync_rate_limit` writes per EC hash per partner per hour. Default: 100. Configurable per partner in `partner_store`. +`ja4_class` (Section 1 only) and `platform_class` are category signals, not +unique device identifiers. They are equivalent in precision to `geo.country` +— they identify a class of client, not an individual. The full JA4 fingerprint +(Sections 2 and 3) is never stored, as it approaches unique device +identification and would require explicit consent basis under GDPR Art. 4(1). --- -## 9. S2S Batch Sync API (`POST /api/v1/sync`) +## 8. Prebid EID Cookie Ingestion -### 9.1 Module: `ec/sync_batch.rs` +> **Note:** The pixel sync endpoint (`GET /_ts/api/v1/sync`) has been removed. Partner ID sync from the browser is now handled via the Prebid EID cookie, which is written client-side by the TSJS Prebid integration and ingested server-side in `ec_finalize_response()`. + +### 8.1 Module: `ec/prebid_eids.rs` ```rust -pub async fn handle_batch_sync( - settings: &Settings, +/// Parses a `ts-eids` cookie value and writes matched partner UIDs to KV. +/// +/// Best-effort: all errors are logged and swallowed so the main request +/// path is never affected. +pub fn ingest_prebid_eids( + cookie_value: &str, + ec_id: &str, kv: &KvIdentityGraph, - partner_store: &PartnerStore, + registry: &PartnerRegistry, +); +``` + +### 8.2 Cookie format + +| Attribute | Value | +| ---------- | -------------------------------------------------------------------------------------------- | +| Name | `ts-eids` | +| Format | Base64-encoded (standard RFC 4648) JSON array of OpenRTB-style EIDs (`{source, uids:[...]}`) | +| Max size | JS writer targets 3 KB; backend parser accepts up to 8 KiB raw cookie length | +| Written by | TSJS Prebid integration (client-side JS) | +| Read by | `ec_finalize_response()` (server-side, via `ingest_prebid_eids()`) | + +**Example decoded value:** + +```json +[ + { + "source": "uidapi.com", + "uids": [{ "id": "A4A...", "atype": 3 }] + }, + { + "source": "liveramp.com", + "uids": [{ "id": "LR_xyz", "atype": 3 }] + } +] +``` + +### 8.3 JS side + +The TSJS Prebid integration calls `pbjs.getUserIdsAsEids()` in the `bidsBackHandler` callback after each auction. The returned OpenRTB-style EID array is base64-encoded and written to the `ts-eids` cookie. This runs entirely client-side — no server round-trip is needed for the write. Current writers preserve the full `{source, uids:[...]}` shape; the backend remains backward-compatible with the earlier flattened `{source, id, atype}` payload during rollout. + +### 8.4 Backend side + +`ingest_prebid_eids()` is called from `ec_finalize_response()` on both returning-user and new-EC paths when a `ts-eids` cookie is present and consent is granted. The flow: + +1. Base64-decode the cookie value. +2. JSON-parse into OpenRTB-style `Eid` entries; if that parse fails, fall back to the earlier flattened `{source, id, atype}` payload for backward compatibility. +3. For each EID entry: + a. Look up `registry.find_by_source_domain(&eid.source)`. Skip if no match. + b. Find the first non-empty UID in `eid.uids`. Skip the source if none is present. + c. Skip oversized UID values. + d. Call `kv.upsert_partner_id(ec_id, &partner.id, &uid.id)`. The upsert skips the KV write when the stored UID already matches. +4. All errors are logged and swallowed — EID ingestion never blocks the response. + +### 8.5 Source domain matching + +Source domains are matched via `PartnerRegistry.find_by_source_domain()`, which performs a case-insensitive lookup against the `source_domain` field configured on each partner in `[[ec.partners]]`. The registry builds a `by_source_domain` HashMap at startup for O(1) lookups. + +### 8.6 Write suppression + +EC identity entries no longer store per-partner sync timestamps. Instead of a +time-based debounce, `upsert_partner_id()` skips the KV write when the stored UID +already matches the incoming UID. Different UIDs replace the stored value. + +--- + +## 9. S2S Batch Sync API (`POST /_ts/api/v1/batch-sync`) + +### 9.1 Module: `ec/batch_sync.rs` + +```rust +pub fn handle_batch_sync( + kv: &KvIdentityGraph, + registry: &PartnerRegistry, + rate_limiter: &dyn RateLimiter, req: Request, ) -> Result>; ``` ### 9.2 Authentication -`Authorization: Bearer ` header required. Auth flow: +`Authorization: Bearer ` header required. Auth flow: -1. Compute `sha256_hex(api_key)`. -2. Look up `partner_store.find_by_api_key_hash(hash)` — uses the `apikey:{hash}` secondary index (§13.1) for O(1) lookup instead of scanning all partners. -3. If the index returns a partner, verify the partner's stored `api_key_hash` matches the computed hash (constant-time comparison). This guards against stale index entries from key rotation. -4. If no match or verification fails → `401 Unauthorized` with no body processing. -5. If KV lookup fails (store unavailable) → `503 Service Unavailable`. +1. Compute `sha256_hex(api_token)`. +2. Look up `registry.find_by_api_key_hash(hash)` — the `PartnerRegistry` maintains a `by_api_key_hash` HashMap built at startup from `[[ec.partners]]` config for O(1) lookup. +3. If no match → `401 Unauthorized` with no body processing. -Key rotation does not require binary redeployment — partners update via `/admin/partners/register`, which handles old API-key index cleanup (§13.1). +Key rotation requires updating the `api_token` in `[[ec.partners]]` TOML and redeploying. ### 9.2.1 API-key rate limiting -After successful auth, check the API-key level rate limit: `partner.batch_rate_limit` requests per partner per minute (default 60). Uses the same Fastly rate-limiting API as pixel sync (§14.3), with key `batch:{partner_id}`. +After successful auth, check the API-key level rate limit: `partner.batch_rate_limit` requests per partner per minute (default 60). Uses Fastly's Edge Rate Limiting API (§14.3), with key `batch:{partner_id}`. Exceeded → `429 Too Many Requests` with body `{ "error": "rate_limit_exceeded" }`. No mappings are processed. ### 9.3 Request format ``` -POST /api/v1/sync +POST /_ts/api/v1/batch-sync Content-Type: application/json Authorization: Bearer { "mappings": [ { - "ec_hash": "<64-character hex hash>", + "ec_id": "", "partner_uid": "abc123", "timestamp": 1741824000 } @@ -846,13 +1316,13 @@ Maximum batch size: 1000 mappings. Requests exceeding this receive `400 Bad Requ ### 9.4 Processing -The authenticated partner's ID (from the `PartnerRecord` resolved via API key in §9.2) determines the `ids[partner_id]` namespace for all writes in this batch. A partner can only write to their own namespace. +The authenticated partner's ID (from the `PartnerConfig` resolved via API key hash in §9.2) determines the `ids[partner_id]` namespace for all writes in this batch. A partner can only write to their own namespace. For each mapping: -1. Validate `ec_hash` format (must be exactly 64 lowercase hex characters). Invalid format → reject with `reason: "invalid_ec_hash"`. -2. Read KV metadata for `ec_hash`. If not found → reject with `reason: "ec_hash_not_found"`. If `consent.ok = false` → reject with `reason: "consent_withdrawn"`. -3. `kv.upsert_partner_id(ec_hash, partner_id, partner_uid, timestamp)`. The upsert internally skips the write if the existing `ids[partner_id].synced ≥ timestamp` (idempotent — counted as accepted, no error). On KV failure → reject all remaining mappings with `reason: "kv_unavailable"`, return `207`. +1. Validate `ec_id` format (must match `{64-hex}.{6-alnum}` pattern). Invalid format → reject with `reason: "invalid_ec_id"`. +2. Read KV metadata for `ec_id`. If not found → reject with `reason: "ec_id_not_found"`. If `consent.ok = false` → reject with `reason: "consent_withdrawn"`. +3. `kv.upsert_partner_id_if_exists(ec_id, partner_id, partner_uid)`. Mapping `timestamp` is retained for API compatibility but is not used for ordering. The upsert skips the write if the existing UID already matches (counted as accepted). A different UID overwrites the stored value. On KV failure → reject all remaining mappings with `reason: "kv_unavailable"`, return `207`. ### 9.5 Response format @@ -861,7 +1331,7 @@ For each mapping: "accepted": 998, "rejected": 2, "errors": [ - { "index": 45, "reason": "ec_hash_not_found" }, + { "index": 45, "reason": "ec_id_not_found" }, { "index": 72, "reason": "consent_withdrawn" } ] } @@ -875,7 +1345,6 @@ HTTP status rules: | Some accepted, some rejected | `207 Multi-Status` | | All rejected (auth valid, batch valid) | `207 Multi-Status` with `accepted: 0` | | Auth invalid | `401 Unauthorized` | -| Auth KV lookup failed (store down) | `503 Service Unavailable` | | Malformed JSON or > 1000 mappings | `400 Bad Request` | | KV entirely unavailable | `207 Multi-Status`, all rejected with `kv_unavailable` | @@ -893,10 +1362,10 @@ pub struct BatchSyncError { #[derive(Debug, derive_more::Display)] pub enum BatchSyncRejection { - #[display("invalid_ec_hash")] - InvalidEcHash, - #[display("ec_hash_not_found")] - EcHashNotFound, + #[display("invalid_ec_id")] + InvalidEcId, + #[display("ec_id_not_found")] + EcIdNotFound, #[display("consent_withdrawn")] ConsentWithdrawn, #[display("kv_unavailable")] @@ -932,7 +1401,7 @@ impl PullSyncDispatcher { &self, ec_context: &EcContext, client_ip: IpAddr, - partners: &[PartnerRecord], + partners: &[&PartnerConfig], kv: &KvIdentityGraph, ); } @@ -940,9 +1409,9 @@ impl PullSyncDispatcher { /// Fires a single partner pull request via `send_async()`, waits for the /// response via `PendingRequest::wait()`, and writes the result to KV. fn pull_one_partner( - ec_hash: &str, + ec_id: &str, ip: IpAddr, - partner: &PartnerRecord, + partner: &PartnerConfig, kv: &KvIdentityGraph, ); ``` @@ -951,15 +1420,17 @@ fn pull_one_partner( A pull sync is dispatched for a partner when all of the following are true on a request: -1. The request was routed to an **organic handler** (`handle_publisher_request` or `integration_registry.handle_proxy`). Pull sync never fires on EC route handlers (`/sync`, `/identify`, `/api/v1/sync`, `/admin/*`) or `/auction`. This matches the PRD requirement that pull calls must not happen during the pixel sync flow. +1. The request was routed to an **organic handler** (`handle_publisher_request` or `integration_registry.handle_proxy`). Pull sync never fires on EC route handlers (`/_ts/api/v1/identify`, `/_ts/api/v1/batch-sync`) or `/auction`. 2. A valid EC is present (`ec_context.ec_hash().is_some()`). This includes an EC newly generated on the current organic request — pull sync may run immediately after first-page EC creation because the response cookie is flushed before the background dispatch starts. 3. `allows_ec_creation(&ec_context.consent) == true` 4. `partner.pull_sync_enabled == true` -5. Either: no entry exists for this partner in the KV graph, or the existing `synced` timestamp is older than `partner.pull_sync_ttl_sec` (default 86400 seconds) -6. Rate limit not exceeded: `partner.pull_sync_rate_limit` calls per EC hash per partner per hour (default 10) +5. The partner UID is missing from the KV graph. If `ids[partner_id]` is already present, pull sync is skipped. +6. Rate limit not exceeded: `partner.pull_sync_rate_limit` calls per EC ID per partner per hour (default 10) + +`partner.pull_sync_ttl_sec` is retained for configuration compatibility, but is not used by the current fill-missing-only behavior because EC entries no longer store per-partner sync timestamps. ### 10.3 Execution model @@ -967,18 +1438,18 @@ Pull calls are dispatched using Fastly's background task / `send_async` model af Maximum concurrent pull calls per request: `settings.ec.pull_sync_concurrency` (default 3). -**Architectural divergence from PRD:** The PRD describes excess partner calls being queued and dispatched on subsequent requests for the same user. A persistent queue is not implementable in the stateless Fastly WASM edge environment — there is no cross-request mutable state. This spec adapts the intent using a stateless rotating offset: sort qualifying partners by ID, then use `(unix_timestamp_secs / 3600) % partner_count` as the starting index (wrapping). This ensures different partners are prioritized across different requests without persisted state. Partners not called on a given request remain eligible on the next qualifying request per their `pull_sync_ttl_sec` condition. The practical outcome (all partners eventually called) matches the PRD intent; the mechanism differs due to the platform constraint. +**Architectural divergence from PRD:** The PRD describes excess partner calls being queued and dispatched on subsequent requests for the same user. A persistent queue is not implementable in the stateless Fastly WASM edge environment — there is no cross-request mutable state. This spec adapts the intent using a stateless rotating offset: sort qualifying partners by ID, then use `(unix_timestamp_secs / 3600) % partner_count` as the starting index (wrapping). This ensures different missing partners are prioritized across requests without persisted queue state. Once a partner UID is stored, that partner is no longer eligible for pull sync under the current fill-missing-only behavior. ### 10.4 Outbound request ``` -GET {partner.pull_sync_url}?ec_hash={64-char-hex}&ip={ip_address} +GET {partner.pull_sync_url}?ec_id={64-hex}.{6-alnum} Authorization: Bearer {partner.ts_pull_token} ``` -Before dispatching, `pull_sync.rs` validates that `pull_sync_url`'s hostname is present in `partner.pull_sync_allowed_domains`. If not, the call is skipped and an `error` is logged — this is a configuration error that should not occur at runtime if admin validation is working correctly (§13.2 step 3). +Before dispatching, `pull_sync.rs` validates that `pull_sync_url`'s hostname is present in `partner.pull_sync_allowed_domains`. If not, the call is skipped and an `error` is logged — this is a configuration error that should not occur at runtime if startup validation in `PartnerRegistry::from_config()` is working correctly. -Only the EC hash and IP are sent. No consent strings, geo data, or other partner IDs are included. +Only the full EC ID is sent. No client IP, consent strings, geo data, or other partner IDs are included. **Expected partner responses:** @@ -993,69 +1464,97 @@ Any other non-200 response is treated as a transient failure. No retry. The next ### 10.5 KV write on success -On a non-null `uid`: call `kv.upsert_partner_id(ec_hash, partner_id, uid, now())`. If the root entry is missing, the upsert creates a minimal live entry first (same recovery path as `/sync`). On KV failure: log `warn` and discard the result. Retry occurs on the next qualifying request. - -The write updates `ids[partner_id].synced` to the current timestamp, resetting the `pull_sync_ttl_sec` window. +On a non-null `uid`: call `kv.upsert_partner_id(ec_id, partner_id, uid)`. If the root entry is missing, the upsert creates a minimal live entry first. If the same UID is already stored, the upsert skips the KV write. On KV failure: log `warn` and discard the result. Retry occurs on the next qualifying request while the partner UID remains missing. --- -## 11. Identity Resolution Endpoint (`GET /identify`) +## 11. Identity Resolution Endpoint (`GET /_ts/api/v1/identify`) ### 11.1 Module: `ec/identify.rs` ```rust -pub async fn handle_identify( +pub fn handle_identify( settings: &Settings, kv: &KvIdentityGraph, - partner_store: &PartnerStore, + registry: &PartnerRegistry, req: &Request, ec_context: &EcContext, ) -> Result>; ``` -### 11.2 Call patterns +### 11.2 Authentication + +**Bearer token required.** The `Authorization: Bearer ` header identifies the requesting partner. Auth flow: + +1. Parse the Bearer token from the `Authorization` header. +2. Compute `sha256_hex(api_token)`. +3. Look up `registry.find_by_api_key_hash(hash)` — O(1) in-memory lookup. +4. If no match → `401 Unauthorized` with `{ "error": "invalid_token" }`. + +The authenticated partner determines which UID is returned — each partner sees only their own synced UID for the given EC, not all partners' UIDs. -**Browser-direct:** The browser sends the request to `ec.publisher.com/identify`. Cookies and consent cookies are sent automatically (same-site). No special header forwarding required. +### 11.2.1 Call patterns -**Server-side proxy (for use case 2):** The publisher's origin server must forward: +**Browser-direct:** The browser sends the request to `ec.publisher.com/_ts/api/v1/identify` with the partner's API token in the `Authorization` header. Cookies (including `ts-ec` and consent cookies) are sent automatically (same-site). + +**Server-side proxy:** The publisher's origin server must forward: | Header | Required | | --------------------------------------------------------- | -------------------------------------- | +| `Authorization: Bearer ` | Yes | | `Cookie: ts-ec=` or `X-ts-ec: ` | Yes | | `Cookie: euconsent-v2=` or `Cookie: __gpp=` | Yes for EU/UK/US users | | `X-consent-advertising: ` | Optional — takes precedence if present | ### 11.3 EC and consent handling -`/identify` follows `EcContext` retrieval priority (Section 4.2). It does **not** -generate a new EC, and the handler itself does not write cookies. However, -`ec_finalize_response()` still runs after the handler: on consent withdrawal it -deletes the EC cookie, and on header/cookie mismatch it may reconcile the cookie -to the authoritative header-derived EC. +`/_ts/api/v1/identify` follows `EcContext` retrieval priority (Section 4.2). It does **not** +generate a new EC, and the handler itself does not write cookies. After the +handler, `ec_finalize_response()` may still delete the EC cookie on consent +withdrawal. Ordinary returning-user responses set the `x-ts-ec` header only; +they do not refresh or repair the browser cookie. Consent is evaluated using the same logic as Section 6. ### 11.4 Response -**`200 OK` — EC present, consent granted:** +**`401 Unauthorized` — missing or invalid Bearer token:** + +```json +{ "error": "invalid_token" } +``` + +This is checked first, before consent or EC presence. + +**`200 OK` — EC present, consent granted, partner UID resolved:** ```json { "ec": "a1b2c3...AbC123", "consent": "ok", "degraded": false, - "uids": { - "uid2": "A4A...", - "liveramp": "LR_xyz" - }, - "eids": [ - { "source": "uidapi.com", "uids": [{ "id": "A4A...", "atype": 3 }] }, - { "source": "liveramp.com", "uids": [{ "id": "LR_xyz", "atype": 3 }] } - ] + "partner_id": "liveramp", + "uid": "LR_xyz", + "eid": { "source": "liveramp.com", "uids": [{ "id": "LR_xyz", "atype": 3 }] }, + "cluster_size": 2 +} +``` + +The response is scoped to the requesting partner only. `partner_id` identifies which partner was authenticated. `uid` is the partner's resolved UID for this EC. `eid` is the OpenRTB 2.6 EID object for this partner. `cluster_size` is included when the network cluster has been evaluated (see §7A.8); absent when not yet evaluated. + +**`200 OK` — EC present, consent granted, no UID for this partner:** + +```json +{ + "ec": "a1b2c3...AbC123", + "consent": "ok", + "degraded": false, + "partner_id": "liveramp", + "cluster_size": null } ``` -`uids` contains one key per partner with `bidstream_enabled: true` and a resolved UID in the KV graph. Partners with no resolved UID for this user are omitted. +`uid` and `eid` are omitted when the partner has no synced UID for this EC. **`200 OK` — KV unavailable (degraded):** @@ -1064,8 +1563,8 @@ Consent is evaluated using the same logic as Section 6. "ec": "a1b2c3...AbC123", "consent": "ok", "degraded": true, - "uids": {}, - "eids": [] + "partner_id": "liveramp", + "cluster_size": null } ``` @@ -1078,8 +1577,8 @@ This case occurs by design when `create_or_revive()` fails on EC generation (bes "ec": "a1b2c3...AbC123", "consent": "ok", "degraded": false, - "uids": {}, - "eids": [] + "partner_id": "liveramp", + "cluster_size": null } ``` @@ -1091,7 +1590,7 @@ Note: `degraded` is `false` because the KV read succeeded (it returned `None`, m { "consent": "denied" } ``` -Consent is evaluated **before** EC presence. If `!allows_ec_creation(&consent)`, return `403` immediately — do not fall through to the `204` branch. This ensures consent denial is always surfaced, even for users with no EC. +Consent is evaluated **after** auth but **before** EC presence. If `!allows_ec_creation(&consent)`, return `403` immediately — do not fall through to the `204` branch. This ensures consent denial is always surfaced, even for users with no EC. **`204 No Content` — no EC present, consent not denied.** No body. @@ -1099,18 +1598,15 @@ Consent is evaluated **before** EC presence. If `!allows_ec_creation(&consent)`, Set on `200` responses only: -| Header | Value | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `X-ts-ec` | `{ec_hash.suffix}` | -| `X-ts-eids` | Standard base64 (RFC 4648, with `=` padding) of the JSON array of OpenRTB 2.6 `user.eids` objects. Capped at **4 KB** after encoding. If the encoded value exceeds 4 KB, the array is truncated (fewest partners first — highest `synced` timestamp retained) until it fits, and a `x-ts-eids-truncated: true` header is added. | -| `X-ts-` | Resolved UID per partner (e.g., `X-ts-uid2`). One header per partner with a resolved UID. **Capped at 20 partners** — partners sorted by most-recently synced; excess partners are omitted silently. | -| `X-ts-ec-consent` | `ok` (always — denied consent returns `403`, not `200`) | +| Header | Value | +| --------- | --------------------------------- | +| `X-ts-ec` | `{64-hex}.{6-alnum}` — full EC ID | -These are supplementary — callers should read the JSON body as the primary contract. The 4 KB cap on `X-ts-eids` and the 20-partner cap on `X-ts-` headers reflect typical proxy and browser total-header-budget constraints. Both caps apply independently. +The JSON body is the primary contract. The `X-ts-ec` header is supplementary for proxy-layer consumers. ### 11.6 Performance target -`/identify` must respond within 30ms (excluding network latency) when EC is present and KV read succeeds. This requires the KV read to be on the fast path with no retries. +`/_ts/api/v1/identify` must respond within 30ms (excluding network latency) when EC is present and KV read succeeds. This requires the KV read to be on the fast path with no retries. CORS headers must be set to allow browser-direct calls from the publisher's page. The `Access-Control-Allow-Origin` header is dynamically reflected from the `Origin` request header if the origin is an exact match or a subdomain of `settings.publisher.domain`: @@ -1122,20 +1618,18 @@ CORS headers must be set to allow browser-direct calls from the publisher's page Access-Control-Allow-Origin: Access-Control-Allow-Credentials: true Access-Control-Allow-Methods: GET, OPTIONS -Access-Control-Allow-Headers: Cookie, X-ts-ec, X-consent-advertising -Access-Control-Expose-Headers: X-ts-ec, X-ts-eids, X-ts-ec-consent, X-ts-eids-truncated, +Access-Control-Allow-Headers: Authorization, X-ts-ec +Access-Control-Max-Age: 600 Vary: Origin ``` -**`Access-Control-Expose-Headers` note:** The dynamic `X-ts-` headers must be enumerated per-response, not as a static constant. The handler builds the expose list by iterating the partner IDs that have resolved UIDs in the response. `x-ts-eids-truncated` is always included in the expose list (browser JS should be able to detect truncation even when it occurs). - **Origin validation logic:** CORS headers are only relevant when the `Origin` request header is present (browser requests always send it; server-side proxy calls typically do not). -- **No `Origin` header present:** Process normally. No CORS headers added. No `403`. This is the server-side proxy path from §11.2 — origin-server calls forwarding `Cookie` and consent headers. +- **No `Origin` header present:** Process normally. No CORS headers added. No `403`. This is the server-side proxy path from §11.2.1 — origin-server calls forwarding `Cookie`, consent headers, and `Authorization`. - **`Origin` header present, hostname matches `publisher.domain` or ends with `.{publisher.domain}` and scheme is `https`:** Reflect origin in `Access-Control-Allow-Origin`. Add `Vary: Origin`. - **`Origin` header present but does not match:** Return `403`. No body. -Browser `fetch()` with `credentials: "include"` sends an `OPTIONS` preflight. The router handles `OPTIONS /identify` identically — returns `200 OK` with the CORS headers above and no body. +Browser `fetch()` with `credentials: "include"` sends an `OPTIONS` preflight. The router handles `OPTIONS /_ts/api/v1/identify` identically — returns `200 OK` with the CORS headers above and no body. --- @@ -1176,7 +1670,7 @@ let (user_id, eids) = match ec_context.ec_hash() { Some(hash) => { let kv_entry = kv.get(hash).ok().flatten(); let eids = match kv_entry { - Some((entry, _gen)) => build_eids_from_kv(&entry, partner_store), + Some((entry, _gen)) => build_eids_from_kv(&entry, ®istry), None => vec![], // KV read failed or no entry — degrade gracefully }; (ec_context.ec_value.clone(), eids) @@ -1224,7 +1718,7 @@ The current `/auction` path returns a JSON response inline to the JS caller (`en | Header | Value | | --------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `X-ts-ec` | `{ec_hash.suffix}` — when EC is present | +| `X-ts-ec` | `{64-hex}.{6-alnum}` — full EC ID, when EC is present | | `X-ts-eids` | Standard base64 (RFC 4648) of OpenRTB 2.6 `user.eids` JSON array. Capped at 4 KB — same truncation rules as §11.5. | | `X-ts-eids-truncated` | `true` — present only when `X-ts-eids` was truncated | | `X-ts-ec-consent` | `ok` — only present when consent granted; on withdrawal `ec_finalize_response()` strips all EC headers | @@ -1233,200 +1727,221 @@ The current `/auction` path returns a JSON response inline to the JS caller (`en --- -## 13. Partner Registry and Admin Endpoint +## 13. Partner Registry (Config-Based) + +### 13.1 Overview + +Partners are defined in `[[ec.partners]]` TOML configuration and loaded into an in-memory `PartnerRegistry` at startup. There is no KV-backed partner store and no admin registration endpoint. Partner changes require a config update and redeployment. + +### 13.2 Module: `ec/partner.rs` + +Contains only validation helpers and API key hashing. The full partner data model and registry live in `ec/registry.rs`. + +```rust +/// Validates a partner ID format and checks against reserved names. +/// +/// # Errors +/// +/// Returns a descriptive error string on validation failure. +pub fn validate_partner_id(id: &str) -> Result<(), String>; +// Must match `^[a-z0-9_-]{1,32}$`. Reserved names rejected: +// `ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`. + +/// Computes the SHA-256 hex digest of an API key. +pub fn hash_api_key(api_key: &str) -> String; +``` -### 13.1 Module: `ec/partner.rs` +### 13.3 Module: `ec/registry.rs` ```rust -pub struct PartnerRecord { - /// Partner identifier. Must match `^[a-z0-9_-]{1,32}$` (lowercase, no spaces). - /// Used to build `X-ts-` response headers — header-safety is required. - /// Reserved names that would collide with existing managed headers are rejected - /// at registration: `ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`. +/// Runtime-ready partner configuration with precomputed API key hash. +#[derive(Debug, Clone)] +pub struct PartnerConfig { pub id: String, pub name: String, - pub allowed_return_domains: Vec, - pub api_key_hash: String, // SHA-256 hex of the partner's API key + pub source_domain: String, + pub openrtb_atype: u8, pub bidstream_enabled: bool, - pub source_domain: String, // OpenRTB source (e.g., "liveramp.com") - pub openrtb_atype: u8, // typically 3 - pub sync_rate_limit: u32, // per EC hash per partner per hour - pub batch_rate_limit: u32, // API-key level: requests per partner per minute (default 60) + pub api_key_hash: String, // SHA-256 hex, precomputed at startup + pub batch_rate_limit: u32, // requests per partner per minute (default 60) pub pull_sync_enabled: bool, - pub pull_sync_url: Option, // required when pull_sync_enabled; validated at registration - pub pull_sync_allowed_domains: Vec, // allowlist of domains TS may call for this partner - pub pull_sync_ttl_sec: u64, // default 86400 - pub pull_sync_rate_limit: u32, // default 10 - pub ts_pull_token: Option, // required when pull_sync_enabled; outbound bearer token + pub pull_sync_url: Option, + pub pull_sync_allowed_domains: Vec, + pub pull_sync_ttl_sec: u64, // default 86400 + pub pull_sync_rate_limit: u32, // default 10 + pub ts_pull_token: Option, // outbound bearer token for pull sync } -pub struct PartnerStore { - store_name: String, -} - -impl PartnerStore { - pub fn new(store_name: impl Into) -> Self; - - /// Looks up a partner by ID. Returns `None` if not found. - pub fn get(&self, partner_id: &str) -> Result, Report>; - - /// Verifies an API key against the stored hash for a given partner. - /// Uses constant-time comparison. - pub fn verify_api_key(&self, partner_id: &str, api_key: &str) -> bool; - - /// Writes or updates a partner record. - /// Returns `true` if this was a new partner (create), `false` if an existing - /// partner was updated. The pre-read needed for index maintenance (old API key - /// deletion) also determines this. - pub fn upsert(&self, record: &PartnerRecord) -> Result>; - - /// Looks up the partner owning a given API key hash (for batch sync auth). - /// Uses the `apikey:{hash}` secondary index for O(1) lookup, then verifies the - /// stored `api_key_hash` matches (guards against stale index from key rotation). - pub fn find_by_api_key_hash(&self, hash: &str) -> Result, Report>; - - /// Returns all partner records with `pull_sync_enabled == true`. - /// Used by the pull sync dispatcher after each organic request. Implementations - /// must re-check `pull_sync_enabled` on the fetched record before returning it, - /// because the `_pull_enabled` secondary index is best-effort and may be stale. - pub fn pull_enabled_partners(&self) -> Result, Report>; +/// In-memory partner registry with O(1) lookups by ID, API key hash, +/// and source domain. +/// +/// Built once at startup from `[[ec.partners]]` in `trusted-server.toml`. +/// All validation happens during construction. +pub struct PartnerRegistry { + by_id: HashMap, + by_api_key_hash: HashMap, + by_source_domain: HashMap, } -``` -**Storage layout:** Partner records are stored as JSON values in `partner_store` KV, keyed by `partner_id`. Two operations require access patterns beyond single-key lookup: +impl PartnerRegistry { + /// Builds a registry from the config-defined partner list. + /// + /// # Errors + /// + /// Returns `TrustedServerError::Configuration` if any partner has an + /// invalid ID, duplicate ID, duplicate API token hash, duplicate source + /// domain, or invalid pull sync configuration. + pub fn from_config(partners: &[EcPartner]) -> Result>; -1. **`find_by_api_key_hash(hash)`** — batch sync auth needs to find the partner owning a given API key hash. Implementation: maintain a secondary index entry `apikey:{sha256_hex} → partner_id` in the same KV store. Written on `upsert()`, looked up on batch auth. **On key rotation:** `upsert()` must read the existing record first, and if the `api_key_hash` has changed, delete the old `apikey:{old_hash}` index entry before writing the new one. This prevents old API keys from remaining valid after rotation. + /// Returns an empty registry (no partners configured). + pub fn empty() -> Self; -2. **`pull_enabled_partners()`** — pull sync needs all partners with `pull_sync_enabled == true`. Implementation: maintain an index entry `_pull_enabled → [partner_id_1, partner_id_2, ...]` (JSON array of partner IDs) in the same KV store. Updated on `upsert()` when `pull_sync_enabled` changes. The dispatcher reads this list, then does individual `get()` calls for each partner record. This bounds the number of KV reads to `1 + pull_partner_count` per organic request. + /// Looks up a partner by ID. + pub fn get(&self, partner_id: &str) -> Option<&PartnerConfig>; -**Consistency model:** These index writes are **best-effort, not atomic** — Fastly KV does not support multi-key transactions. `upsert()` writes in order: (1) primary record, (2) old API-key index deletion (if key changed), (3) new API-key index, (4) `_pull_enabled` list. If the process fails mid-sequence, indexes may be stale. All readers handle this defensively: + /// Looks up a partner by the SHA-256 hex hash of their API token. + pub fn find_by_api_key_hash(&self, hash: &str) -> Option<&PartnerConfig>; -- `find_by_api_key_hash()`: if the index points to a partner whose stored `api_key_hash` does not match the lookup hash, treat as auth failure (stale index from a rotation). -- `pull_enabled_partners()`: if a listed partner ID returns `None` from `get()`, skip it silently. If the fetched record has `pull_sync_enabled == false`, also skip it silently — that is a stale `_pull_enabled` index entry. -- The `_pull_enabled` list is vulnerable to lost updates under concurrent registrations. This is accepted — partner registration is a low-frequency admin operation (not a hot path). If lost updates become an issue, a CAS-based read-modify-write can be added later. + /// Looks up a partner by their `source_domain` (case-insensitive). + /// Used by Prebid EID ingestion to match EID sources to partners. + pub fn find_by_source_domain(&self, domain: &str) -> Option<&PartnerConfig>; -### 13.2 Admin endpoint (`POST /admin/partners/register`) + /// Returns all partners with `pull_sync_enabled = true`. + pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig>; -**Module:** `ec/admin.rs` + /// Returns an iterator over all configured partners. + pub fn all(&self) -> impl Iterator; -> **Codebase invariant — requires test update:** `Settings::ADMIN_ENDPOINTS` in `settings.rs` lists routes that must be covered by a `[[handlers]]` Basic Auth entry. The existing test at `settings.rs:1504-1530` scans `main.rs` for **every** `/admin/` route string and asserts it appears in `ADMIN_ENDPOINTS`. When `/admin/partners/register` is added to `main.rs`, this test will fail. -> -> **Required changes:** -> -> 1. Do **NOT** add `/admin/partners/register` to `ADMIN_ENDPOINTS` — it uses bearer-token-in-handler auth. -> 2. Update the admin-route-scan test (`settings.rs:1504-1530`) to maintain an exclusion list of bearer-token-authed admin routes (e.g., `const BEARER_AUTH_ADMIN_ROUTES: &[&str] = &["/admin/partners/register"]`) and skip those when asserting `ADMIN_ENDPOINTS` coverage. -> 3. Narrow the `[[handlers]]` pattern in `trusted-server.toml` from `"^/admin"` to `"^/admin/keys"` so that `/admin/partners/register` is not intercepted by `enforce_basic_auth()` before reaching its bearer-token handler. + /// Returns the number of configured partners. + pub fn len(&self) -> usize; -```rust -pub async fn handle_register_partner( - settings: &Settings, - partner_store: &PartnerStore, - req: Request, -) -> Result>; + /// Returns true if no partners are configured. + pub fn is_empty(&self) -> bool; +} ``` -Authentication: `Authorization: Bearer ` header, validated inside the handler against `settings.ec.admin_token_hash` (SHA-256 constant-time comparison). This is a publisher-level admin credential — separate from partner API keys, and enforced in-handler (not via `[[handlers]]` Basic Auth). Returns `401 Unauthorized` with no body if the token is missing or invalid. - -**Request:** +### 13.4 TOML configuration -``` -POST /admin/partners/register -Authorization: Bearer -Content-Type: application/json +Partners are defined in `trusted-server.toml` as `[[ec.partners]]` array entries: -{ - "id": "ssp_x", - "name": "SSP Example", - "allowed_return_domains": ["sync.example-ssp.com"], - "api_key": "raw_key_to_hash_and_store", - "bidstream_enabled": true, - "source_domain": "example-ssp.com", - "openrtb_atype": 3, - "sync_rate_limit": 100, - "batch_rate_limit": 60, - "pull_sync_enabled": false, - "pull_sync_url": null, - "pull_sync_allowed_domains": [], - "pull_sync_ttl_sec": 86400, - "pull_sync_rate_limit": 10, - "ts_pull_token": null -} +```toml +[[ec.partners]] +id = "liveramp" +name = "LiveRamp ATS" +source_domain = "liveramp.com" +openrtb_atype = 3 +bidstream_enabled = true +api_token = "partner-api-token-here" +batch_rate_limit = 60 +pull_sync_enabled = true +pull_sync_url = "https://api.liveramp.com/resolve" +pull_sync_allowed_domains = ["api.liveramp.com"] +pull_sync_ttl_sec = 86400 +pull_sync_rate_limit = 10 +ts_pull_token = "outbound-bearer-token" + +[[ec.partners]] +id = "uid2" +name = "UID 2.0" +source_domain = "uidapi.com" +openrtb_atype = 3 +bidstream_enabled = true +api_token = "uid2-api-token" +batch_rate_limit = 60 ``` -**Processing:** +### 13.5 Startup validation -1. Validate `Authorization: Bearer `: SHA-256 hash the token and compare against `settings.ec.admin_token_hash` using constant-time comparison. `401` if missing or invalid. -2. Validate required fields (`id`, `name`, `allowed_return_domains`, `api_key`, `source_domain`). `400` on failure. - Validate `id` format: must match `^[a-z0-9_-]{1,32}$`. Must not be a reserved name - (`ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`). `400` with descriptive message on failure. -3. If `pull_sync_enabled == true`, validate that both `pull_sync_url` and `ts_pull_token` are present and non-empty. `400` with `"pull_sync_url and ts_pull_token are required when pull_sync_enabled is true"` if either is missing. - If `pull_sync_url` is set, validate that its hostname is present in `pull_sync_allowed_domains`. `400` on failure with `"pull_sync_url domain must be in pull_sync_allowed_domains"`. This prevents TS from being directed to call arbitrary URLs — the allowlist must be declared in the same registration payload. -4. Hash `api_key` with SHA-256 before writing — never store plaintext. -5. `let created = partner_store.upsert(record)?`. `503` on KV failure. - `upsert()` returns `true` for a new partner, `false` for an update. -6. Return `201 Created` if new partner (`created == true`), or `200 OK` if update - (`created == false`). Use an explicit response DTO — do NOT serialize the full - `PartnerRecord` (which contains `api_key_hash` and `ts_pull_token`). +`PartnerRegistry::from_config()` validates during construction: -**Response:** +1. Each partner ID matches `^[a-z0-9_-]{1,32}$` and is not reserved. +2. No duplicate partner IDs. +3. No duplicate API token hashes (collision detection). +4. No duplicate source domains. +5. Rate limits are within valid bounds. +6. If `pull_sync_enabled`, both `pull_sync_url` and `ts_pull_token` must be present. +7. If `pull_sync_url` is set, its hostname must be in `pull_sync_allowed_domains`. -```json -{ - "id": "ssp_x", - "name": "SSP Example", - "pull_sync_enabled": false, - "bidstream_enabled": true, - "created": true -} -``` - -The response confirms the registration succeeded and echoes key fields. `api_key_hash`, `ts_pull_token`, and `api_key` are never returned. `PartnerRecord` does not have a `registered_at` field — use the `created` boolean to signal first registration vs. upsert update. +Any validation failure causes a startup error (`TrustedServerError::Configuration`). --- ## 14. Configuration -### 14.1 New `EdgeCookie` settings struct +### 14.1 `Ec` settings struct Added to `crates/trusted-server-core/src/settings.rs`: ```rust #[derive(Debug, Clone, Deserialize, Serialize, Validate)] -pub struct EdgeCookie { +pub struct Ec { /// Publisher passphrase used as HMAC key for EC generation. /// Must be identical across all of the publisher's owned domains. /// Publishers sharing this value with partners form an identity-federated consortium. - #[validate(custom(function = EdgeCookie::validate_passphrase))] - pub passphrase: String, + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, /// Fastly KV store name for the EC identity graph. - #[validate(length(min = 1))] - pub ec_store: String, - - /// Fastly KV store name for the partner registry. - #[validate(length(min = 1))] - pub partner_store: String, - - /// SHA-256 hex of the publisher admin token for `POST /admin/partners/register`. - /// The plaintext token is provided in the `Authorization: Bearer` header; - /// it is never stored in plaintext. - #[validate(custom(function = EdgeCookie::validate_sha256_hex))] - pub admin_token_hash: String, + #[serde(default)] + pub ec_store: Option, /// Maximum concurrent pull sync calls dispatched per request. - #[validate(range(min = 1))] - #[serde(default = "EdgeCookie::default_pull_sync_concurrency")] + #[serde(default = "Ec::default_pull_sync_concurrency")] pub pull_sync_concurrency: usize, + + /// Network cluster trust threshold. Entries with `cluster_size <= threshold` + /// are treated as individual users for identity resolution purposes. + /// B2B publishers should raise this to 50+ for office-heavy audiences. + #[serde(default = "Ec::default_cluster_trust_threshold")] + pub cluster_trust_threshold: u32, + + /// Seconds between cluster size re-evaluations per entry. + /// Avoids repeated list-prefix API calls on every /identify request. + #[serde(default = "Ec::default_cluster_recheck_secs")] + pub cluster_recheck_secs: u64, + + /// Partners (SSPs, DSPs, identity vendors) for EC identity sync. + #[serde(default)] + pub partners: Vec, } -impl EdgeCookie { +impl Ec { fn validate_passphrase(passphrase: &str) -> Result<(), ValidationError>; - // Rejects "passphrase" or empty string as placeholder. - - fn validate_sha256_hex(value: &str) -> Result<(), ValidationError>; - // Requires exactly 64 lowercase hex characters. + // Rejects known placeholder values as non-production passphrases. fn default_pull_sync_concurrency() -> usize { 3 } + fn default_cluster_trust_threshold() -> u32 { 10 } + fn default_cluster_recheck_secs() -> u64 { 3600 } +} +``` + +The `EcPartner` struct (see §13.4 for TOML format): + +```rust +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct EcPartner { + pub id: String, + pub name: String, + pub source_domain: String, + #[serde(default = "EcPartner::default_openrtb_atype")] + pub openrtb_atype: u8, // default 3 + #[serde(default)] + pub bidstream_enabled: bool, + pub api_token: Redacted, // hashed at startup + #[serde(default = "EcPartner::default_batch_rate_limit")] + pub batch_rate_limit: u32, // default 60 + #[serde(default)] + pub pull_sync_enabled: bool, + #[serde(default)] + pub pull_sync_url: Option, + #[serde(default)] + pub pull_sync_allowed_domains: Vec, + #[serde(default = "EcPartner::default_pull_sync_ttl_sec")] + pub pull_sync_ttl_sec: u64, // default 86400 + #[serde(default = "EcPartner::default_pull_sync_rate_limit")] + pub pull_sync_rate_limit: u32, // default 10 + #[serde(default)] + pub ts_pull_token: Option>, } ``` @@ -1436,11 +1951,11 @@ Added to `Settings`: pub struct Settings { // ... existing fields ... #[validate(nested)] - pub ec: EdgeCookie, // Required — omitting [ec] is a startup error + pub ec: Ec, // Required — omitting [ec] is a startup error } ``` -`EdgeCookie` does not derive `Default` — omitting the `[ec]` section from TOML is a deserialization error at startup. This is intentional: `passphrase`, `ec_store`, `partner_store`, and `admin_token_hash` have no safe defaults. The `#[validate(nested)]` attribute ensures `EdgeCookie::validate_passphrase()` runs when `settings.validate()` is called at startup (`settings_data.rs:28`), matching the pattern used by `Publisher` and `Rewrite` in the existing `Settings` struct (`Synthetic` is removed in PR #479). +`Ec` does not derive `Default` — omitting the `[ec]` section from TOML is a deserialization error at startup. This is intentional: `passphrase` has no safe default. The `#[validate(nested)]` attribute ensures `Ec::validate_passphrase()` runs when `settings.validate()` is called at startup, matching the pattern used by `Publisher` and `Rewrite` in the existing `Settings` struct. ### 14.2 TOML configuration example @@ -1448,24 +1963,42 @@ pub struct Settings { [ec] passphrase = "publisher-chosen-secret" ec_store = "ec_identity_store" -partner_store = "ec_partner_store" -admin_token_hash = "sha256-hex-of-publisher-admin-token" pull_sync_concurrency = 3 +# cluster_trust_threshold = 10 # raise to 50+ for B2B publishers +# cluster_recheck_secs = 3600 # legacy compatibility; cluster_size is computed once per entry + +[[ec.partners]] +id = "liveramp" +name = "LiveRamp ATS" +source_domain = "liveramp.com" +api_token = "partner-api-token-here" +bidstream_enabled = true +batch_rate_limit = 60 +pull_sync_enabled = true +pull_sync_url = "https://api.liveramp.com/resolve" +pull_sync_allowed_domains = ["api.liveramp.com"] +ts_pull_token = "outbound-bearer-token" + +[[ec.partners]] +id = "uid2" +name = "UID 2.0" +source_domain = "uidapi.com" +api_token = "uid2-api-token" +bidstream_enabled = true ``` ### 14.3 Rate Limit Storage -Pixel sync and pull sync rate limits (per EC hash per partner per hour) cannot use in-memory state in a WASM/Fastly Compute environment — there is no shared memory across requests. +Batch sync and pull sync rate limits cannot use in-memory state in a WASM/Fastly Compute environment — there is no shared memory across requests. -**Implementation:** Use Fastly's Edge Rate Limiting API (`fastly::erl::RateCounter`), which provides distributed per-key counting without KV latency and is designed for high-frequency counting without per-key write limits. +**Implementation:** Use Fastly's Edge Rate Limiting API (`fastly::erl::RateCounter`), which provides distributed per-key counting without KV latency and is designed for high-frequency counting without per-key write limits. The `RateLimiter` trait abstracts this for testability. | Counter | Key format | Window | | ---------- | ----------------------------- | -------- | -| Pixel sync | `{partner_id}:{ec_hash}` | 1 hour | -| Pull sync | `pull:{partner_id}:{ec_hash}` | 1 hour | | Batch sync | `batch:{partner_id}` | 1 minute | +| Pull sync | `pull:{partner_id}:{ec_hash}` | 1 hour | -Engineering must confirm `fastly::erl::RateCounter` availability in the target before implementation of Steps 7, 9, and 10 is considered complete. Do NOT silently skip rate limiting in production if ERL is unavailable. Do NOT fall back to KV-based counters — they would hit the same 1 write/sec/key limit that necessitates `update_last_seen()` debouncing, and would thrash under real sync traffic. If ERL is unavailable, the rate-limited routes are blocked on an approved alternative counting mechanism. +Engineering must confirm `fastly::erl::RateCounter` availability in the target before implementation is considered complete. Do NOT silently skip rate limiting in production if ERL is unavailable. Do NOT fall back to KV-based counters — they would hit the same 1 write/sec/key limit that motivated removing recurring organic-request KV writes, and would thrash under real sync traffic. If ERL is unavailable, the rate-limited routes are blocked on an approved alternative counting mechanism. ### 14.4 Deprecation note @@ -1478,33 +2011,25 @@ Engineering must confirm `fastly::erl::RateCounter` availability in the target b New constants in `crates/trusted-server-core/src/constants.rs`: ```rust -// EC cookie name -pub const COOKIE_EC: &str = "ts-ec"; - -// EC response header -pub const HEADER_X_TS_EC: &str = "x-ts-ec"; - -// Supplementary identity headers -pub const HEADER_X_TS_EIDS: &str = "x-ts-eids"; -pub const HEADER_X_TS_EC_CONSENT: &str = "x-ts-ec-consent"; -pub const HEADER_X_TS_EIDS_TRUNCATED: &str = "x-ts-eids-truncated"; - -// Consent cookies (must match existing constants in constants.rs) -pub const COOKIE_TCF: &str = "euconsent-v2"; -pub const COOKIE_GPP: &str = "__gpp"; -pub const COOKIE_GPP_SID: &str = "__gpp_sid"; -pub const COOKIE_US_PRIVACY: &str = "us_privacy"; - -// No EC-specific geo/IP header constants — use req.get_client_ip_addr() and GeoInfo::from_request(req). +// EC cookie names +pub const COOKIE_TS_EC: &str = "ts-ec"; +pub const COOKIE_TS_EIDS: &str = "ts-eids"; + +// EC response headers +pub const HEADER_X_TS_EC: HeaderName = HeaderName::from_static("x-ts-ec"); +pub const HEADER_X_TS_EIDS: HeaderName = HeaderName::from_static("x-ts-eids"); +pub const HEADER_X_TS_EC_CONSENT: HeaderName = HeaderName::from_static("x-ts-ec-consent"); +pub const HEADER_X_TS_EIDS_TRUNCATED: HeaderName = HeaderName::from_static("x-ts-eids-truncated"); ``` -The following EC headers must be added to `INTERNAL_HEADERS` in `constants.rs` to ensure they are stripped before proxying to downstream backends: +The following EC headers are included in `INTERNAL_HEADERS` in `constants.rs` to ensure they are stripped before proxying to downstream backends: + +- `x-ts-ec` +- `x-ts-eids` +- `x-ts-ec-consent` +- `x-ts-eids-truncated` -- `HEADER_X_TS_EC` (`x-ts-ec`) -- `HEADER_X_TS_EIDS` (`x-ts-eids`) -- `HEADER_X_TS_EC_CONSENT` (`x-ts-ec-consent`) -- `HEADER_X_TS_EIDS_TRUNCATED` (`x-ts-eids-truncated`) -- Dynamic `X-ts-` headers — these cannot be registered statically because partners are added at runtime via `/admin/partners/register`. The `INTERNAL_HEADERS` filter **must use prefix stripping** (`x-ts-` prefix match) rather than enumerating partner IDs. A startup snapshot would miss partners registered after deployment. The current filter in `http_util.rs` uses explicit header names — extend it to also strip any header matching the `x-ts-` prefix pattern. +The `INTERNAL_HEADERS` filter uses `x-ts-` prefix stripping in `http_util.rs` to also strip any dynamic `X-ts-` headers without needing to enumerate partner IDs. --- @@ -1525,14 +2050,14 @@ pub enum TrustedServerError { // Maps to StatusCode::INTERNAL_SERVER_ERROR (500) // Used for: EC-specific handler errors only (not organic-path generation) - /// Partner not found in partner_store. + /// Partner not found in registry. #[display("Partner not found: {partner_id}")] PartnerNotFound { partner_id: String }, // Maps to StatusCode::BAD_REQUEST (400) /// Partner API key authentication failed. - #[display("Invalid API key for partner: {partner_id}")] - PartnerAuthFailed { partner_id: String }, + #[display("Invalid API key")] + PartnerAuthFailed, // Maps to StatusCode::UNAUTHORIZED (401) } ``` @@ -1544,23 +2069,17 @@ pub enum TrustedServerError { New routes added to `route_request()` in `crates/trusted-server-adapter-fastly/src/main.rs`: ```rust -// EC sync pixel — no auth required (partner validation is internal) -(GET, "/sync") → handle_sync(settings, &kv, &partner_store, &req, &mut ec_context) - -// EC identity resolution — no auth required (consent-gated) -(GET, "/identify") → handle_identify(settings, &kv, &partner_store, &req, &ec_context) +// EC identity resolution — Bearer token auth (internal to handler) +(GET, "/_ts/api/v1/identify") → handle_identify(settings, &kv, ®istry, &req, &ec_context) // CORS preflight for /identify — must be registered explicitly, current router dispatches by exact method/path -(OPTIONS, "/identify") → cors_preflight_identify(settings, &req) +(OPTIONS, "/_ts/api/v1/identify") → cors_preflight_identify(settings, &req) // S2S batch sync — partner API key auth (internal to handler) -(POST, "/api/v1/sync") → handle_batch_sync(settings, &kv, &partner_store, req) - -// Partner registration — publisher admin auth enforced in-handler (Bearer token) -(POST, "/admin/partners/register") → handle_register_partner(settings, &partner_store, req) +(POST, "/_ts/api/v1/batch-sync") → handle_batch_sync(&kv, ®istry, &limiter, req) ``` -Route ordering: EC routes are inserted before the fallback `handle_publisher_request()`. The `/admin/partners/register` route uses bearer-token auth in-handler (not `[[handlers]]` Basic Auth). The current `trusted-server.toml` has `path = "^/admin"` which catches **all** `/admin/*` paths via `enforce_basic_auth()` before routing — this would block bearer-token requests to `/admin/partners/register`. **Required change:** narrow the existing `[[handlers]]` pattern from `"^/admin"` to `"^/admin/keys"` so it covers only `/admin/keys/rotate` and `/admin/keys/deactivate` (the routes in `Settings::ADMIN_ENDPOINTS`). `/admin/partners/register` then passes through `enforce_basic_auth()` unchallenged and reaches the bearer-token handler. +Route ordering: EC routes are inserted before the fallback `handle_publisher_request()`. ### 17.1 EC integration in `main.rs` @@ -1574,9 +2093,17 @@ EC follows the same pre-routing pattern as `GeoInfo::from_request()` (line 70). This is a supported Fastly Compute pattern — `Response::send_to_client()` flushes the response to the client immediately and allows the WASM invocation to continue. This is not a small wiring change; it restructures how the application returns responses. ```rust -async fn route_request(...) -> Result<(), Error> { +fn route_request(...) -> Result<(), Error> { let geo_info = GeoInfo::from_request(&req); + // Phase 0 — bot gate (pure in-memory, no KV I/O). See §7A. + let device_signals = derive_device_signals(&req); + let is_real_browser = device_signals.looks_like_browser(); + if !is_real_browser { + log::debug!("Bot gate: blocking EC operations (ja4={:?}, platform={:?})", + device_signals.ja4_class, device_signals.platform_class); + } + // Pre-routing — read only, no generation (matches GeoInfo pattern). // EcContext stores client_ip internally (same req.get_client_ip_addr() // already called by GeoInfo::from_request() above). @@ -1584,20 +2111,35 @@ async fn route_request(...) -> Result<(), Error> { let mut ec_context = match ec_context_result { Ok(ctx) => ctx, Err(e) => { - // Pre-routing failure — no route matched yet, but we still need to - // send an HTTP error response. Construct one and flush immediately. log::error!("EcContext initialization failed: {e:?}"); let mut response = to_error_response(&e); response.send_to_client(); return Ok(()); } }; - let kv = KvIdentityGraph::new(&settings.ec.ec_store); - let partner_store = PartnerStore::new(&settings.ec.partner_store); - let pull_sync_dispatcher = PullSyncDispatcher::new(settings.ec.pull_sync_concurrency); + + // Pass device signals through for KvDevice on creation. + ec_context.set_device_signals(device_signals); + + // Build partner registry from config at startup. + let registry = PartnerRegistry::from_config(&settings.ec.partners)?; + + // Extract ts-eids cookie before routing consumes the request. + let eids_cookie = extract_cookie_value(&req, COOKIE_TS_EIDS); + + // Bot gate: suppress all KV operations for unrecognized clients. + let kv = if is_real_browser { + settings.ec.ec_store.as_deref().map(KvIdentityGraph::new) + } else { + None + }; + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); if let Some(mut response) = enforce_basic_auth(settings, &req) { - ec_finalize_response(settings, geo_info.as_ref(), &ec_context, &kv, &mut response); + // Bot gate: skip EC cookie writes for unrecognized clients. + if is_real_browser { + ec_finalize_response(settings, &ec_context, kv.as_ref(), ®istry, eids_cookie.as_deref(), &mut response); + } response.send_to_client(); return Ok(()); } @@ -1609,49 +2151,37 @@ async fn route_request(...) -> Result<(), Error> { // is_organic tracks whether pull sync should fire (organic routes only — §10.2). let mut is_organic = false; let result = match (method, path.as_str()) { - // EC-specific routes — all read-only except /sync which takes &mut. - // /sync may assign fallback consent into ec_context.consent when the - // query param is the only signal — see §8.3. - (GET, "/sync") => handle_sync(settings, &kv, &partner_store, &req, &mut ec_context).await, - (GET, "/identify") => handle_identify(settings, &kv, &partner_store, &req, &ec_context).await, - (OPTIONS, "/identify") => cors_preflight_identify(settings, &req), - (POST, "/api/v1/sync") => handle_batch_sync(settings, &kv, &partner_store, req).await, - (POST, "/admin/partners/register") => handle_register_partner(settings, &partner_store, req).await, - - // /auction — EC-read-only; never generates EC. - // NOTE: handle_auction signature changes from (settings, orchestrator, req) to - // (settings, orchestrator, &kv, req, &ec_context) — this is a call-graph change, - // not just wiring. See §12 for the full auction integration. - (POST, "/auction") => handle_auction(settings, orchestrator, &kv, req, &ec_context).await, - - // Organic routes — generate EC if needed (best-effort, never 500s), then dispatch + (GET, "/_ts/api/v1/identify") => handle_identify(settings, kv.as_ref(), ®istry, &req, &ec_context), + (OPTIONS, "/_ts/api/v1/identify") => cors_preflight_identify(settings, &req), + (POST, "/_ts/api/v1/batch-sync") => handle_batch_sync(kv.as_ref(), ®istry, &limiter, req), + (POST, "/auction") => handle_auction(settings, orchestrator, kv.as_ref(), req, &ec_context), + (m, path) if integration_registry.has_route(&m, path) => { is_organic = true; - ec_context.generate_if_needed(settings, &kv); - integration_registry.handle_proxy(&m, path, settings, req, &ec_context).await + ec_context.generate_if_needed(settings, kv.as_ref()); + integration_registry.handle_proxy(&m, path, settings, req, &ec_context) }, _ => { is_organic = true; - ec_context.generate_if_needed(settings, &kv); + ec_context.generate_if_needed(settings, kv.as_ref()); handle_publisher_request(settings, integration_registry, req, &ec_context) }, }; - // Unwrap result — errors become error responses (matches existing pattern) let mut response = result.unwrap_or_else(|e| to_error_response(&e)); - // finalize_response runs on every route — enforces cookie write/deletion/last_seen - ec_finalize_response(settings, geo_info.as_ref(), &ec_context, &kv, &mut response); + // Bot gate: skip EC cookie writes and finalize for unrecognized clients. + if is_real_browser { + ec_finalize_response(settings, &ec_context, kv.as_ref(), ®istry, eids_cookie.as_deref(), &mut response); + } - // Flush response to client; WASM continues for background pull sync. response.send_to_client(); - // Background pull sync — organic routes only (§10.2). Never fires on /sync, - // /identify, /auction, /api/v1/sync, or /admin/* routes. - // Fires outbound HTTP calls via send_async(), blocks on PendingRequest::wait(). - if is_organic { - if let (Some(ip), Ok(pull_partners)) = (ec_context.client_ip, partner_store.pull_enabled_partners()) { - pull_sync_dispatcher.dispatch_background(&ec_context, ip, &pull_partners, &kv); + // Background pull sync — organic routes only, real browsers only (§7A.4, §10.2). + if is_real_browser && is_organic { + if let Some(ip) = ec_context.client_ip { + let pull_partners = registry.pull_enabled_partners(); + pull_sync_dispatcher.dispatch_background(&ec_context, ip, &pull_partners, kv.as_ref()); } } @@ -1659,7 +2189,7 @@ async fn route_request(...) -> Result<(), Error> { } ``` -The existing `finalize_response()` in `main.rs` becomes `ec_finalize_response()` with the extended signature that accepts `ec_context` and `kv`. The `#[fastly::main]` entrypoint changes to call `route_request()` and return `Ok(())` (the response is already sent via `send_to_client()`). +The existing `finalize_response()` in `main.rs` becomes `ec_finalize_response()` with the extended signature that accepts `ec_context`, `kv`, `registry`, and `eids_cookie`. The `#[fastly::main]` entrypoint changes to call `route_request()` and return `Ok(())` (the response is already sent via `send_to_client()`). The `PartnerRegistry` is built once at startup via `PartnerRegistry::from_config(&settings.ec.partners)` and passed by reference throughout the request lifecycle. `PullSyncDispatcher::dispatch_background` uses `Request::send_async()` to fire outbound HTTP calls, then calls `PendingRequest::wait()` (blocking) on each handle under `settings.ec.pull_sync_concurrency` concurrency. No async runtime is needed — this is synchronous blocking code running after `send_to_client()` has flushed the response. The Fastly WASM invocation stays alive until `dispatch_background` returns. This does not add latency to the user-facing response. @@ -1673,53 +2203,55 @@ Follow the project's **Arrange-Act-Assert** pattern. Test both happy paths and e Each module in `ec/` has a `#[cfg(test)]` module covering: -| Module | Key test cases | -| --------------- | --------------------------------------------------------------------------------------------------------- | -| `identity.rs` | IPv4/IPv6 normalization, /64 truncation, HMAC determinism, output format | -| `finalize.rs` | `ec_finalize_response()`: cookie write on generation, deletion on withdrawal, `update_last_seen` debounce | -| `cookie.rs` | Cookie string format, Max-Age=0 for deletion, domain derivation | -| `kv.rs` | Serialization/deserialization roundtrip, CAS merge logic, metadata extraction | -| `partner.rs` | API key hash verification (constant-time), record serialization | -| `sync_pixel.rs` | All `ts_synced` redirect codes, 429 rate limit, return URL construction | -| `sync_batch.rs` | Status code selection (200/207/401/400/429), per-mapping rejection reasons, API-key rate limit | -| `pull_sync.rs` | Trigger conditions, null/404 no-op, dispatch limit | -| `identify.rs` | All response codes (200/403/204), degraded flag, `uids` filtering | +| Module | Key test cases | +| ---------------- | --------------------------------------------------------------------------------------------------------------------- | +| `generation.rs` | IPv4/IPv6 normalization, /64 truncation, HMAC determinism, output format | +| `finalize.rs` | `ec_finalize_response()`: cookie write on generation, deletion on withdrawal, returning-user EC header, EID ingestion | +| `cookies.rs` | Cookie string format, Max-Age=0 for deletion, domain derivation | +| `kv.rs` | Serialization/deserialization roundtrip, CAS merge logic, metadata extraction | +| `partner.rs` | Partner ID validation, API key hashing | +| `registry.rs` | `from_config()` validation, duplicate detection, O(1) lookups by ID/hash/domain | +| `prebid_eids.rs` | Base64 decode, JSON parse, source domain matching, debounce | +| `batch_sync.rs` | Status code selection (200/207/401/400/429), per-mapping rejection reasons, API-key rate limit | +| `pull_sync.rs` | Trigger conditions, null/404 no-op, dispatch limit | +| `identify.rs` | Bearer auth (200/401/403/204), scoped partner response, degraded flag, CORS | ### 18.2 Integration tests KV behavior is tested with Viceroy (local Fastly Compute simulator) using real KV store operations. Key scenarios: -- Consent withdrawal: cookie deletion + tombstone write (`write_withdrawal_tombstone()`) + all EC response headers stripped — in same request +- Explicit consent withdrawal: cookie deletion + tombstone write (`write_withdrawal_tombstone()`) + all EC response headers stripped — in same request - Concurrent writes: CAS retry logic under simulated generation conflicts - KV degraded: EC cookie still set when KV `create_or_revive()` fails (best-effort) -- Sync-then-identify flow: pixel sync writes partner ID, then `/identify` returns it +- Prebid EID ingestion: `ts-eids` cookie parsed, source domain matched, partner UID written to KV +- Batch sync then identify: batch sync writes partner UID, then `/_ts/api/v1/identify` returns it for that partner **Eventually-consistent caveat:** Fastly KV does not guarantee read-after-write consistency. The sync→identify scenario may not be immediately visible on production — Viceroy may behave differently. Tests for this flow should use retry with backoff (up to 1s) and be documented as Viceroy-only consistency. Do not write assertions that assume immediate visibility after a KV write. ### 18.3 JS tests (if applicable) -If any JS changes are made for EC (e.g., publisher-side `/identify` fetch helper in `crates/js/`), use Vitest with `vi.hoisted()` for mocks. +If any JS changes are made for EC (e.g., publisher-side `/_ts/api/v1/identify` fetch helper in `crates/js/`), use Vitest with `vi.hoisted()` for mocks. --- ## 19. Implementation Order -Suggested order to minimize risk and allow incremental testing. Each step should pass `cargo test --workspace` before the next begins. - -| Step | Scope | Deliverable | -| ---- | --------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| 1 | `ec/identity.rs` + constants + settings | `generate_ec()`, `normalize_ip()`, `EcContext` | -| 2 | `ec/finalize.rs` | `ec_finalize_response()` (cookie write, deletion, tombstone, last_seen) | -| 3 | `ec/cookie.rs` | Cookie creation, deletion, response header | -| 4 | `ec/kv.rs` | `KvIdentityGraph` CRUD with CAS | -| 5 | `ec/partner.rs` + `ec/admin.rs` | `PartnerStore`, `/admin/partners/register` | -| 6 | EC middleware in `main.rs`, `publisher.rs`, `registry.rs` | `EcContext::read_from_request()` pre-routing, `generate_if_needed()`, `ec_finalize_response()` | -| 7 | `ec/sync_pixel.rs` | `GET /sync` handler + route | -| 8 | `ec/identify.rs` | `GET /identify` handler + route | -| 9 | `ec/sync_batch.rs` | `POST /api/v1/sync` handler + route | -| 10 | `ec/pull_sync.rs` | Background pull sync dispatch (blocking, after `send_to_client()`) | -| 11 | Auction integration | EC injection into `user.id`, `user.eids`, `user.consent` | -| 12 | End-to-end integration tests | Viceroy-based flow tests | +Implementation was completed in the following order. Each step passed `cargo test --workspace` before the next began. + +| Step | Scope | Deliverable | +| ---- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| 1 | `ec/generation.rs` + constants + settings | `generate_ec()`, `normalize_ip()`, `EcContext` | +| 2 | `ec/cookies.rs` | Cookie creation, deletion, response header | +| 3 | `ec/kv.rs` + `ec/kv_types.rs` | `KvIdentityGraph` CRUD with CAS | +| 4 | `ec/finalize.rs` | `ec_finalize_response()` (cookie write on generation, deletion, tombstone, returning-user header) | +| 5 | `ec/partner.rs` + `ec/registry.rs` | `PartnerRegistry` (config-based), partner validation helpers | +| 6 | EC middleware in `main.rs`, `publisher.rs`, `registry.rs` | `EcContext::read_from_request()` pre-routing, `generate_if_needed()`, `ec_finalize_response()` | +| 7 | `ec/prebid_eids.rs` | Prebid EID cookie ingestion (replaces pixel sync) | +| 8 | `ec/identify.rs` | `GET /_ts/api/v1/identify` handler + route (Bearer auth, scoped response) | +| 9 | `ec/batch_sync.rs` + `ec/rate_limiter.rs` | `POST /_ts/api/v1/batch-sync` handler + route | +| 10 | `ec/pull_sync.rs` | Background pull sync dispatch (blocking, after `send_to_client()`) | +| 11 | Auction integration | EC injection into `user.id`, `user.eids`, `user.consent` | +| 12 | End-to-end integration tests | Viceroy-based flow tests | --- @@ -1733,7 +2265,7 @@ and auction decoration — without relying on third-party cookies. **Done when:** All 12 stories below are complete, `cargo test --workspace` and `cargo clippy` pass with no warnings, and the end-to-end Viceroy flow tests -cover the full sync → identify → auction path. +cover the full EID ingestion → identify → auction path. **Spec ref:** This document. PRD: `docs/internal/ssc-prd.md`. @@ -1744,7 +2276,7 @@ cover the full sync → identify → auction path. Implement the core EC data types, generation logic, and per-request context struct that all subsequent stories depend on. -**Scope:** `ec/identity.rs`, `ec/mod.rs`, `trusted-server.toml` `[ec]` section, +**Scope:** `ec/generation.rs`, `ec/mod.rs`, `trusted-server.toml` `[ec]` section, `Settings` struct update. **Acceptance criteria:** @@ -1764,7 +2296,7 @@ struct that all subsequent stories depend on. Calls `build_consent_context()` with the EC hash as identity key and stores the result as `consent: ConsentContext` (see §6.1.1). Does not generate. Does not write to EC identity KV. (Note: `build_consent_context()` may write - to the consent KV store when an EC hash is available.) + using the request-local consent context.) - `EcContext::generate_if_needed(settings, kv)` generates a new EC when `ec_value == None && allows_ec_creation(&consent)`, sets `ec_generated = true`, and writes the initial KV entry via `kv.create_or_revive()` (best-effort). @@ -1774,8 +2306,8 @@ struct that all subsequent stories depend on. without setting `ec_generated`. It never returns an error — organic traffic must not 500 on EC failure. - `[ec]` settings block parses from TOML: `passphrase`, `ec_store`, - `partner_store`, `admin_token_hash`, `pull_sync_concurrency`. -- All unit tests in `identity.rs` pass (HMAC determinism, format, IP normalization). + `pull_sync_concurrency`, `partners`. +- All unit tests in `generation.rs` pass (HMAC determinism, format, IP normalization). **Spec ref:** §2, §3, §4, §5.4, §14.1 @@ -1784,26 +2316,19 @@ struct that all subsequent stories depend on. ### Story 2 — EC finalize response Implement `ec_finalize_response()` — the post-routing function that enforces -cookie writes, deletions, tombstones, and last-seen updates on every response. +cookie writes on generation, cookie deletion on withdrawal, tombstones, returning-user `x-ts-ec` headers, and EID ingestion on responses. **Scope:** `ec/finalize.rs` (new file) **Acceptance criteria:** - `ec_finalize_response(settings, geo, ec_context, kv, response)` runs on every route. -- Consent gating uses the existing `allows_ec_creation()` — no new gating function. -- When `!allows_ec_creation(&consent) && cookie_was_present`: calls - `clear_ec_on_response()` (deletes cookie and strips all EC response headers) - and writes tombstone for each valid EC hash available. When the cookie is - malformed and no valid header exists, no tombstone is written — cookie - deletion alone enforces withdrawal (see §6.2). -- When `ec_was_present && !ec_generated && allows_ec_creation(&consent)`: calls - `kv.update_last_seen(ec_hash, now())` (debounced at 300s). If `cookie_ec_value` - is set (header/cookie mismatch), also calls `set_ec_on_response()` to reconcile - the browser cookie to the header-derived identity. -- When `ec_generated == true`: calls `set_ec_on_response()`. -- Unit tests cover all four branches: withdrawal (with and without valid hash), - returning-user last_seen + mismatch reconciliation, and new-EC generation. +- Consent gating uses `allows_ec_creation()` for current-request EC usage and `has_explicit_ec_withdrawal()` for cookie-expiry/tombstone decisions. +- When `!allows_ec_creation(&consent)`: strips all EC response headers. +- When `has_explicit_ec_withdrawal(&consent) && cookie_was_present`: additionally expires the cookie and writes tombstones for each valid EC ID available. When the cookie is malformed and no valid header exists, no tombstone is written — cookie deletion alone enforces withdrawal (see §6.2). +- When `ec_was_present && !ec_generated && allows_ec_creation(&consent)`: sets the `x-ts-ec` response header only. It does not refresh the EC cookie, repair header/cookie mismatches, or write KV solely to extend TTL. +- When `ec_generated == true`: calls `set_ec_cookie_and_header_on_response()`. +- Unit tests cover explicit-withdrawal, fail-closed header stripping, returning-user header behavior, and new-EC generation. **Spec ref:** §5.4, §6.2 @@ -1814,7 +2339,7 @@ cookie writes, deletions, tombstones, and last-seen updates on every response. Implement the low-level functions that create and delete the `ts-ec` cookie and set EC response headers. These are called by `ec_finalize_response()` (Story 2). -**Scope:** `ec/cookie.rs` +**Scope:** `ec/cookies.rs` **Acceptance criteria:** @@ -1822,7 +2347,7 @@ and set EC response headers. These are called by `ec_finalize_response()` (Story `Max-Age=31536000`, `SameSite=Lax; Secure`. `HttpOnly` is NOT set (JS on the publisher page must be able to read the cookie). - `delete_ec_cookie()` produces a cookie with `Max-Age=0`, same attributes. -- `set_ec_on_response()` sets `Set-Cookie` and `X-ts-ec` response headers. +- `set_ec_header_on_response()` sets only `X-ts-ec`; `set_ec_cookie_and_header_on_response()` sets both `Set-Cookie` and `X-ts-ec`. - `clear_ec_on_response()` sets `Set-Cookie` with `Max-Age=0` **and** strips all EC-related response headers: `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, and any `X-ts-` headers. This prevents @@ -1854,19 +2379,14 @@ CAS-based concurrent write protection and consent withdrawal delete. - `KvIdentityGraph::create_or_revive(ec_hash, &entry)` creates a new entry OR overwrites an existing tombstone (`consent.ok = false`) with a fresh entry; no-ops if a live entry already exists. Called by `generate_if_needed()`. -- `KvIdentityGraph::update_last_seen(ec_hash, timestamp)` updates `last_seen` - without overwriting partner IDs (CAS merge), and only writes if the stored - value is more than 300s older than `timestamp` (debounce to avoid 1 write/sec - KV limit). Callers pass `now()` as `timestamp`. +- Returning-user page views do not update a last-seen field; EC entries no longer store `last_seen` or mutable publisher-domain visit timestamps. - `KvIdentityGraph::write_withdrawal_tombstone(ec_hash)` sets `consent.ok = false`, clears partner IDs, and applies a 24-hour TTL (see §6.2). Returns `Result` — callers log `error` on failure and continue (cookie deletion is the primary enforcement mechanism). - `KvIdentityGraph::delete(ec_hash)` hard-deletes the entry — used only for IAB data deletion requests, not for consent withdrawal (which uses tombstones). -- `kv.upsert_partner_id(ec_hash, partner_id, uid, timestamp)` writes to - `ids[partner_id]`, creating a minimal live root entry first if the key is - absent, and skips if existing `synced >= timestamp` (idempotent). +- `kv.upsert_partner_id(ec_hash, partner_id, uid)` writes to `ids[partner_id]`, creating a minimal live root entry first if the key is absent, and skips writes when the existing UID already matches (idempotent). - KV schema matches §7 exactly (JSON roundtrip test). - Unit tests cover CAS merge logic, tombstone write, tombstone error handling, serialization/deserialization roundtrip, metadata extraction. @@ -1875,39 +2395,31 @@ CAS-based concurrent write protection and consent withdrawal delete. --- -### Story 5 — Partner registry and admin endpoint +### Story 5 — Partner registry (config-based) -Implement `PartnerRecord`, `PartnerStore`, and the admin registration endpoint -that operators use to onboard ID sync partners. +Implement partner ID validation, API key hashing, and the in-memory +`PartnerRegistry` that replaces the KV-backed `PartnerStore`. -**Scope:** `ec/partner.rs`, `ec/admin.rs`, router update +**Scope:** `ec/partner.rs`, `ec/registry.rs` **Acceptance criteria:** -- `PartnerRecord` contains all fields from §13.1 including +- `validate_partner_id()` enforces `^[a-z0-9_-]{1,32}$` and rejects reserved + names (`ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, + `version`, `env`). +- `hash_api_key()` computes SHA-256 hex of the plaintext API token. +- `PartnerConfig` contains all fields from §13.3 including `pull_sync_allowed_domains` and `batch_rate_limit`. -- `PartnerStore::get()`, `upsert()`, `find_by_api_key_hash()` operate on - `partner_store` KV. -- `pull_enabled_partners()` re-checks `pull_sync_enabled == true` on fetched - records so stale `_pull_enabled` index entries do not dispatch disabled partners. -- API key stored as SHA-256 hex; plaintext never written to KV. -- `verify_api_key()` uses constant-time comparison. -- `POST /admin/partners/register` validates `Authorization: Bearer ` inside - the handler against `settings.ec.admin_token_hash` (constant-time SHA-256 comparison). - Returns `401` if missing or invalid — before any request body is read. -- Admin endpoint validates: `pull_sync_url` hostname must be in - `pull_sync_allowed_domains` when set — returns `400` otherwise. -- Returns `201 Created` on new partner or `200 OK` on update, with an explicit - response DTO (see §13.2 step 6 — do NOT serialize full `PartnerRecord`). - Returns `400` on validation failure; `503` on KV failure. -- `/admin/partners/register` is **NOT** added to `Settings::ADMIN_ENDPOINTS` — - it uses bearer-token-in-handler auth, not `[[handlers]]` Basic Auth. -- The admin-route-scan test (`settings.rs:1504-1530`) must be updated to exclude - bearer-token-authed routes from its `ADMIN_ENDPOINTS` assertion. Add an exclusion - list (see §13.2 codebase invariant note). -- The `[[handlers]]` pattern in `trusted-server.toml` must be narrowed from - `"^/admin"` to `"^/admin/keys"` (see §13.2). -- Unit tests cover API key hash verification and record serialization. +- `PartnerRegistry::from_config()` builds the registry from `Vec` + with O(1) `by_id`, `by_api_key_hash`, and `by_source_domain` indexes. +- Startup validation catches: invalid IDs, duplicate IDs, duplicate API token + hashes, duplicate source domains, invalid pull sync configuration. +- `get()`, `find_by_api_key_hash()`, `find_by_source_domain()` return + `Option<&PartnerConfig>`. +- `pull_enabled_partners()` returns only partners with `pull_sync_enabled = true`. +- No admin endpoint — partner changes require config update and redeployment. +- Unit tests cover partner ID validation, hash computation, registry + construction, and duplicate detection. **Spec ref:** §13 @@ -1925,22 +2437,17 @@ Wire `EcContext` into the request pipeline following the two-phase model - `EcContext::read_from_request()` is called before the route match on every request, passed the existing `geo_info` (no duplicate geo header parsing). -- EC route handlers receive `ec_context` without EC generation. `/identify`, - `/auction`, `/api/v1/sync`, and `/admin/*` use read-only `&EcContext` and - never mutate it. **Exception:** `/sync` receives `&mut EcContext`; when the - consent query-param fallback applies (`ec_context.consent.is_empty()`), it - assigns the locally-decoded consent into `ec_context.consent` so that both - the sync write decision and `ec_finalize_response()` share the same effective - consent view. This prevents a same-request "write partner ID, then withdraw - EC" conflict. See §8.3 for full details. +- EC route handlers receive `ec_context` without EC generation. `/_ts/api/v1/identify`, + `/auction`, and `/_ts/api/v1/batch-sync` use read-only `&EcContext` and + never mutate it. - `/auction` consumes EC identity but never bootstraps it. - `handle_publisher_request()` and `integration_registry.handle_proxy()` call `ec_context.generate_if_needed(settings, &kv)` before their handler logic (best-effort, never 500s). - `ec_finalize_response()` receives `ec_context` and `kv` and: - - Deletes the EC cookie and writes a withdrawal tombstone when `!allows_ec_creation(&consent) && cookie_was_present` (runs on all routes). - - Calls `kv.update_last_seen(ec_hash, now())` when `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)` (returning user with valid consent). - - Calls `set_ec_on_response()` when `ec_context.ec_generated == true`, and also - on returning-user mismatch reconciliation when `cookie_ec_value.is_some()`. + - Strips EC response headers whenever `!allows_ec_creation(&consent)`. + - Additionally deletes the EC cookie and writes a withdrawal tombstone when `has_explicit_ec_withdrawal(&consent) && cookie_was_present` (runs on all routes). + - Sets `x-ts-ec` header when `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)` (returning user with valid consent). Also ingests Prebid EIDs from `ts-eids` cookie. + - Calls `set_ec_cookie_and_header_on_response()` when `ec_context.ec_generated == true` (newly generated ECs). Returning-user mismatch repair is not performed. Also ingests Prebid EIDs. - `route_request()` return type changes from `Result` to `Result<(), Error>`; response is flushed via `response.send_to_client()` instead of being returned. The `#[fastly::main]` entrypoint must also change to match. @@ -1962,102 +2469,84 @@ Wire `EcContext` into the request pipeline following the two-phase model --- -### Story 7 — Pixel sync (`GET /sync`) - -Implement the pixel-based ID sync endpoint that partners use to write their -user ID against an EC hash. +### Story 7 — Prebid EID cookie ingestion -**Scope:** `ec/sync_pixel.rs`, router update +Implement the server-side ingestion of the `ts-eids` cookie, which replaces +the pixel sync endpoint as the browser-side ID sync mechanism. +**Scope:** `ec/prebid_eids.rs`, `ec/finalize.rs` update **Acceptance criteria:** -- Missing required query params (`partner`, `uid`, `return`) → `400`. -- No valid `ts-ec` cookie (missing or malformed) → redirect to - `{return}?ts_synced=0&ts_reason=no_ec`. -- Unknown `partner` ID → `400`. -- `return` URL hostname not in `partner.allowed_return_domains` → `400`. -- Consent uses `ec_context.consent`. The optional `consent` query param is a fallback - only: it is used exclusively when `ec_context.consent.is_empty()` returns `true` - — meaning no consent signals of any kind are present (no TCF string, no GPP - string, no US Privacy string, no AC string, no GPC, no decoded consent objects). - Use the `ConsentContext::is_empty()` method directly; do not reimplement the - check from this description. If consent KV fallback or any other pre-routing - source has already populated `ec_context.consent`, `is_empty()` is `false` and - the param is ignored. - When the fallback applies, decode the consent string locally into a - `ConsentContext` and **assign it into `ec_context.consent`** so that both - the sync write and `ec_finalize_response()` share the same effective consent - (prevents a same-request "write partner ID, then withdraw EC" conflict). - Do NOT re-call `build_consent_context()` (that would trigger consent KV writes). - Denied or absent → redirect to `{return}?ts_synced=0&ts_reason=no_consent`. -- Rate limit exceeded → `429 Too Many Requests` (no redirect). -- KV write failure → redirect to `{return}?ts_synced=0&ts_reason=write_failed`. -- `kv.upsert_partner_id()` creates a minimal live root entry first when the EC - exists in the cookie but the identity graph key is still missing because the - original best-effort `create_or_revive()` failed on generation. -- Success → redirect to `{return}?ts_synced=1`. -- Return URL construction correctly appends `&` or `?` based on existing query string. -- Rate counter key: `{partner_id}:{ec_hash}`, 1-hour window, via `fastly::erl::RateCounter`. -- Unit tests cover all redirect/response codes and return URL construction. +- `ingest_prebid_eids(cookie_value, ec_id, kv, registry)` decodes a base64 JSON + array of OpenRTB-style `{source, uids:[...]}` objects and syncs matched partners to KV. The backend also accepts the earlier flattened `{source, id, atype}` payload for backward compatibility. +- Source domain matching via `registry.find_by_source_domain()` (case-insensitive). +- Sources with no non-empty UID are skipped. +- Idempotent write suppression: if the stored UID already matches the incoming UID, the write is skipped for that partner. +- KV write via `kv.upsert_partner_id()` — best-effort, errors logged at `warn`. +- Called from `ec_finalize_response()` on both returning-user and new-EC paths + when a `ts-eids` cookie is present and consent is granted. +- JS writer target size: 3 KB; backend parser raw-cookie limit: 8 KiB. +- All errors are logged and swallowed — never blocks the response. +- Unit tests cover base64 decode, JSON parse, source domain matching, size limits, + and empty/oversized UID handling. **Spec ref:** §8 --- -### Story 8 — Identity lookup (`GET /identify`) +### Story 8 — Identity lookup (`GET /_ts/api/v1/identify`) -Implement the browser-facing endpoint that publishers call to retrieve the EC -hash and synced partner UIDs for the current user. +Implement the partner-facing endpoint that authenticated partners call to +retrieve their own synced UID for the current EC. **Scope:** `ec/identify.rs`, router update **Acceptance criteria:** +- **Bearer token required.** Missing or invalid `Authorization: Bearer` → `401` + with `{ "error": "invalid_token" }`. Auth uses `registry.find_by_api_key_hash()`. - `!allows_ec_creation(consent)` (consent denied, regardless of EC presence) → `403 Forbidden`. - When EC is present but consent is denied, the handler returns `403` and - `ec_finalize_response()` deletes the cookie and writes a tombstone. + When the denial is an explicit withdrawal signal and a `ts-ec` cookie was present, `ec_finalize_response()` also deletes the cookie and writes a tombstone. Fail-closed / unverifiable-consent cases still return `403`, but they strip EC headers only. - No EC present (`ec_was_present == false`) and consent not denied → `204 No Content`. -- Valid EC, consent granted, KV read succeeds with entry → `200` with full JSON body - including `ec`, `consent`, `uids`, `eids`. -- Valid EC, consent granted, KV read succeeds but no entry (never synced or - `create_or_revive()` failed on generation) → `200` with `degraded: false`, - empty `uids`/`eids`. This is not an error — see §11.4. -- `uids` filtered to partners where `bidstream_enabled = true` and consent - granted. -- KV read error (store unavailable) → `200` with `degraded: true` and empty - `uids`/`eids`. +- Valid EC, consent granted, KV read succeeds with entry → `200` with scoped JSON body + including `ec`, `consent`, `partner_id`, `uid` (single partner's UID), `eid` + (single partner's OpenRTB EID object), `cluster_size`. +- Valid EC, consent granted, KV read succeeds but no entry for this partner → + `200` with `degraded: false`, `uid` and `eid` absent. Not an error — see §11.4. +- KV read error (store unavailable) → `200` with `degraded: true`, `uid` and + `eid` absent. +- Response scoped to the authenticated partner only — no multi-partner `uids`/`eids` maps. +- `X-ts-ec` response header set on `200` responses. - No `Origin` header (server-side proxy): process normally, no CORS headers, no `403`. - `Origin` header present and matches `publisher.domain` or subdomain: reflect in `Access-Control-Allow-Origin` + `Vary: Origin`. - `Origin` header present but does not match: `403`, no body. -- `OPTIONS /identify` preflight → `200` with CORS headers, no body. -- `generate_if_needed()` is never called — no new EC is generated. The handler - itself does not write cookies, but `ec_finalize_response()` may still delete - the cookie on withdrawal or reconcile it on header/cookie mismatch. +- `Access-Control-Allow-Headers` includes `Authorization, X-ts-ec`. +- `OPTIONS /_ts/api/v1/identify` preflight → `200` with CORS headers, no body. +- `generate_if_needed()` is never called — no new EC is generated. - Response time target: 30ms p95 (documented, not gate). -- Unit tests cover all response codes, degraded flag, `uids` filtering, - CORS origin validation. +- Unit tests cover Bearer auth (200/401/403/204), scoped partner response, + degraded flag, CORS origin validation. **Spec ref:** §11 --- -### Story 9 — S2S batch sync (`POST /api/v1/sync`) +### Story 9 — S2S batch sync (`POST /_ts/api/v1/batch-sync`) Implement the server-to-server batch sync endpoint for partners to bulk-write their UIDs against a list of EC hashes. -**Scope:** `ec/sync_batch.rs`, router update +**Scope:** `ec/batch_sync.rs`, `ec/rate_limiter.rs`, router update **Acceptance criteria:** -- Missing or invalid `Authorization: Bearer` → `401`. Auth uses index-based - lookup via `find_by_api_key_hash()` (§9.2) with constant-time hash verification. -- Auth KV lookup failure (store unavailable) → `503 Service Unavailable`. +- Missing or invalid `Authorization: Bearer` → `401`. Auth uses in-memory + lookup via `registry.find_by_api_key_hash()` (§9.2). - API-key rate limit exceeded (`batch_rate_limit` per partner per minute) → `429` with `{ "error": "rate_limit_exceeded" }`. - More than 1000 mappings → `400`. -- Per-mapping rejections: `invalid_ec_hash`, `ec_hash_not_found`, +- Per-mapping rejections: `invalid_ec_id`, `ec_id_not_found`, `consent_withdrawn`, `kv_unavailable`. - KV write failure aborts remaining mappings with `kv_unavailable`; partial results returned as `207`. @@ -2085,7 +2574,7 @@ runtime). Only fires on organic routes (§10.2). - Dispatch only when: EC present (including an EC generated on the current organic request), consent granted, `pull_sync_enabled = true`, and either no - existing partner entry or existing `synced` is older than `pull_sync_ttl_sec`. + existing partner entry; existing partner UIDs are not refreshed by pull sync. - Rate limit: `pull_sync_rate_limit` per EC hash per partner per hour; counter key `pull:{partner_id}:{ec_hash}`. - Maximum concurrent pulls per request: `settings.ec.pull_sync_concurrency` @@ -2100,7 +2589,7 @@ runtime). Only fires on organic routes (§10.2). - Dispatch runs after `send_to_client()` — does not add latency to the user-facing response. Uses `send_async()` + `PendingRequest::wait()` (blocking). - Only fires on organic routes (`handle_publisher_request`, `handle_proxy`) — - never on `/sync`, `/identify`, `/auction`, `/api/v1/sync`, or `/admin/*`. + never on `/_ts/api/v1/identify`, `/_ts/api/v1/batch-sync`, or `/auction`. - Unit tests cover trigger conditions, null/404 no-op, domain allowlist check, dispatch limit enforcement. @@ -2143,21 +2632,21 @@ across multiple handlers in a single simulated environment. **Acceptance criteria:** -- **Full flow:** First-party page load → EC generated → pixel sync writes - partner UID → `/identify` returns that UID → auction includes EID. +- **Full flow:** First-party page load → EC generated → Prebid EID cookie + ingestion writes partner UID → `/_ts/api/v1/identify` returns that UID + (scoped to authenticated partner) → auction includes EID. - **Consent withdrawal:** Request with denied consent clears EC cookie and writes a KV tombstone (`consent.ok = false`, 24h TTL) in the same request; subsequent - `/identify` with consent still denied returns `403` (consent denied → §11.4); + `/_ts/api/v1/identify` with consent still denied returns `403` (consent denied → §11.4); batch sync returns `consent_withdrawn` within the tombstone TTL. - **KV create failure:** EC cookie is still set when `create_or_revive()` fails - (best-effort). Subsequent `/identify` returns `200` with `degraded: false` and + (best-effort). Subsequent `/_ts/api/v1/identify` returns `200` with `degraded: false` and empty `uids`/`eids` (KV read succeeds — entry simply does not exist). -- **KV read failure:** `/identify` returns `200` with `degraded: true` and empty +- **KV read failure:** `/_ts/api/v1/identify` returns `200` with `degraded: true` and empty `uids`/`eids` (store unavailable, entry might exist but can't be read). - **Concurrent writes:** Two simultaneous EC creates for the same hash resolve without data loss (CAS retry). -- **Rate limits:** Pixel sync returns `429` after `sync_rate_limit` is - exceeded; batch sync returns `429` after `batch_rate_limit` is exceeded. +- **Rate limits:** Batch sync returns `429` after `batch_rate_limit` is exceeded. - **Pull sync no-op:** Partner returning `{ "uid": null }` produces no KV write and no error log. - All tests pass under `cargo test --workspace` with Viceroy. diff --git a/docs/superpowers/specs/2026-04-02-pr10-logging-initialization-design.md b/docs/superpowers/specs/2026-04-02-pr10-logging-initialization-design.md new file mode 100644 index 00000000..4bf5af21 --- /dev/null +++ b/docs/superpowers/specs/2026-04-02-pr10-logging-initialization-design.md @@ -0,0 +1,69 @@ +# PR 10 Logging Initialization Design + +**Goal:** Keep logging backend initialization adapter-owned so `trusted-server-core` remains platform-agnostic while Fastly continues to initialize its own `log-fastly` backend. + +## Problem + +`trusted-server-core` still declares a `log-fastly` dependency even though log +backend setup already happens in the Fastly adapter entrypoint. That keeps a +Fastly-specific crate in the core dependency graph and weakens the migration +boundary needed for future EdgeZero adapters such as Cloudflare, Spin, and +Axum. + +## Design + +### Responsibility split + +- `trusted-server-core` emits logs only through `log` macros. +- Each adapter crate owns logger initialization and backend wiring. +- Fastly-specific logger setup moves behind an adapter-local module boundary. + +This keeps core free of platform logging backends while establishing a clean +pattern future adapters can mirror without forcing a shared abstraction too +early. + +### Fastly adapter shape + +Create an adapter-local logging module in +`crates/trusted-server-adapter-fastly/src/logging.rs` that exposes a small +`init_logger()` function. The implementation stays Fastly-specific and can keep +using `log-fastly`, `fern`, and the existing formatting choices. + +`crates/trusted-server-adapter-fastly/src/main.rs` should only import that +module and call `logging::init_logger()` during startup. + +### Core crate shape + +Remove `log-fastly` from +`crates/trusted-server-core/Cargo.toml`. No production code in core should +change unless compilation reveals an unexpected dependency. The intended end +state is that core depends on `log` only. + +## File impact + +- Create: `crates/trusted-server-adapter-fastly/src/logging.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/Cargo.toml` +- Modify: `Cargo.lock` + +## Testing and verification + +- Add or update small adapter-local tests only if needed for the extracted + logging module. +- Run the standard project gates: + - `cargo fmt --all -- --check` + - `cargo clippy --workspace --all-targets --all-features -- -D warnings` + - `cargo test --workspace` + +## Out of scope + +- Introducing a cross-adapter logging trait in core +- Changing log formatting semantics beyond what is needed to extract the module +- Adding logging implementations for non-Fastly adapters + +## Acceptance + +- `log-fastly` exists only in the Fastly adapter dependency graph +- Core uses `log` macros without any Fastly-specific logging backend dependency +- Fastly adapter still initializes logging at startup +- Workspace verification gates pass diff --git a/docs/superpowers/specs/2026-04-15-sourcepoint-gpp-consent-design.md b/docs/superpowers/specs/2026-04-15-sourcepoint-gpp-consent-design.md new file mode 100644 index 00000000..6c23502e --- /dev/null +++ b/docs/superpowers/specs/2026-04-15-sourcepoint-gpp-consent-design.md @@ -0,0 +1,170 @@ +# Sourcepoint GPP Consent for Edge Cookie Generation + +**Issue:** #640 +**Date:** 2026-04-15 +**Status:** Approved + +## Problem + +Edge Cookie (EC) generation fails for sites using Sourcepoint when consent is +stored only in `localStorage` and not surfaced via the standard cookies Trusted +Server reads. Sourcepoint stores US consent under `_sp_user_consent_*` keys in +`localStorage`, including a full GPP string and applicable section IDs. + +Today, Trusted Server only reads consent from `euconsent-v2`, `__gpp`, +`__gpp_sid`, `us_privacy` cookies and the `Sec-GPC` header. Even if `__gpp` / +`__gpp_sid` were present, the server only decodes the EU TCF v2 section from +GPP — it does not use GPP US sections as a consent signal for EC gating. + +This creates two gaps: + +1. **Transport gap:** The server cannot read browser `localStorage`, so no + consent reaches the backend unless client code mirrors it into cookies. +2. **Semantics gap:** Even with `__gpp` / `__gpp_sid` cookies present, current + US-state EC gating does not recognize GPP US sections as valid consent. + +## Approach + +Thin GPP pass-through: mirror Sourcepoint localStorage consent into standard +cookies on the client, and extend server-side EC gating to recognize GPP US +`sale_opt_out` as a consent signal. No compatibility bridge (`us_privacy` +derivation) — both client and server changes ship together. + +## Design + +### 1. Client-side: Sourcepoint JS integration + +New JS-only integration at `crates/js/lib/src/integrations/sourcepoint/index.ts`. +No Rust-side `IntegrationRegistration` (same pattern as `creative`). + +**On page load:** + +1. Scan `localStorage` keys matching `_sp_user_consent_*`. +2. Take the first valid match, parse the JSON value. +3. Extract `gppData.gppString` and `gppData.applicableSections` from the payload. +4. Write first-party cookies: + - `__gpp=` (path `/`, `SameSite=Lax`) + - `__gpp_sid=` (path `/`, `SameSite=Lax`) + - `_ts_gpp_src=sp` marker (path `/`, `SameSite=Lax`) +5. Log what was written for debugging. + +Cookies are session-scoped (no `max-age` / `expires`) since the source of truth +stays in `localStorage` and we re-mirror on each page load. The marker cookie +tracks Trusted Server's Sourcepoint-owned writes so the integration only clears +`__gpp` / `__gpp_sid` values that it previously mirrored; this avoids clobbering +cookies written by other CMPs. This design assumes a single active Sourcepoint +property per page; if multiple `_sp_user_consent_*` entries coexist, the first +valid one wins. The integration runs immediately, performs bounded first-load +retries, and re-mirrors on page focus/visibility refresh so session cookies do +not remain stale after mid-session consent updates. + +### 2. Server-side: GPP US section decoding + +**`crates/trusted-server-core/src/consent/types.rs`** — extend `GppConsent`: + +```rust +pub struct GppConsent { + pub version: u8, + pub section_ids: Vec, + pub eu_tcf: Option, + pub us_sale_opt_out: Option, // new +} +``` + +- `Some(true)` — a US section is present and `sale_opt_out == OptedOut` +- `Some(false)` — a US section is present and `sale_opt_out != OptedOut` +- `None` — no US section exists in the GPP string + +**`crates/trusted-server-core/src/consent/gpp.rs`** — add `decode_us_sale_opt_out`: + +Checks for any US section ID (7–23) in the parsed `GPPString`. For the first +match, decodes the section via `iab_gpp` and extracts `sale_opt_out`. Maps +`OptOut::OptedOut` to `true`, everything else to `false`. + +The `iab_gpp` crate uses different structs per state (`UsNat`, `UsCa`, `UsTn`, +etc.) but they all have `sale_opt_out: OptOut` via `us_common`. We match on the +decoded `Section` enum to extract it. + +### 3. Server-side: EC gating update + +**`crates/trusted-server-core/src/consent/mod.rs`** — update `allows_ec_creation()` +for `Jurisdiction::UsState(_)`. + +New precedence chain: + +``` +GPC → TCF → GPP US sale_opt_out → us_privacy → fail-closed +``` + +Insert between the existing TCF and `us_privacy` branches: + +```rust +// Check GPP US section for sale opt-out. +if let Some(gpp) = &ctx.gpp { + if let Some(opted_out) = gpp.us_sale_opt_out { + return !opted_out; + } +} +``` + +Semantics: + +- GPC still short-circuits at the top and blocks EC creation. +- TCF still takes priority for CMPs like Didomi. In US-state jurisdictions, an + effective TCF Purpose 1 signal is treated as the authoritative EC storage + consent decision and is evaluated before GPP US sale opt-out. +- GPP US `sale_opt_out != OptedOut` → EC allowed when no effective TCF signal is + present. +- GPP US `sale_opt_out == OptedOut` → EC blocked when no effective TCF signal is + present. +- No GPP US section → falls through to `us_privacy`. + +The TCF-before-GPP precedence is intentional rather than accidental: it preserves +existing CMP behavior where TCF Purpose 1 is the explicit storage/access signal +for the EC cookie itself. Publishers that need US-section-wins behavior should +raise that as a separate consent-policy configuration change. + +### 4. Files touched + +| File | Change | +|---|---| +| `crates/js/lib/src/integrations/sourcepoint/index.ts` | New — localStorage auto-discovery, cookie mirroring | +| `crates/js/lib/src/integrations/sourcepoint/index.test.ts` | New — Vitest tests | +| `crates/trusted-server-core/src/consent/types.rs` | Add `us_sale_opt_out: Option` to `GppConsent` | +| `crates/trusted-server-core/src/consent/gpp.rs` | Add US section decoding, extract `sale_opt_out` | +| `crates/trusted-server-core/src/consent/mod.rs` | Add GPP US branch in `allows_ec_creation()`, tests | + +No config changes and no new crate dependencies. `IntegrationRegistry` includes +`sourcepoint` in the JS-only always-shipped module list; the client-side marker +cookie prevents the always-shipped module from clearing other CMPs' GPP cookies. + +### 5. Testing + +**JS (Vitest):** + +- Mirrors `__gpp` and `__gpp_sid` from `_sp_user_consent_*` localStorage +- No cookies written when no `_sp_user_consent_*` key exists +- Graceful handling of malformed JSON in localStorage + +**Rust — EC gating (`consent/mod.rs`):** + +- EC allowed: US state + GPP `us_sale_opt_out = Some(false)` +- EC blocked: US state + GPP `us_sale_opt_out = Some(true)` +- EC blocked: GPC overrides permissive GPP US +- TCF takes priority over GPP US when both present +- GPP US takes priority over `us_privacy` when both present +- No GPP US section falls through to `us_privacy` +- No signals → fail-closed + +**Rust — GPP decoding (`consent/gpp.rs`):** + +- Extracts `us_sale_opt_out` from GPP string with UsNat section (ID 7) +- `us_sale_opt_out` is `None` when GPP has no US sections + +### 6. Non-goals + +- No `us_privacy` compatibility bridge (skipped per decision) +- No richer US GPP field extraction (sharing, targeted advertising opt-outs) +- No publisher configuration for Sourcepoint property ID (auto-discovery) +- No Sourcepoint CMP API integration (localStorage-only approach) +- No consent-policy knob for making GPP US sale opt-out override TCF Purpose 1 diff --git a/fastly.toml b/fastly.toml index 9d6c0f26..2ea512a6 100644 --- a/fastly.toml +++ b/fastly.toml @@ -19,17 +19,32 @@ build = """ [local_server] address = "127.0.0.1:7676" - + [local_server.backends] [local_server.kv_stores] + [[local_server.kv_stores.counter_store]] + key = "placeholder" + data = "placeholder" + + [[local_server.kv_stores.opid_store]] + key = "placeholder" + data = "placeholder" + [[local_server.kv_stores.creative_store]] key = "placeholder" data = "placeholder" - [[local_server.kv_stores.consent_store]] + [[local_server.kv_stores.ec_identity_store]] key = "placeholder" data = "placeholder" + + # Pre-seeded test EC entry for local script testing (test-prebid-eids.sh). + # Matches the TEST_EC_ID used in that script. + [[local_server.kv_stores.ec_identity_store]] + key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01" + data = '{"v":1,"created":1700000000,"last_seen":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US"}}' + [local_server.secret_stores] [[local_server.secret_stores.signing_keys]] key = "ts-2025-10-A" diff --git a/scripts/integration-tests-browser.sh b/scripts/integration-tests-browser.sh index 888adb13..fb1289d3 100755 --- a/scripts/integration-tests-browser.sh +++ b/scripts/integration-tests-browser.sh @@ -32,7 +32,7 @@ echo "==> Validating shared integration-test dependency versions..." echo "==> Building WASM binary (origin=http://127.0.0.1:$ORIGIN_PORT)..." TRUSTED_SERVER__PUBLISHER__ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" \ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET="integration-test-proxy-secret" \ -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY="integration-test-secret-key" \ +TRUSTED_SERVER__EC__PASSPHRASE="integration-test-ec-secret" \ TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 3b9ec974..318b9323 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -53,7 +53,7 @@ fi echo "==> Building WASM binary (origin=http://127.0.0.1:$ORIGIN_PORT)..." TRUSTED_SERVER__PUBLISHER__ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" \ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET="integration-test-proxy-secret" \ -TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY="integration-test-secret-key" \ +TRUSTED_SERVER__EC__PASSPHRASE="integration-test-ec-secret" \ TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 diff --git a/trusted-server.toml b/trusted-server.toml index d9189aaa..4c458682 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -4,7 +4,7 @@ username = "user" password = "pass" [[handlers]] -path = "^/admin" +path = "^/_ts/admin" username = "admin" password = "changeme" @@ -14,8 +14,45 @@ cookie_domain = ".test-publisher.com" origin_url = "https://origin.test-publisher.com" proxy_secret = "change-me-proxy-secret" -[edge_cookie] -secret_key = "trusted-server" +[ec] +passphrase = "local-dev-passphrase-32-bytes-min" +ec_store = "ec_identity_store" +pull_sync_concurrency = 3 +# cluster_trust_threshold = 10 # Entries with cluster_size <= this are individual users +# cluster_recheck_secs = 3600 # Re-evaluate cluster_size after this many seconds + +# [[ec.partners]] +# id = "liveramp" +# name = "LiveRamp" +# source_domain = "liveramp.com" +# openrtb_atype = 3 +# bidstream_enabled = true +# api_token = "partner-api-token" +# batch_rate_limit = 60 +# pull_sync_enabled = false + +[[ec.partners]] +id = "sharedid" +name = "Prebid SharedID" +source_domain = "sharedid.org" +openrtb_atype = 1 +bidstream_enabled = true +api_token = "sharedid-internal-token" + +# Integration test partners (used by crates/integration-tests) +[[ec.partners]] +id = "inttest" +name = "Integration Test Partner" +source_domain = "inttest.example.com" +bidstream_enabled = true +api_token = "inttest-api-key-1" + +[[ec.partners]] +id = "inttest2" +name = "Integration Test Partner 2" +source_domain = "inttest2.example.com" +bidstream_enabled = true +api_token = "inttest2-api-key-2" # Custom headers to be included in every response # Allows publishers to include tags such as X-Robots-Tag: noindex @@ -47,6 +84,15 @@ debug = false # be statically imported in the JS bundle. client_side_bidders = ["rubicon"] +# Optional Prebid.js User ID configuration. The matching User ID modules must +# be included at JS build time with TSJS_PREBID_USER_ID_MODULES, for example: +# TSJS_PREBID_USER_ID_MODULES=sharedId,pubProvided,uid2 node build-all.mjs +# [integrations.prebid.user_sync] +# auctionDelay = 100 +# [[integrations.prebid.user_sync.userIds]] +# name = "sharedId" +# storage = {name = "_sharedID", type = "cookie", expires = 365} + # Zone-specific bid param overrides for Kargo s2s placement IDs. # The JS adapter reads the zone from mediaTypes.banner.name on each ad unit # and includes it in the request. The server maps zone → s2s placementId here. @@ -126,8 +172,9 @@ rewrite_script = true # mode = "restrictive" # "restrictive" | "newest" | "permissive" # freshness_threshold_days = 30 -# KV Store consent persistence (requires a KV store named "consent_store" in fastly.toml) -# consent_store = "consent_store" +# Consent is interpreted from request cookies, headers, geolocation, and these +# policy settings. EC identity lifecycle state and withdrawal tombstones are +# stored in the KV store configured by [ec].ec_store. # Rewrite configuration for creative HTML/CSS processing # [rewrite] @@ -189,5 +236,3 @@ timeout_ms = 1000 # query parameter name. Arrays are joined with commas. [integrations.adserver_mock.context_query_params] permutive_segments = "permutive" - -