Add support for generator http client

Cargo.toml

@@ -15,16 +15,24 @@ path = "src/main.rs"
 
 [dependencies]
 anyhow = "1.0.86"
+async-trait = "0.1.81"
+async-stream = "0.3.5"
 axum = { version = "0.7.5", features = ["json"] }
 axum-extra = "0.9.3"
 clap = { version = "4.5.15", features = ["derive", "env"] }
 futures = "0.3.30"
 ginepro = "0.8.1"
+http-serde = "2.1.1"
 hyper = { version = "1.4.1", features = ["http1", "http2", "server"] }
 hyper-util = { version = "0.1.7", features = ["server-auto", "server-graceful", "tokio"] }
 mio = "1.0.2"
+opentelemetry = { version = "0.24.0", features = ["trace", "metrics"] }
+opentelemetry-http = { version = "0.13.0", features = ["reqwest"] }
+opentelemetry-otlp = { version = "0.17.0", features = ["http-proto"] }
+opentelemetry_sdk = { version = "0.24.1", features = ["rt-tokio", "metrics"] }
 prost = "0.13.1"
 reqwest = { version = "0.12.5", features = ["blocking", "rustls-tls", "json"] }
+reqwest-eventsource = "0.6.0"
 rustls = {version = "0.23.12", default-features = false, features = ["std"]}
 rustls-pemfile = "2.1.3"
 rustls-webpki = "0.102.6"
@@ -36,13 +44,13 @@ tokio = { version = "1.39.2", features = ["rt", "rt-multi-thread", "parking_lot"
 tokio-rustls = { version = "0.26.0" }
 tokio-stream = { version = "0.1.15", features = ["sync"] }
 tonic = { version = "0.12.1", features = ["tls", "tls-roots", "tls-webpki-roots"] }
+tower-http = { version = "0.5.2", features = ["trace"] }
 tower-service = "0.3"
 tracing = "0.1.40"
+tracing-opentelemetry = "0.25.0"
 tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter"] }
 url = "2.5.2"
 uuid = { version = "1.10.0", features = ["v4", "fast-rng"] }
-async-trait = "0.1.81"
-async-stream = "0.3.5"
 
 [build-dependencies]
 tonic-build = "0.12.1"

Dockerfile

@@ -1,6 +1,7 @@
 ARG UBI_MINIMAL_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi-minimal
 ARG UBI_BASE_IMAGE_TAG=latest
 ARG PROTOC_VERSION=26.0
+ARG CONFIG_FILE=config/config.yaml
 
 ## Rust builder ################################################################
 # Specific debian version so that compatible glibc version is used
@@ -23,10 +24,10 @@ RUN rustup component add rustfmt
 ## Orchestrator builder #########################################################
 FROM rust-builder as fms-guardrails-orchestr8-builder
 
-COPY build.rs *.toml LICENSE /app
-COPY config/ /app/config
-COPY protos/ /app/protos
-COPY src/ /app/src
+COPY build.rs *.toml LICENSE /app/
+COPY ${CONFIG_FILE} /app/config/config.yaml
+COPY protos/ /app/protos/
+COPY src/ /app/src/
 
 WORKDIR /app
 
@@ -50,7 +51,7 @@ RUN cargo fmt --check
 FROM ${UBI_MINIMAL_BASE_IMAGE}:${UBI_BASE_IMAGE_TAG} as fms-guardrails-orchestr8-release
 
 COPY --from=fms-guardrails-orchestr8-builder /app/bin/ /app/bin/
-COPY config /app/config
+COPY ${CONFIG_FILE} /app/config/config.yaml
 
 RUN microdnf install -y --disableplugin=subscription-manager shadow-utils compat-openssl11 && \
     microdnf clean all --disableplugin=subscription-manager

config/config.yaml

@@ -11,6 +11,12 @@ generation:
     service:
         hostname: localhost
         port: 8033
+# Generation server used for chat endpoints
+# chat_generation:
+#   service:
+#     hostname: localhost
+#     port: 8080
+#   # health_service:
 # Any chunker servers that will be used by any detectors
 chunkers:
     # Chunker ID/name
@@ -26,11 +32,16 @@ chunkers:
 detectors:
     # Detector ID/name to be used in user requests
     hap-en:
+        # Detector type (text_contents, text_generation, text_chat, text_context_doc)
+        type: text_contents
         service:
-            hostname: https://localhost/api/v1/text/contents # Full url / endpoint currently expected
+            hostname: localhost
             port: 8080
             # TLS ID/name, optional (detailed in `tls` section)
             tls: detector
+        health_service:
+            hostname: localhost
+            port: 8081
         # Chunker ID/name from `chunkers` section if applicable
         chunker_id: en_regex
         # Default score threshold for a detector. If a user
@@ -53,8 +64,7 @@ tls:
     detector_bundle_no_ca:
         cert_path: /path/to/client-bundle.pem
         insecure: true
-
 # Following section can be used to configure the allowed headers that orchestrator will pass to
 # NLP provider and detectors. Note that, this section takes header keys, not values.
 # passthrough_headers:
-#     - header-key
+#     - header-key