[otap-df-quiver] Scaffolding for otap-df-quiver crate (#1495)

Introduces the experimental `otap-df-quiver` crate into the main
`otap-dataflow` Rust workspace. Changes include configuration
scaffolding, placeholder APIs, and benchmarking stubs, but do not yet
implement actual data persistence. Integration is opt-in via the new
`quiver-persistence` Cargo feature, which is disabled by default to keep
the experimental code path out of release builds.

Author: AaronRM

rust/experimental/quiver/README.md

@@ -1,12 +1,36 @@
 # Quiver (Experimental) - Arrow-Based Persistence for OTAP Dataflow - README
 
-This crate hosts the experimental Arrow-based persistence components described
-in [ARCHITECTURE.md](./ARCHITECTURE.md). It provides a standalone persistence
-library which embed into `otap-dataflow` or other telemetry pipelines that need durable
-buffering of Arrow payloads.
+The source for the experimental crate now lives under
+`rust/otap-dataflow/crates/quiver`. This directory keeps the architecture notes
+and high level design documents, while the shared `otap-dataflow` workspace
+hosts the actual Rust code so the project inherits the same lint, fmt, and test
+infrastructure.
+
+The crate currently exposes configuration scaffolding, placeholder engine APIs,
+and Criterion bench harness stubs. No bytes are persisted yet; every ingest
+call intentionally returns `QuiverError::Unimplemented` until Phase 1 wires in
+the WAL.
+
+Integration with the `otap-df` binary is opt-in via the Cargo feature
+`quiver-persistence`. The feature is *disabled by default*, so release builds
+never pull in the experimental persistence code path unless the flag is
+explicitly enabled.
 
 ## Status
 
-**Not production-ready** This crate is being prototyped based on the specifications
-below (which may be updated as development proceeds). It is not yet stable or suitable
-for taking a dependency on.
+**Not production-ready** This crate is being developed based on the
+specifications in `ARCHITECTURE.md` (which may be updated as development
+proceeds). It is not yet stable or suitable for taking a dependency on.
+
+## Quick start
+
+```bash
+cd rust/otap-dataflow
+cargo test -p otap-df-quiver      # unit tests + doc tests
+cargo bench -p otap-df-quiver     # opt-in Criterion bench stub
+# Enable the downstream integration (still a stub) when needed
+cargo test -p otap-df --features quiver-persistence
+```
+
+The bench currently measures the placeholder ingest path so we have a home for
+future perf instrumentation once real I/O lands.

rust/otap-dataflow/Cargo.toml

@@ -34,6 +34,7 @@ otap-df-config.workspace = true
 otap-df-controller.workspace = true
 otap-df-otap.workspace = true
 thiserror.workspace = true
+quiver = { workspace = true, optional = true }
 serde_json.workspace = true
 clap.workspace = true
 mimalloc-rust.workspace = true
@@ -44,12 +45,14 @@ otap-df-pdata-otlp-model = { path = "./crates/pdata/src/otlp/model"}
 otap-df-config = { path = "crates/config" }
 otap-df-controller = { path = "crates/controller" }
 otap-df-otap = { path = "crates/otap" }
+quiver = { package = "otap-df-quiver", path = "crates/quiver" }
 
 ahash = "0.8.11"
 arrayvec = "0.7.6"
 arrow = "57.0"
 arrow-ipc = { version = "57.0", features=["zstd"] }
 arrow-schema = { version = "57.0" }
+arrow-array = { version = "57.0" }
 async-stream = "0.3.6"
 async-trait = "0.1.88"
 async-unsync = "0.3.0"
@@ -149,6 +152,8 @@ default = []
 unsafe-optimizations = ["unchecked-index", "unchecked-arithmetic"]
 unchecked-index = []
 unchecked-arithmetic = []
+# Experimental persistence prototype (opt-in)
+quiver-persistence = ["quiver"]
 # Experimental exporters (opt-in)
 experimental-exporters = ["otap-df-otap/experimental-exporters"]
 geneva-exporter = ["otap-df-otap/geneva-exporter"]

rust/experimental/quiver/ARCHITECTURE.md renamed to rust/otap-dataflow/crates/quiver/ARCHITECTURE.md

@@ -0,0 +1,37 @@
+[package]
+name = "otap-df-quiver"
+description = "Embeddable Arrow-based segment store"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+repository.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+keywords = ["opentelemetry", "arrow", "persistence", "quiver"]
+categories = ["database", "data-structures"]
+
+[lib]
+name = "quiver"
+path = "src/lib.rs"
+
+[features]
+default = ["otap-dataflow-integrations"]
+otap-dataflow-integrations = []
+serde = ["dep:serde"]
+
+[dependencies]
+arrow-array.workspace = true
+arrow-schema.workspace = true
+serde = { workspace = true, optional = true }
+thiserror.workspace = true
+
+[dev-dependencies]
+criterion.workspace = true
+
+[lints]
+workspace = true
+
+[[bench]]
+name = "ingest"
+harness = false

rust/otap-dataflow/crates/quiver/Cargo.toml

@@ -0,0 +1,33 @@
+# Quiver (Experimental) - Arrow-Based Persistence for OTAP Dataflow - README
+
+Quiver is a standalone, embeddable Arrow-based segment store
+packaged as a reusable Rust crate. See `ARCHITECTURE.md`
+for more details.
+
+The crate currently exposes configuration scaffolding, placeholder engine APIs,
+and Criterion bench harness stubs. No bytes are persisted yet; every ingest
+call intentionally returns `QuiverError::Unimplemented`.
+
+Integration with the `otap-df` binary is opt-in via the Cargo feature
+`quiver-persistence`. The feature is *disabled by default*, so release builds
+never pull in the experimental persistence code path unless the flag is
+explicitly enabled.
+
+## Status
+
+**Not production-ready** This crate is being prototyped based on the
+specifications in `ARCHITECTURE.md` (which may be updated as development proceeds).
+*It is not yet complete, stable or suitable for taking a dependency on.*
+
+## Quick start
+
+```bash
+cd rust/otap-dataflow
+cargo test -p otap-df-quiver      # unit tests + doc tests
+cargo bench -p otap-df-quiver     # opt-in Criterion bench stub
+# Enable the downstream integration (still a stub) when needed
+cargo test -p otap-df --features quiver-persistence
+```
+
+The bench currently measures the placeholder ingest path so we have a home for
+future perf instrumentation once real I/O lands.

rust/otap-dataflow/crates/quiver/README.md

@@ -0,0 +1,71 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//! Criterion micro-benchmark for the placeholder Quiver ingest path.
+
+#![allow(missing_docs)]
+
+use std::sync::Arc;
+use std::time::SystemTime;
+
+use arrow_array::RecordBatch;
+use arrow_schema::{DataType, Field, Schema};
+use criterion::{Criterion, criterion_group, criterion_main};
+use quiver::config::QuiverConfig;
+use quiver::engine::QuiverEngine;
+use quiver::record_bundle::{BundleDescriptor, PayloadRef, RecordBundle, SlotDescriptor, SlotId};
+
+struct DummyBundle {
+    descriptor: BundleDescriptor,
+    batch: RecordBatch,
+}
+
+impl DummyBundle {
+    fn new() -> Self {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "value",
+            DataType::Int64,
+            false,
+        )]));
+        Self {
+            descriptor: BundleDescriptor::new(vec![SlotDescriptor::new(SlotId::new(0), "Logs")]),
+            batch: RecordBatch::new_empty(schema),
+        }
+    }
+}
+
+impl RecordBundle for DummyBundle {
+    fn descriptor(&self) -> &BundleDescriptor {
+        &self.descriptor
+    }
+
+    fn ingestion_time(&self) -> SystemTime {
+        SystemTime::now()
+    }
+
+    fn payload(&self, slot: SlotId) -> Option<PayloadRef<'_>> {
+        if slot == SlotId::new(0) {
+            Some(PayloadRef {
+                schema_fingerprint: [0; 32],
+                batch: &self.batch,
+            })
+        } else {
+            None
+        }
+    }
+}
+
+/// Measures the placeholder ingest path so we can wire perf tooling later.
+fn ingest_placeholder(c: &mut Criterion) {
+    let engine = QuiverEngine::new(QuiverConfig::default()).expect("config valid");
+    let bundle = DummyBundle::new();
+
+    let _ = c.bench_function("ingest_placeholder", |b| {
+        b.iter(|| {
+            let _ = engine.ingest(&bundle);
+        });
+    });
+}
+
+criterion_group!(benches, ingest_placeholder);
+criterion_main!(benches);