Optimize telemetry pipeline and improve batching

Author: cijothomas

codecov.yaml renamed to .github/codecov.yaml

@@ -23,3 +23,7 @@ component_management:
       name: "syslog_cef_receivers"
       paths:
         - "rust/experimental/syslog-cef-receivers/"
+    - component_id: "otel_arrow_go"
+      name: "otel-arrow-go"
+      paths:
+        - "go/"

.github/repository-settings.md

@@ -4,6 +4,14 @@ Maintainers are expected to maintain this log. This is required as per
 [OpenTelemetry Community
 guidelines](https://github.com/open-telemetry/community/blob/main/docs/how-to-configure-new-repository.md#collaborators-and-teams).
 
+## 2025-05-12
+
+- Configure branch protection for `main` branch to 'Require' certain checks:
+  - Go-CI
+    - `pkg\otel` component
+      - test_and_coverage (renamed from build_test)
+    - gen_otelarrowcol
+
 ## 2025-05-07
 
 - Configure branch protection for `main` branch to 'Require' certain checks:

.github/workflows/go-ci.yml

@@ -6,7 +6,40 @@ on: [push, pull_request]
 
 jobs:
 
-  build_test:
+  test_and_coverage:
+    strategy:
+      fail-fast: false
+      matrix:
+        folder:
+          - pkg/otel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: "1.23"
+      - name: Run tests with coverage
+        run: go test -cover -coverprofile=coverage.txt -covermode=atomic ./${{ matrix.folder}}/...
+        working-directory: ./go
+      - name: Upload to codecov.io
+        uses: codecov/codecov-action@ad3126e916f78f00edff4ed0317cf185271ccc2d # v5.4.2
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          fail_ci_if_error: true
+
+  gen_otelarrowcol:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Build the test collector
+        run: make otelarrowcol
+
+  codeql:
     permissions:
       # needed for codeql
       security-events: write
@@ -19,9 +52,5 @@ jobs:
     - uses: github/codeql-action/init@1b549b9259bda1cb5ddde3b41741a82a2d15a841 # v3.28.13
       with:
         languages: go
-    - name: make build
-      run: make build
-    - name: make test
-      run: make test
     - uses: github/codeql-action/analyze@1b549b9259bda1cb5ddde3b41741a82a2d15a841 # v3.28.13
       timeout-minutes: 60

.github/workflows/pipeline-perf-test.yml

@@ -31,9 +31,20 @@ jobs:
             pip install -r tools/pipeline_perf_test/orchestrator/requirements.txt
             pip install -r tools/pipeline_perf_test/load_generator/requirements.txt
 
-        - name: Run pipeline performance tests
+        - name: Run pipeline performance test with default configuration
           run: |
             cd tools/pipeline_perf_test
-            python orchestrator/orchestrator.py --collector-config system_under_test/otel-collector/collector-config.yaml --duration 30
+            python orchestrator/orchestrator.py --collector-config system_under_test/otel-collector/collector-config.yaml --duration 30 --results-dir ./results/default
+
+        - name: Run pipeline performance test with batch processor configuration
+          run: |
+            cd tools/pipeline_perf_test
+            python orchestrator/orchestrator.py --collector-config system_under_test/otel-collector/collector-config-with-batch-processor.yaml --duration 30 --results-dir ./results/batch-processor
+
+        - name: Upload performance test results
+          uses: actions/upload-artifact@v4
+          with:
+            name: performance-test-results
+            path: tools/pipeline_perf_test/results/
 
 

.github/workflows/rust-ci.yml

@@ -31,6 +31,8 @@ jobs:
           toolchain: stable
       - name: install cargo-llvm-cov
         uses: taiki-e/install-action@5075451c95db43b063f20f0c8fef04c04d5bf0ba # cargo-llvm-cov
+      - name: Build the test collector
+        run: make otelarrowcol
       - name: Run tests with coverage
         run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
         working-directory: ./rust/${{ matrix.folder }}

.gitignore

@@ -53,3 +53,4 @@ coverage.html
 
 # Collector test binary
 collector/cmd/otelarrowcol/otelarrowcol
+tools/pipeline_perf_test/load_generator/loadgen-manifest.yaml.tmp

CHANGELOG.md

@@ -9,6 +9,8 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## Unreleased
 
+- Remove `concurrentbatch` and `obfuscation` processors. [#409](https://github.com/open-telemetry/otel-arrow/pull/409)
+
 ## [0.35.0](https://github.com/open-telemetry/otel-arrow/releases/tag/v0.35.0) - 2025-04-07
 
 - Upgrade to v0.123.x of collector dependencies.

Makefile

@@ -32,7 +32,7 @@ gotidy:
 	for dir in $(GODIRS); do (cd $${dir}; GOWORK="off" $(GOCMD) mod tidy) || exit 1; done
 
 doc:
-	$(GOCMD) run tools/data_model_gen/main.go
+	$(GOCMD) run go/tools/data_model_gen/main.go
 
 # Multimod can be installed using:
 #

README.md

@@ -4,14 +4,22 @@
 [![Go-CI](https://github.com/open-telemetry/otel-arrow/actions/workflows/go-ci.yml/badge.svg)](https://github.com/open-telemetry/otel-arrow/actions/workflows/go-ci.yml)
 [![Rust-CI](https://github.com/open-telemetry/otel-arrow/actions/workflows/rust-ci.yml/badge.svg)](https://github.com/open-telemetry/otel-arrow/actions/workflows/rust-ci.yml)
 [![OpenSSF Scorecard for otel-arrow](https://api.scorecard.dev/projects/github.com/open-telemetry/otel-arrow/badge)](https://scorecard.dev/viewer/?uri=github.com/open-telemetry/otel-arrow)
+[![codecov](https://codecov.io/gh/open-telemetry/otel-arrow/graph/badge.svg?token=7u3gFLH54G)](https://codecov.io/gh/open-telemetry/otel-arrow)
 
-The [OpenTelemetry Protocol with Apache
+The [OpenTelemetry with Apache
 Arrow](https://github.com/open-telemetry/otel-arrow) project is an
 effort within [OpenTelemetry](https://opentelemetry.io/) to use
 [Apache Arrow](https://arrow.apache.org/) libraries for bulk data
-transport in OpenTelemetry pipelines.  This repository is
-the home of the OpenTelemetry Protocol with Apache Arrow protocol,
-which we refer to as "OTAP", and reference implementations.
+transport in OpenTelemetry pipelines.  This repository is the home of
+the OpenTelemetry Protocol with Apache Arrow protocol, which we refer
+to as "OTAP", and reference implementations in Golang and Rust.
+
+The [OpenTelemetry Arrow project has an OpenTelemetry SIG with weekly
+meetings][PROJECTDEF]. Find our [meeting schedule on the OpenTelemetry
+calendar][OTELCAL].
+
+[PROJECTDEF]: https://github.com/open-telemetry/community/blob/main/projects/otelarrow.md
+[OTELCAL]: https://github.com/open-telemetry/community/blob/main/README.md#calendar
 
 ## Quick start
 
@@ -22,6 +30,10 @@ for details.
 The [examples](./collector/examples/README.md) in this repository use
 a test collector named `otelarrowcol`, see [collector/BUILDING.md](./collector/BUILDING.md).
 
+[Phase 2 of the project is being actively developed](#phase-2). We are
+building an end-to-end OpenTelemetry Arrow pipeline in Rust.  Find us
+in the CNCF Slack `#otel-arrow` channel.
+
 [RECEIVER]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/otelarrowreceiver/README.md
 [EXPORTER]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/otelarrowexporter/README.md
 
@@ -229,12 +241,12 @@ you're more than welcome to participate!
 
 - [Joshua MacDonald](https://github.com/jmacd), Microsoft
 - [Laurent Quérel](https://github.com/lquerel), F5
+- [Drew Relmas](https://github.com/drewrelmas), Microsoft
 
 [Approvers](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#approver)
 ([@open-telemetry/arrow-approvers](https://github.com/orgs/open-telemetry/teams/arrow-approvers)):
 
 - [Alex Boten](https://github.com/codeboten), Honeycomb
-- [Drew Relmas](https://github.com/drewrelmas), Microsoft
 - [Moh Osman](https://github.com/moh-osman3), ServiceNow
 
 ### Thanks to all the people who have contributed

collector/README.md

@@ -19,26 +19,6 @@ official OpenTelemetry Collector-Contrib release images since v0.105.0.
 
 See [Building][BUILDING].
 
-## Components included in this repository
-
-Several components were developed to facilitate testing and debugging the
-primary OpenTelemetry Protocol with Apache Arrow components.  Most importantly,
-these tools can be used to report problematic data to the maintainers.  These
-components are:
-
-### For production use
-
-- [`processor/concurrentbatchprocessor`][CONCURRENTBATCHPROCESSOR]: Derived from
-  the upstream [batchprocessor][UPSTREAMBATCHPROCESSOR], this component is
-  enhanced with the ability to send batches concurrently, with an overall
-  in-flight-bytes limit.
-
-### For research and validation
-
-- [`processor/obfuscationprocessor`][OBFUSCATIONPROCESSOR]: Supports obfuscation
-  of OpenTelemetry data using a [Feistel
-  cipher](https://en.wikipedia.org/wiki/Feistel_cipher).
-
 ## Other components built into `otelarrowcol`
 
 Several Collector-Contrib extensions are included in the build:
@@ -57,22 +37,43 @@ use the upstream [fileexporter][FILEEXPORTER] and
 
 From the core collector repository:
 
-- [otelhttpexporter][UPSTREAMHTTPOTLP]:  Useful for debugging, sends standard
+- [otlphttpexporter][UPSTREAMHTTPOTLP]:  Useful for debugging, sends standard
       OTLP over HTTP
+- [otlpexporter][UPSTREAMOTLP]:  Useful for testing and validation, the
+      core OTLP exporter
 - [debugexporter][UPSTREAMDEBUG]:   Useful for debugging, prints OTLP data to
       the console
 
+## Phase 1 components (removed)
+
+During Phase 1 of the project, several components were built that
+could not be maintained given the pace of OpenTelemetry Collector
+development.  Notable former components:
+
+- **Concurrent Batch Processor**: This component was at the time an
+  improvement on the OpenTelemetry core `batchprocessor`. Today we
+  recommend the `exporterhelper` built-in support for batching,
+  configurable after the queue sender. [Removed.][PHASE1REMOVAL1]
+- **Obfuscation Processor**: This component applied a Feistel cipher to all
+  string fields of the OpenTelemetry data model. This could be revived as
+  a Collector-Contrib component. [Removed.][PHASE1REMOVAL1]
+- **Validation Connector**: This component routed telemetry in two
+  ways such that a collector could self-validate an OTel-Arrow
+  bridge. It was difficult to make ensure reliable connector ordering
+  at startup, required changes in the core Collector libraries.
+  [Removed.][PHASE1REMOVAL3]
+- **Protobuf/JSON file exporter and receiver**: These components had
+  support for reading compressed protocol buffers, matching data that
+  we used in Phase 1. Today we recommend the Collector-Contrib
+  `fileexporter` and `otlpjsonfilereceiver`. [Removed.][PHASE1REMOVAL2]
+
 [BUILDING]: ./BUILDING.md
 [COLLECTORCONTRIB]:
     https://github.com/open-telemetry/opentelemetry-collector-contrib
 [ARROWEXPORTER]:
     https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/otelarrowexporter/README.md
 [ARROWRECEIVER]:
     https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/otelarrowreceiver/README.md
-[UPSTREAMBATCHPROCESSOR]:
-    https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/batchprocessor/README.md
-[CONCURRENTBATCHPROCESSOR]: ./processor/concurrentbatchprocessor/README.md
-[OBFUSCATIONPROCESSOR]: ./processor/obfuscationprocessor/README.md
 [FILEEXPORTER]:
     https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/fileexporter/README.md
 [FILERECEIVER]:
@@ -85,5 +86,10 @@ From the core collector repository:
     https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/pprofextension/README.md
 [UPSTREAMHTTPOTLP]:
     https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/otlphttpexporter/README.md
+[UPSTREAMOTLP]:
+    https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/otlpexporter/README.md
 [UPSTREAMDEBUG]:
     https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/debugexporter/README.md
+[PHASE1REMOVAL1]: https://github.com/open-telemetry/otel-arrow/pull/409
+[PHASE1REMOVAL2]: https://github.com/open-telemetry/otel-arrow/pull/278
+[PHASE1REMOVAL3]: https://github.com/open-telemetry/otel-arrow/pull/252