Streaming pings

.buildkite/pipeline.yml

@@ -13,24 +13,44 @@ steps:
   - name: ":go::robot_face: Lint"
     key: check-code-committed
     command: .buildkite/steps/check-code-committed.sh
-    if_changed: "{go.mod,go.sum,**.go,.buildkite/steps/check-code-committed.sh}"
+    if_changed:
+      - go.{mod,sum}
+      - "**.go"
+      - .buildkite/steps/check-code-committed.sh
     plugins:
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
           cli-version: 2
           mount-buildkite-agent: true
           run: lint
 
+  - name: ":go::robot_face: Check protobuf generation"
+    key: check-protobuf-genreation
+    command: .buildkite/steps/check-protobuf-generation.sh
+    if_changed:
+      - api/proto/**
+      - .buildkite/steps/check-protobuf-generation.sh
+    plugins:
+    - docker-compose#v4.14.0:
+        config: .buildkite/docker-compose.yml
+        cli-version: 2
+        mount-buildkite-agent: true
+        run: lint
+
   - group: ":go::scientist: Tests and Coverage"
-    if_changed: "{go.mod,go.sum,**.go,**/fixtures/**,.buildkite/steps/{tests,test-coverage-report}.sh}"
+    if_changed:
+      - go.{mod,sum}
+      - "**.go"
+      - "**/fixtures/**"
+      - .buildkite/steps/{tests,test-coverage-report}.sh
     steps:
     - name: ":linux: Linux AMD64 Tests"
       key: test-linux-amd64
       command: ".buildkite/steps/tests.sh"
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       plugins:
         - docker-compose#v4.14.0:
             config: .buildkite/docker-compose.yml
@@ -51,7 +71,7 @@ steps:
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
       plugins:
@@ -74,7 +94,7 @@ steps:
       parallelism: 2
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_WINDOWS_QUEUE
       plugins:
@@ -93,7 +113,7 @@ steps:
       parallelism: 3
       artifact_paths:
         - junit-*.xml
-        - "coverage/**/*"
+        - "coverage-*/**"
       agents:
         queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
       plugins:
@@ -112,7 +132,7 @@ steps:
 
     - name: ":coverage: Test coverage report Linux ARM64"
       key: test-coverage-linux-arm64
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-arm64"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -124,12 +144,11 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-linux-arm64
+            download: "coverage-linux-arm64/**"
 
     - name: ":coverage: Test coverage report Linux AMD64"
       key: test-coverage-linux-amd64
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-amd64"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -141,12 +160,11 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-linux-amd64
+            download: "coverage-linux-amd64/**"
 
     - name: ":coverage: Test coverage report Linux ARM64 Race"
       key: test-coverage-linux-arm64-race
-      command: ".buildkite/steps/test-coverage-report.sh"
+      command: ".buildkite/steps/test-coverage-report.sh coverage-linux-arm64-race"
       artifact_paths:
         - "cover.html"
         - "cover.out"
@@ -158,8 +176,7 @@ steps:
             cli-version: 2
             run: agent
         - artifacts#v1.9.4:
-            download: "coverage/**"
-            step: test-race-linux-arm64
+            download: "coverage-linux-arm64-race/**"
 
     - label: ":writing_hand: Annotate with Test Failures"
       depends_on:

.buildkite/steps/check-protobuf-generation.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+
+set -euf
+
+cd api/proto
+
+echo --- :buf: Installing buf...
+go install github.com/bufbuild/buf/cmd/buf@v1.61.0
+go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.10
+go install connectrpc.com/connect/cmd/protoc-gen-connect-go@v1.19.1
+
+echo --- :connectrpc: Checking protobuf file generation...
+buf generate
+if ! git diff --no-ext-diff --exit-code; then
+  echo ^^^ +++
+  echo "Generated protobuf files are out of sync with the source code"
+  echo "Please run \`buf generate\` in the internal/proto directory locally, and commit the result."
+  exit 1
+fi

.buildkite/steps/test-coverage-report.sh

@@ -2,5 +2,5 @@
 set -euo pipefail
 
 echo 'Producing coverage report'
-go tool covdata textfmt -i "coverage" -o cover.out
+go tool covdata textfmt -i "$1" -o cover.out
 go tool cover -html cover.out -o cover.html

.buildkite/steps/tests.sh

@@ -4,6 +4,11 @@ set -euo pipefail
 go version
 echo arch is "$(uname -m)"
 
+RACE=''
+if [[ $* == *-race* ]] ; then
+  RACE='-race'
+fi
+
 export BUILDKITE_TEST_ENGINE_SUITE_SLUG=buildkite-agent
 export BUILDKITE_TEST_ENGINE_TEST_RUNNER=gotest
 export BUILDKITE_TEST_ENGINE_RESULT_PATH="junit-${BUILDKITE_JOB_ID}.xml"
@@ -13,8 +18,8 @@ if [[ "$(go env GOOS)" == "windows" ]]; then
   # need a Windows VM to debug.
   export BUILDKITE_TEST_ENGINE_TEST_CMD="go tool gotestsum --junitfile={{resultPath}} -- -count=1 $* {{packages}}"
 else
-  mkdir -p coverage
-  COVERAGE_DIR="$PWD/coverage"
+  COVERAGE_DIR="${PWD}/coverage-$(go env GOOS)-$(go env GOARCH)${RACE}"
+  mkdir -p "${COVERAGE_DIR}"
   export BUILDKITE_TEST_ENGINE_TEST_CMD="go tool gotestsum --junitfile={{resultPath}} -- -count=1 -cover $* {{packages}} -test.gocoverdir=${COVERAGE_DIR}"
 fi
 

agent/agent_configuration.go

@@ -69,4 +69,6 @@ type AgentConfiguration struct {
 	TraceContextEncoding         string
 	DisableWarningsFor           []string
 	AllowMultipartArtifactUpload bool
+
+	PingMode string
 }

agent/agent_pool.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"strconv"
 	"sync"
+	"time"
 
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/status"
@@ -17,13 +18,15 @@ import (
 
 // AgentPool manages multiple parallel AgentWorkers.
 type AgentPool struct {
-	workers []*AgentWorker
+	workers     []*AgentWorker
+	idleTimeout time.Duration
 }
 
 // NewAgentPool returns a new AgentPool.
-func NewAgentPool(workers []*AgentWorker) *AgentPool {
+func NewAgentPool(workers []*AgentWorker, config *AgentConfiguration) *AgentPool {
 	return &AgentPool{
-		workers: workers,
+		workers:     workers,
+		idleTimeout: config.DisconnectAfterIdleTimeout,
 	}
 }
 
@@ -58,7 +61,7 @@ func (r *AgentPool) Start(ctx context.Context) error {
 	defer done()
 	setStat("🏃 Spawning workers...")
 
-	idleMon := newIdleMonitor(len(r.workers))
+	idleMon := NewIdleMonitor(ctx, len(r.workers), r.idleTimeout)
 
 	errCh := make(chan error)
 
@@ -82,7 +85,7 @@ func (r *AgentPool) Start(ctx context.Context) error {
 func runWorker(ctx context.Context, worker *AgentWorker, idleMon *idleMonitor) error {
 	agentWorkersStarted.Inc()
 	defer agentWorkersEnded.Inc()
-	defer idleMon.markDead(worker)
+	defer idleMon.MarkDead(worker)
 
 	// Connect the worker to the API
 	if err := worker.Connect(ctx); err != nil {