Skip to content

SRE-3584 Fix test reporting #17568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JohnMalmberg wants to merge 1 commit into
base: master
Choose a base branch
from
+46 −24
Open

SRE-3584 Fix test reporting #17568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -886,7 +886,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand All @@ -908,7 +909,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand All @@ -930,7 +932,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand All @@ -952,7 +955,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2',
image_version: 'leap15.6'))
}
Expand All @@ -975,7 +979,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand Down
43 changes: 27 additions & 16 deletions ci/functional/test_main.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright 2020-2024 Intel Corporation.
# Copyright 2025 Hewlett Packard Enterprise Development LP
# Copyright 2025-2026 Hewlett Packard Enterprise Development LP
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
Expand All @@ -14,6 +14,12 @@ fi

test_tag="$TEST_TAG"

: "${NODELIST:=localhost}"
: "${TEST_RPMS:=false}"

def_node_count="$(echo "$NODELIST" | awk -F, '{print NF}')"
: "${NODE_COUNT:=$def_node_count}"

tnodes=$(echo "$NODELIST" | cut -d ',' -f 1-"$NODE_COUNT")
first_node=${NODELIST%%,*}

Expand Down Expand Up @@ -42,14 +48,17 @@ cluster_reboot () {
test_cluster() {
# Test that all nodes in the cluster are healthy
clush -B -S -o '-i ci_key' -l root -w "${tnodes}" \
"OPERATIONS_EMAIL=${OPERATIONS_EMAIL} \
"OPERATIONS_EMAIL=${OPERATIONS_EMAIL:-} \
FIRST_NODE=${first_node} \
TEST_RPMS=${TEST_RPMS} \
NODELIST=${tnodes} \
BUILD_URL=\"${BUILD_URL:-Unknown in GHA}\" \
STAGE_NAME=\"$STAGE_NAME\" \
STAGE_NAME=\"${STAGE_NAME:-Unknown}\" \
JENKINS_URL=\"${JENKINS_URL:-}\" \
DAOS_DEVOPS_EMAIL=\"${DAOS_DEVOPS_EMAIL:-}\" \
DAOS_INFINIBAND=${DAOS_INFINIBAND:-} \
DAOS_NVME=${DAOS_NVME:-} \
DAOS_PMEM=${DAOS_PMEM:-} \
$(cat ci/functional/test_main_prep_node.sh)"
}

Expand All @@ -65,7 +74,7 @@ if ! test_cluster; then
echo "Hardware test failed again after reboot"
fi
else
echo "Cluster reboot failed"
echo "Cluster reboot failed"
fi
else
hardware_ok=true
Expand All @@ -88,8 +97,10 @@ trap 'clush -B -S -o "-i ci_key" -l root -w "${tnodes}" '\

# Setup the Jenkins build artifacts directory before running the tests to ensure
# there is enough disk space to report the results.
rm -rf "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
mkdir "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
if [ -n "${STAGE_NAME:-}" ]; then
rm -rf "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
mkdir "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
fi

# set DAOS_TARGET_OVERSUBSCRIBE env here
export DAOS_TARGET_OVERSUBSCRIBE=1
Expand All @@ -98,24 +109,24 @@ rm -rf install/lib/daos/TESTING/ftest/avocado ./*_results.xml
mkdir -p install/lib/daos/TESTING/ftest/avocado/job-results

if "$hardware_ok"; then
if $TEST_RPMS; then
if "$TEST_RPMS"; then
# shellcheck disable=SC2029
ssh -i ci_key -l jenkins "${first_node}" \
"TEST_TAG=\"$test_tag\" \
TNODES=\"$tnodes\" \
FTEST_ARG=\"${FTEST_ARG:-}\" \
WITH_VALGRIND=\"${WITH_VALGRIND:-}\" \
STAGE_NAME=\"$STAGE_NAME\" \
DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \
ssh -i ci_key -l jenkins "${first_node}" \
"TEST_TAG=\"$test_tag\" \
TNODES=\"$tnodes\" \
FTEST_ARG=\"${FTEST_ARG:-}\" \
WITH_VALGRIND=\"${WITH_VALGRIND:-}\" \
STAGE_NAME=\"${STAGE_NAME:-Unknown}\" \
DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \
$(cat ci/functional/test_main_node.sh)"
else
./ftest.sh "$test_tag" "$tnodes" "$FTEST_ARG"
./ftest.sh "$test_tag" "$tnodes" "${FTEST_ARG:-}"
fi
fi

# Now rename the previously collected hardware test data for Jenkins
# to use them for Junit processing.
: "${STAGE_NAME:=}"
: "${STAGE_NAME:=unknown}"
mkdir -p "${STAGE_NAME}/hardware_prep/"
for node in ${tnodes//,/ }; do
old_name="./hardware_prep_node_results.xml.$node"
Expand Down
12 changes: 9 additions & 3 deletions ci/functional/test_main_prep_node.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ mail_domain="${domain1%%/*}"
: "${DAOS_PMEM:=0}"
: "${DAOS_NVME:=0}"

#cn is for a cleaned up stage name.
cn=$(echo "$STAGE_NAME" | sed 's/[^a-zA-Z0-9_]/_/g' | sed 's/__*/_/g')

result=0
mail_message=''
mail_type='warning'
nl="
"

Expand Down Expand Up @@ -51,7 +55,7 @@ function do_mail {
fi
# shellcheck disable=SC2059
build_info="BUILD_URL = $BUILD_URL$nl STAGE = $STAGE_NAME$nl$nl"
mail -s "Hardware check failed after reboot!" \
mail -s "Hardware check $mail_type after reboot!" \
-r "$DAOS_DEVOPS_EMAIL" "$OPERATIONS_EMAIL" \
<<< "$build_info$mail_message"
set -x
Expand Down Expand Up @@ -348,13 +352,15 @@ tf="failures=\"$testfails\""
te="errors=\"0\""
tc="tests=\"$testruns\""

# shellcheck disable=SC2089
junit_xml="<testsuite name=\"$ts\" skipped=\"0\" $tf $te $tc>$nl
junit_xml="<testsuite package=\"$ts\" name=\"$cn\" skipped=\"0\" $tf $te $tc>$nl
$testcases</testsuite>$nl"

# Each junit file needs the same name for when they are collected.
echo "$junit_xml" > "./hardware_prep_node_results.xml"

if [ "$testfails" -gt 0 ]; then
mail_type='failed'
fi
do_mail

if [ "$result" -ne 0 ]; then
Expand Down
Loading