diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index afc08f3e6376..438c01966e0e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5,6 +5,7 @@ env: PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy + push: pull_request: types: - synchronize @@ -23,38 +24,37 @@ on: # yamllint disable-line rule:truthy ##################################### SMALL CHECKS ####################################### ########################################################################################## jobs: - CheckLabels: - runs-on: [self-hosted, style-checker] - # Run the first check always, even if the CI is cancelled - if: ${{ always() }} - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Labels check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 run_check.py - PythonUnitTests: - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Python unit tests - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - echo "Testing the main ci directory" - python3 -m unittest discover -s . -p '*_test.py' - for dir in *_lambda/; do - echo "Testing $dir" - python3 -m unittest discover -s "$dir" -p '*_test.py' - done + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done DockerHubPushAarch64: - needs: CheckLabels - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -70,7 +70,6 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -87,7 +86,7 @@ jobs: name: changed_images_amd64 path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests] + needs: [DockerHubPushAmd64, DockerHubPushAarch64] runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -113,4814 +112,4814 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - StyleCheck: - needs: DockerHubPush - runs-on: [self-hosted, style-checker] - # We need additional `&& ! cancelled()` to have the job being able to cancel - if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{ runner.temp }}/style_check - ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/fasttest - REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.TEMP_PATH }} - - name: Fast Test - run: | - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - CompatibilityCheckX86: - needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/compatibility_check - REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheckX86 - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - CompatibilityCheckAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/compatibility_check - REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheckAarch64 - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -######################################################################################### -#################################### ORDINARY BUILDS #################################### -######################################################################################### - BuilderDebRelease: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - BuilderBinRelease: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_asan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebUBsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_ubsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebTsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_tsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebMsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_msan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebDebug: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_debug - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Apply sparse checkout for contrib # in order to check that it doesn't break build - run: | - rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' - git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' - "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' - du -hs "$GITHUB_WORKSPACE/contrib" ||: - find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -########################################################################################## -##################################### SPECIAL BUILDS ##################################### -########################################################################################## - BuilderBinClangTidy: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_tidy - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwin: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinFreeBSD: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_freebsd - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinPPC64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_ppc64le - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAmd64Compat: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_amd64_compat - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAarch64V80Compat: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_aarch64_v80compat - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -############################################################################################ -##################################### Docker images ####################################### -############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-server --image-path docker/server - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################ -##################################### BUILD REPORTER ####################################### -############################################################################################ - BuilderReport: - needs: - - BuilderBinRelease - - BuilderDebAarch64 - - BuilderDebAsan - - BuilderDebDebug - - BuilderDebMsan - - BuilderDebRelease - - BuilderDebTsan - - BuilderDebUBsan - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - CHECK_NAME=ClickHouse build check - REPORTS_PATH=${{runner.temp}}/reports_dir - TEMP_PATH=${{runner.temp}}/report_check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - BuilderSpecialReport: - needs: - - BuilderBinAarch64 - - BuilderBinDarwin - - BuilderBinDarwinAarch64 - - BuilderBinFreeBSD - - BuilderBinPPC64 - - BuilderBinAmd64Compat - - BuilderBinAarch64V80Compat - - BuilderBinClangTidy - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/report_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=ClickHouse special build check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################ -#################################### INSTALL PACKAGES ###################################### -############################################################################################ - InstallPackagesTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (amd64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - InstallPackagesTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, style-checker-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (arm64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -########################### FUNCTIONAl STATELESS TESTS ####################################### -############################################################################################## - FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseWideParts: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_wide_parts - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, wide parts enabled) - REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseAnalyzer: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_analyzer - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, analyzer) - REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseS3_0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseS3_1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug3: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug4: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug5: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (aarch64) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan3: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan0: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan1: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan0: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan1: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan2: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan3: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan4: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan5: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug3: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug4: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_flaky_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests flaky check (asan) - REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - TestsBugfixCheck: - needs: [CheckLabels, StyleCheck] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/tests_bugfix_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=tests bugfix validate check - KILL_TIMEOUT=3600 - REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Bugfix test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - - TEMP_PATH="${TEMP_PATH}/integration" \ - REPORTS_PATH="${REPORTS_PATH}/integration" \ - python3 integration_test_check.py "Integration $CHECK_NAME" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - TEMP_PATH="${TEMP_PATH}/stateless" \ - REPORTS_PATH="${REPORTS_PATH}/stateless" \ - python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -############################ FUNCTIONAl STATEFUL TESTS ####################################### -############################################################################################## - FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (aarch64) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - # Parallel replicas - FunctionalStatefulTestDebugParallelReplicas: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsanParallelReplicas: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsanParallelReplicas: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsanParallelReplicas: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsanParallelReplicas: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestReleaseParallelReplicas: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -######################################### STRESS TESTS ####################################### -############################################################################################## - StressTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (asan) - REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestTsan: - needs: [BuilderDebTsan] - # func testers have 16 cores + 128 GB memory - # while stress testers have 36 cores + 72 memory - # It would be better to have something like 32 + 128, - # but such servers almost unavailable as spot instances. - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (tsan) - REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (msan) - REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_undefined - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (ubsan) - REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (debug) - REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ############################################################################################## - ######################################### UPGRADE CHECK ###################################### - ############################################################################################## - UpgradeCheckAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (asan) - REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckTsan: - needs: [BuilderDebTsan] - # same as for stress test with tsan - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (tsan) - REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (msan) - REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (debug) - REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -##################################### AST FUZZERS ############################################ -############################################################################################## - ASTFuzzerTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (asan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (tsan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestUBSan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (ubsan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestMSan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (msan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (debug) - REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -############################# INTEGRATION TESTS ############################################# -############################################################################################# - IntegrationTestsAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan3: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan4: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan5: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan5: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests flaky check (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### UNIT TESTS ############################################# -############################################################################################# - UnitTestsAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (asan) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsReleaseClang: - needs: [BuilderBinRelease] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (release-clang) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (tsan) - REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (msan) - REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (ubsan) - REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### PERFORMANCE TESTS ###################################### -############################################################################################# - PerformanceComparisonX86-0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-0: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-1: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-2: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-3: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -###################################### SQLANCER FUZZERS ###################################### -############################################################################################## - SQLancerTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqlancer_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=SQLancer (release) - REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: SQLancer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqlancer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - SQLancerTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqlancer_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=SQLancer (debug) - REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: SQLancer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqlancer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -###################################### JEPSEN TESTS ######################################### -############################################################################################# - Jepsen: - # This is special test NOT INCLUDED in FinishCheck - # When it's skipped, all dependent tasks will be skipped too. - # DO NOT add it there - if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') - needs: [BuilderBinRelease] - uses: ./.github/workflows/jepsen.yml - FinishCheck: - needs: - - StyleCheck - - DockerHubPush - - DockerServerImages - - CheckLabels - - BuilderReport - - BuilderSpecialReport - - FastTest - - FunctionalStatelessTestDebug0 - - FunctionalStatelessTestDebug1 - - FunctionalStatelessTestDebug2 - - FunctionalStatelessTestDebug3 - - FunctionalStatelessTestDebug4 - - FunctionalStatelessTestRelease - - FunctionalStatelessTestReleaseDatabaseReplicated0 - - FunctionalStatelessTestReleaseDatabaseReplicated1 - - FunctionalStatelessTestReleaseDatabaseReplicated2 - - FunctionalStatelessTestReleaseDatabaseReplicated3 - - FunctionalStatelessTestReleaseWideParts - - FunctionalStatelessTestReleaseAnalyzer - - FunctionalStatelessTestAarch64 - - FunctionalStatelessTestAsan0 - - FunctionalStatelessTestAsan1 - - FunctionalStatelessTestAsan2 - - FunctionalStatelessTestAsan3 - - FunctionalStatelessTestTsan0 - - FunctionalStatelessTestTsan1 - - FunctionalStatelessTestTsan2 - - FunctionalStatelessTestTsan3 - - FunctionalStatelessTestTsan4 - - FunctionalStatelessTestMsan0 - - FunctionalStatelessTestMsan1 - - FunctionalStatelessTestMsan2 - - FunctionalStatelessTestMsan3 - - FunctionalStatelessTestMsan4 - - FunctionalStatelessTestMsan5 - - FunctionalStatelessTestUBsan0 - - FunctionalStatelessTestUBsan1 - - FunctionalStatefulTestDebug - - FunctionalStatefulTestRelease - - FunctionalStatefulTestAarch64 - - FunctionalStatefulTestAsan - - FunctionalStatefulTestTsan - - FunctionalStatefulTestMsan - - FunctionalStatefulTestUBsan - - FunctionalStatelessTestReleaseS3_0 - - FunctionalStatelessTestReleaseS3_1 - - FunctionalStatelessTestS3Debug0 - - FunctionalStatelessTestS3Debug1 - - FunctionalStatelessTestS3Debug2 - - FunctionalStatelessTestS3Debug4 - - FunctionalStatelessTestS3Debug5 - - FunctionalStatelessTestS3Tsan0 - - FunctionalStatelessTestS3Tsan1 - - FunctionalStatelessTestS3Tsan2 - - FunctionalStatelessTestS3Tsan4 - - StressTestDebug - - StressTestAsan - - StressTestTsan - - StressTestMsan - - StressTestUBsan - - ASTFuzzerTestDebug - - ASTFuzzerTestAsan - - ASTFuzzerTestTsan - - ASTFuzzerTestMSan - - ASTFuzzerTestUBSan - - IntegrationTestsAsan0 - - IntegrationTestsAsan1 - - IntegrationTestsAsan2 - - IntegrationTestsAsan3 - - IntegrationTestsAsan4 - - IntegrationTestsAsan5 - - IntegrationTestsRelease0 - - IntegrationTestsRelease1 - - IntegrationTestsRelease2 - - IntegrationTestsRelease3 - - IntegrationTestsTsan0 - - IntegrationTestsTsan1 - - IntegrationTestsTsan2 - - IntegrationTestsTsan3 - - IntegrationTestsTsan4 - - IntegrationTestsTsan5 - - PerformanceComparisonX86-0 - - PerformanceComparisonX86-1 - - PerformanceComparisonX86-2 - - PerformanceComparisonX86-3 - - PerformanceComparisonAarch-0 - - PerformanceComparisonAarch-1 - - PerformanceComparisonAarch-2 - - PerformanceComparisonAarch-3 - - UnitTestsAsan - - UnitTestsTsan - - UnitTestsMsan - - UnitTestsUBsan - - UnitTestsReleaseClang - - CompatibilityCheckX86 - - CompatibilityCheckAarch64 - - IntegrationTestsFlakyCheck - - SQLancerTestRelease - - SQLancerTestDebug - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Finish label - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 finish_check.py - python3 merge_pr.py --check-approved -############################################################################################## -########################### SQLLOGIC TEST ################################################### -############################################################################################## - SQLLogicTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqllogic_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Sqllogic test (release) - REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Sqllogic test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" +# StyleCheck: +# needs: DockerHubPush +# runs-on: [self-hosted, style-checker] +# # We need additional `&& ! cancelled()` to have the job being able to cancel +# if: ${{ success() || failure() || ( always() && ! cancelled() ) }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{ runner.temp }}/style_check +# ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/fasttest +# REPO_COPY=${{runner.temp}}/fasttest/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.TEMP_PATH }} +# - name: Fast Test +# run: | +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# CompatibilityCheckX86: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckX86 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# CompatibilityCheckAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckAarch64 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ######################################################################################### +# #################################### ORDINARY BUILDS #################################### +# ######################################################################################### +# BuilderDebRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# fetch-depth: 0 # for performance artifact +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ runner.temp }}/images_path +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# fetch-depth: 0 # for performance artifact +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_asan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebUBsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_ubsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebTsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_tsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebMsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_msan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebDebug: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_debug +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build +# run: | +# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' +# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' +# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' +# du -hs "$GITHUB_WORKSPACE/contrib" ||: +# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwin: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ +# DockerServerImages: +# needs: +# - BuilderDebRelease +# - BuilderDebAarch64 +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself +# - name: Check docker clickhouse/clickhouse-server building +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_server.py --release-type head --no-push \ +# --image-repo clickhouse/clickhouse-server --image-path docker/server +# python3 docker_server.py --release-type head --no-push \ +# --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# ##################################### BUILD REPORTER ####################################### +# ############################################################################################ +# BuilderReport: +# needs: +# - BuilderBinRelease +# - BuilderDebAarch64 +# - BuilderDebAsan +# - BuilderDebDebug +# - BuilderDebMsan +# - BuilderDebRelease +# - BuilderDebTsan +# - BuilderDebUBsan +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# CHECK_NAME=ClickHouse build check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# TEMP_PATH=${{runner.temp}}/report_check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# BuilderSpecialReport: +# needs: +# - BuilderBinAarch64 +# - BuilderBinDarwin +# - BuilderBinDarwinAarch64 +# - BuilderBinFreeBSD +# - BuilderBinPPC64 +# - BuilderBinAmd64Compat +# - BuilderBinAarch64V80Compat +# - BuilderBinClangTidy +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/report_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=ClickHouse special build check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# #################################### INSTALL PACKAGES ###################################### +# ############################################################################################ +# InstallPackagesTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (amd64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# InstallPackagesTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (arm64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ########################### FUNCTIONAl STATELESS TESTS ####################################### +# ############################################################################################## +# FunctionalStatelessTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseWideParts: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_wide_parts +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, wide parts enabled) +# REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseAnalyzer: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_analyzer +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, analyzer) +# REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug5: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan0: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan1: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan0: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan1: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan2: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan3: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan4: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan5: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_flaky_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# TestsBugfixCheck: +# needs: [CheckLabels, StyleCheck] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/tests_bugfix_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=tests bugfix validate check +# KILL_TIMEOUT=3600 +# REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Bugfix test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" + +# TEMP_PATH="${TEMP_PATH}/integration" \ +# REPORTS_PATH="${REPORTS_PATH}/integration" \ +# python3 integration_test_check.py "Integration $CHECK_NAME" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# TEMP_PATH="${TEMP_PATH}/stateless" \ +# REPORTS_PATH="${REPORTS_PATH}/stateless" \ +# python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ############################ FUNCTIONAl STATEFUL TESTS ####################################### +# ############################################################################################## +# FunctionalStatefulTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# # Parallel replicas +# FunctionalStatefulTestDebugParallelReplicas: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsanParallelReplicas: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsanParallelReplicas: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsanParallelReplicas: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsanParallelReplicas: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestReleaseParallelReplicas: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### STRESS TESTS ####################################### +# ############################################################################################## +# StressTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (asan) +# REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestTsan: +# needs: [BuilderDebTsan] +# # func testers have 16 cores + 128 GB memory +# # while stress testers have 36 cores + 72 memory +# # It would be better to have something like 32 + 128, +# # but such servers almost unavailable as spot instances. +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (tsan) +# REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (msan) +# REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_undefined +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (ubsan) +# REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (debug) +# REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### UPGRADE CHECK ###################################### +# ############################################################################################## +# UpgradeCheckAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (asan) +# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckTsan: +# needs: [BuilderDebTsan] +# # same as for stress test with tsan +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (tsan) +# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (msan) +# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (debug) +# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ##################################### AST FUZZERS ############################################ +# ############################################################################################## +# ASTFuzzerTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (asan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (tsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestUBSan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (ubsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestMSan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (msan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (debug) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ############################# INTEGRATION TESTS ############################################# +# ############################################################################################# +# IntegrationTestsAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan4: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan5: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan5: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### UNIT TESTS ############################################# +# ############################################################################################# +# UnitTestsAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (asan) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsReleaseClang: +# needs: [BuilderBinRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (release-clang) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (tsan) +# REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (msan) +# REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (ubsan) +# REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### PERFORMANCE TESTS ###################################### +# ############################################################################################# +# PerformanceComparisonX86-0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-0: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-1: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-2: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-3: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ###################################### SQLANCER FUZZERS ###################################### +# ############################################################################################## +# SQLancerTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (release) +# REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# SQLancerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (debug) +# REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ###################################### JEPSEN TESTS ######################################### +# ############################################################################################# +# Jepsen: +# # This is special test NOT INCLUDED in FinishCheck +# # When it's skipped, all dependent tasks will be skipped too. +# # DO NOT add it there +# if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') +# needs: [BuilderBinRelease] +# uses: ./.github/workflows/jepsen.yml +# FinishCheck: +# needs: +# - StyleCheck +# - DockerHubPush +# - DockerServerImages +# - CheckLabels +# - BuilderReport +# - BuilderSpecialReport +# - FastTest +# - FunctionalStatelessTestDebug0 +# - FunctionalStatelessTestDebug1 +# - FunctionalStatelessTestDebug2 +# - FunctionalStatelessTestDebug3 +# - FunctionalStatelessTestDebug4 +# - FunctionalStatelessTestRelease +# - FunctionalStatelessTestReleaseDatabaseReplicated0 +# - FunctionalStatelessTestReleaseDatabaseReplicated1 +# - FunctionalStatelessTestReleaseDatabaseReplicated2 +# - FunctionalStatelessTestReleaseDatabaseReplicated3 +# - FunctionalStatelessTestReleaseWideParts +# - FunctionalStatelessTestReleaseAnalyzer +# - FunctionalStatelessTestAarch64 +# - FunctionalStatelessTestAsan0 +# - FunctionalStatelessTestAsan1 +# - FunctionalStatelessTestAsan2 +# - FunctionalStatelessTestAsan3 +# - FunctionalStatelessTestTsan0 +# - FunctionalStatelessTestTsan1 +# - FunctionalStatelessTestTsan2 +# - FunctionalStatelessTestTsan3 +# - FunctionalStatelessTestTsan4 +# - FunctionalStatelessTestMsan0 +# - FunctionalStatelessTestMsan1 +# - FunctionalStatelessTestMsan2 +# - FunctionalStatelessTestMsan3 +# - FunctionalStatelessTestMsan4 +# - FunctionalStatelessTestMsan5 +# - FunctionalStatelessTestUBsan0 +# - FunctionalStatelessTestUBsan1 +# - FunctionalStatefulTestDebug +# - FunctionalStatefulTestRelease +# - FunctionalStatefulTestAarch64 +# - FunctionalStatefulTestAsan +# - FunctionalStatefulTestTsan +# - FunctionalStatefulTestMsan +# - FunctionalStatefulTestUBsan +# - FunctionalStatelessTestReleaseS3_0 +# - FunctionalStatelessTestReleaseS3_1 +# - FunctionalStatelessTestS3Debug0 +# - FunctionalStatelessTestS3Debug1 +# - FunctionalStatelessTestS3Debug2 +# - FunctionalStatelessTestS3Debug4 +# - FunctionalStatelessTestS3Debug5 +# - FunctionalStatelessTestS3Tsan0 +# - FunctionalStatelessTestS3Tsan1 +# - FunctionalStatelessTestS3Tsan2 +# - FunctionalStatelessTestS3Tsan4 +# - StressTestDebug +# - StressTestAsan +# - StressTestTsan +# - StressTestMsan +# - StressTestUBsan +# - ASTFuzzerTestDebug +# - ASTFuzzerTestAsan +# - ASTFuzzerTestTsan +# - ASTFuzzerTestMSan +# - ASTFuzzerTestUBSan +# - IntegrationTestsAsan0 +# - IntegrationTestsAsan1 +# - IntegrationTestsAsan2 +# - IntegrationTestsAsan3 +# - IntegrationTestsAsan4 +# - IntegrationTestsAsan5 +# - IntegrationTestsRelease0 +# - IntegrationTestsRelease1 +# - IntegrationTestsRelease2 +# - IntegrationTestsRelease3 +# - IntegrationTestsTsan0 +# - IntegrationTestsTsan1 +# - IntegrationTestsTsan2 +# - IntegrationTestsTsan3 +# - IntegrationTestsTsan4 +# - IntegrationTestsTsan5 +# - PerformanceComparisonX86-0 +# - PerformanceComparisonX86-1 +# - PerformanceComparisonX86-2 +# - PerformanceComparisonX86-3 +# - PerformanceComparisonAarch-0 +# - PerformanceComparisonAarch-1 +# - PerformanceComparisonAarch-2 +# - PerformanceComparisonAarch-3 +# - UnitTestsAsan +# - UnitTestsTsan +# - UnitTestsMsan +# - UnitTestsUBsan +# - UnitTestsReleaseClang +# - CompatibilityCheckX86 +# - CompatibilityCheckAarch64 +# - IntegrationTestsFlakyCheck +# - SQLancerTestRelease +# - SQLancerTestDebug +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Finish label +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 finish_check.py +# python3 merge_pr.py --check-approved +# ############################################################################################## +# ########################### SQLLOGIC TEST ################################################### +# ############################################################################################## +# SQLLogicTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqllogic_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Sqllogic test (release) +# REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Sqllogic test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index dd21c8552d3b..ae8309953905 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,6 +1,7 @@ # docker build -t clickhouse/binary-builder . ARG FROM_TAG=latest -FROM clickhouse/test-util:latest AS cctools +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:latest AS cctools # The cctools are built always from the clickhouse/test-util:latest and cached inline # Theoretically, it should improve rebuild speed significantly ENV CC=clang-${LLVM_VERSION} @@ -39,7 +40,7 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ # END COMPILE # !!!!!!!!!!! -FROM clickhouse/test-util:$FROM_TAG +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index f68368044541..1ad6f7ab8c07 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/test-base . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 8136fd1fbbcd..e6676c740fd3 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -2,7 +2,8 @@ # docker build --network=host -t clickhouse/codebrowser . # docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser ARG FROM_TAG=latest -FROM clickhouse/binary-builder:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/binary-builder:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index ffb13fc774d4..98d58fa9767c 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/fasttest . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index aa71074c02a7..7401edc4eefe 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/fuzzer . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index de8efa20af4e..6f442be44666 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/integration-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG SHELL ["/bin/bash", "-c"] diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile index a794e076ec02..b0425847048d 100644 --- a/docker/test/keeper-jepsen/Dockerfile +++ b/docker/test/keeper-jepsen/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/keeper-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/server-jepsen/Dockerfile b/docker/test/server-jepsen/Dockerfile index a212427b2a1a..45c28bc6b036 100644 --- a/docker/test/server-jepsen/Dockerfile +++ b/docker/test/server-jepsen/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/server-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/sqllogic/Dockerfile b/docker/test/sqllogic/Dockerfile index 83dcf7e1f56a..f30e3e0b86ed 100644 --- a/docker/test/sqllogic/Dockerfile +++ b/docker/test/sqllogic/Dockerfile @@ -1,6 +1,7 @@ # docker build -t clickhouse/sqllogic-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG RUN apt-get update --yes \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 71a2e92e3a86..d69662997915 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #47031 # docker build -t clickhouse/stateful-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 40109255a7e5..42d7b1053a3a 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/stateless-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index e9712f430fd2..7e4e17da2a68 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/stress-test . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 746cc7bb2d5b..20fd34e57693 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -18,7 +18,12 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ shellcheck \ yamllint \ - && pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ + && if [ "$DOCKER_REPO" = "docker.io" ]; then \ + pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2; \ + else \ + # the `hvac` is an optional dependency for Hashicorp Vault as a parameters store + pip3 install black==23.1.0 boto3 hvac codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2; \ + fi \ && apt-get clean \ && rm -rf /root/.cache/pip diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index b75bfb6661cc..d389e0d3f341 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/unit-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get install gdb diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile index 8e5890b81a0c..86055ff312a9 100644 --- a/docker/test/upgrade/Dockerfile +++ b/docker/test/upgrade/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/upgrade-check . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 64b64896f661..c30d0900cc16 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -7,6 +7,7 @@ import requests # type: ignore from get_robot_token import get_parameter_from_ssm +from env_helper import GITHUB_REPOSITORY from pr_info import PRInfo from report import TestResults @@ -141,7 +142,7 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" + pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/master" base_ref = "master" head_ref = "master" base_repo = pr_info.repo_full_name diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 16a58a90dcf6..1b794f5b0caa 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -15,7 +15,13 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL +from env_helper import ( + GITHUB_WORKSPACE, + RUNNER_TEMP, + GITHUB_RUN_URL, + DOCKER_USER, + DOCKER_REPO, +) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult @@ -94,7 +100,7 @@ def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict: def get_changed_docker_images( - pr_info: PRInfo, images_dict: ImagesDict + pr_info: PRInfo, images_dict: ImagesDict, DOCKER_REPO: str ) -> Set[DockerImage]: if not images_dict: return set() @@ -113,7 +119,7 @@ def get_changed_docker_images( for dockerfile_dir, image_description in images_dict.items(): for f in files_changed: if f.startswith(dockerfile_dir): - name = image_description["name"] + name = f"{DOCKER_REPO}/{image_description['name']}" only_amd64 = image_description.get("only_amd64", False) logging.info( "Found changed file '%s' which affects " @@ -137,7 +143,7 @@ def get_changed_docker_images( dependent, image, ) - name = images_dict[dependent]["name"] + name = f"{DOCKER_REPO}/{images_dict[dependent]['name']}" only_amd64 = images_dict[dependent].get("only_amd64", False) changed_images.append(DockerImage(dependent, name, only_amd64, image)) index += 1 @@ -248,6 +254,7 @@ def build_and_push_one_image( "docker buildx build --builder default " f"--label build-url={GITHUB_RUN_URL} " f"{from_tag_arg}" + f"--build-arg DOCKER_REPO={DOCKER_REPO} " # A hack to invalidate cache, grep for it in docker/ dir f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " f"--tag {image.repo}:{version_string} " @@ -400,7 +407,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, @@ -424,7 +431,7 @@ def main(): # If the event does not contain diff, nothing will be built pass - changed_images = get_changed_docker_images(pr_info, images_dict) + changed_images = get_changed_docker_images(pr_info, images_dict, DOCKER_REPO) if changed_images: logging.info( "Has changed images: %s", ", ".join([im.path for im in changed_images]) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index d89708b9277e..8c740db61844 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -11,7 +11,7 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import RUNNER_TEMP +from env_helper import RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult @@ -174,7 +174,7 @@ def main(): args = parse_args() if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 9303f9ae2936..8e0e0b3f31a0 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -14,6 +14,8 @@ CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") GITHUB_JOB = os.getenv("GITHUB_JOB", "local") +DOCKER_REPO = os.getenv("DOCKER_REPO", "docker.io") +DOCKER_USER = os.getenv("DOCKER_USER", "robotclickhouse") GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse") GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0") GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") @@ -23,6 +25,7 @@ REPORTS_PATH = os.getenv("REPORTS_PATH", p.abspath(p.join(module_dir, "./reports"))) REPO_COPY = os.getenv("REPO_COPY", git_root) RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) +S3_REGION = os.getenv("S3_REGION", "us-east-1") S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") @@ -31,6 +34,10 @@ f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/" "{pr_or_release}/{commit}/{build_name}/{artifact}" ) +VAULT_PATH = os.getenv("VAULT_PATH") +VAULT_TOKEN = os.getenv("VAULT_TOKEN") +VAULT_URL = os.getenv("VAULT_URL") +VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index b41eba49cc32..1136d6152006 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -7,6 +7,8 @@ from github import Github from github.AuthenticatedUser import AuthenticatedUser +from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT, S3_REGION + @dataclass class Token: @@ -16,9 +18,20 @@ class Token: def get_parameter_from_ssm(name, decrypt=True, client=None): - if not client: - client = boto3.client("ssm", region_name="us-east-1") - return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + if VAULT_URL: + import hvac + if not client: + client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) + parameter = client.secrets.kv.v2.read_secret_version( + mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH + )["data"]["data"][name] + else: + if not client: + client = boto3.client("ssm", region_name=S3_REGION) + parameter = client.get_parameter(Name=name, WithDecryption=decrypt)[ + "Parameter" + ]["Value"] + return parameter ROBOT_TOKEN = None # type: Optional[Token] @@ -28,18 +41,49 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): global ROBOT_TOKEN if ROBOT_TOKEN is not None: return ROBOT_TOKEN.value - client = boto3.client("ssm", region_name="us-east-1") - parameters = client.describe_parameters( - ParameterFilters=[ - {"Key": "Name", "Option": "BeginsWith", "Values": [token_prefix_env_name]} - ] - )["Parameters"] - assert parameters - - for token_name in [p["Name"] for p in parameters]: - value = get_parameter_from_ssm(token_name, True, client) + + def get_vault_robot_tokens(): + import hvac + client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) + parameters = client.secrets.kv.v2.read_secret_version( + mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH + )["data"]["data"] + parameters = { + key: value + for key, value in parameters.items() + if key.startswith(token_prefix_env_name) + } + assert parameters + return list(parameters.values()) + + def get_ssm_robot_tokens(): + client = boto3.client("ssm", region_name=S3_REGION) + parameters = client.describe_parameters( + ParameterFilters=[ + { + "Key": "Name", + "Option": "BeginsWith", + "Values": [token_prefix_env_name], + } + ] + )["Parameters"] + assert parameters + for token_name in [p["Name"] for p in parameters]: + value = get_parameter_from_ssm(token_name, True, client) + values.append(value) + return values + + client = None + values = [] + + if VAULT_URL: + values = get_vault_robot_tokens() + else: + values = get_ssm_robot_tokens() + + for value in values: gh = Github(value, per_page=100) - # Do not spend additional request to API by accessin user.login unless + # Do not spend additional request to API by accessing user.login unless # the token is chosen by the remaining requests number user = gh.get_user() rest, _ = gh.rate_limiting diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index c21fafa26055..13808c736165 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -15,7 +15,7 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import RerunHelper, get_commit, post_commit_status from compress_files import compress_fast -from env_helper import REPO_COPY, TEMP_PATH, S3_BUILDS_BUCKET, S3_DOWNLOAD +from env_helper import REPO_COPY, TEMP_PATH, S3_BUILDS_BUCKET, S3_DOWNLOAD, S3_REGION from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult @@ -91,7 +91,7 @@ def get_instances_addresses(ec2_client, instance_ids): def prepare_autoscaling_group_and_get_hostnames(count): - asg_client = boto3.client("autoscaling", region_name="us-east-1") + asg_client = boto3.client("autoscaling", region_name=S3_REGION) asg_client.set_desired_capacity( AutoScalingGroupName=JEPSEN_GROUP_NAME, DesiredCapacity=count ) @@ -105,12 +105,12 @@ def prepare_autoscaling_group_and_get_hostnames(count): if counter > 30: raise Exception("Cannot wait autoscaling group") - ec2_client = boto3.client("ec2", region_name="us-east-1") + ec2_client = boto3.client("ec2", region_name=S3_REGION) return get_instances_addresses(ec2_client, instances) def clear_autoscaling_group(): - asg_client = boto3.client("autoscaling", region_name="us-east-1") + asg_client = boto3.client("autoscaling", region_name=S3_REGION) asg_client.set_desired_capacity( AutoScalingGroupName=JEPSEN_GROUP_NAME, DesiredCapacity=0 ) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 9ee0a4312947..441ebf006f08 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -15,6 +15,7 @@ from env_helper import ( S3_TEST_REPORTS_BUCKET, S3_BUILDS_BUCKET, + S3_REGION, RUNNER_TEMP, CI, S3_URL, @@ -47,7 +48,7 @@ class S3Helper: def __init__(self): config = botocore.config.Config(max_pool_connections=self.max_pool_size) - self.session = boto3.session.Session(region_name="us-east-1") + self.session = boto3.session.Session(region_name=S3_REGION) self.client = self.session.client("s3", endpoint_url=S3_URL, config=config) self.host = S3_URL self.download_host = S3_DOWNLOAD