From f6869aa54173b087f3e1e0a21175b358bde7916b Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Fri, 5 Jan 2024 08:31:15 +0100 Subject: [PATCH] ci,ascent: remove ascent build --- .gitlab-ci.yml | 7 -- .gitlab/ci/ascent.yml | 102 ------------------ .gitlab/ci/config/ecpci-amend-commit.sh | 26 ----- .gitlab/ci/config/ecpci-fetch-commit-trace.py | 56 ---------- .gitlab/ci/config/ecpci-push-branch.sh | 15 --- .gitlab/ci/config/ecpci-remove-branch.sh | 15 --- .gitlab/ci/config/ecpci-wait-commit-status.sh | 80 -------------- .gitlab/ci/config/initial_config.cmake | 4 - .gitlab/ci/ubuntu2004.yml | 26 ----- 9 files changed, 331 deletions(-) delete mode 100644 .gitlab/ci/ascent.yml delete mode 100755 .gitlab/ci/config/ecpci-amend-commit.sh delete mode 100755 .gitlab/ci/config/ecpci-fetch-commit-trace.py delete mode 100755 .gitlab/ci/config/ecpci-push-branch.sh delete mode 100755 .gitlab/ci/config/ecpci-remove-branch.sh delete mode 100755 .gitlab/ci/config/ecpci-wait-commit-status.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8be2f709d..abb097888 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -107,12 +107,6 @@ when: on_success - when: never -.run_ascent_ci: &run_ascent_ci - rules: - - if: '$CI_PROJECT_PATH == "ecpcitest/vtk-m"' - when: on_success - - when: never - .run_crusher_ci: &run_crusher_ci rules: - if: '$CI_PROJECT_PATH == "ci/csc331_crusher/dev/vtk-m"' @@ -237,7 +231,6 @@ stages: include: - - local: '/.gitlab/ci/ascent.yml' - local: '/.gitlab/ci/crusher.yml' - local: '/.gitlab/ci/centos7.yml' - local: '/.gitlab/ci/centos8.yml' diff --git a/.gitlab/ci/ascent.yml b/.gitlab/ci/ascent.yml deleted file mode 100644 index dad36a11d..000000000 --- a/.gitlab/ci/ascent.yml +++ /dev/null @@ -1,102 +0,0 @@ -##============================================================================= -## -## Copyright (c) Kitware, Inc. -## All rights reserved. -## See LICENSE.txt for details. -## -## This software is distributed WITHOUT ANY WARRANTY; without even -## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE. See the above copyright notice for more information. -## -##============================================================================= - -# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Ascent. -.ascent_gcc_cuda: - variables: - CCACHE_BASEDIR: "/gpfs/wolf/" - CCACHE_DIR: "/gpfs/wolf/csc331/scratch/vbolea/ci/ccache" - # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it - CCACHE_IGNOREOPTIONS: "-isystem=*" - CCACHE_NOHASHDIR: "true" - - CC: "gcc" - CXX: "g++" - CMAKE_BUILD_TYPE: "RelWithDebInfo" - CMAKE_GENERATOR: "Ninja" - CUDAHOSTCXX: "g++" - CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc331/scratch/vbolea/ci/vtk-m" - - VTKM_SETTINGS: cuda+ascent+ccache - JOB_MODULES: >- - git-lfs - zstd - cuda/11.4.2 - gcc/10.2.0 - ninja - spectrum-mpi - lsf-tools - cmake - interruptible: true - -.setup_env_ecpci: &setup_env_ecpci | - module purge - module load ${JOB_MODULES} - module list - export PATH="/gpfs/wolf/csc331/scratch/vbolea/ci/utils:$PATH" - -build:ascent_gcc_cuda: - stage: build - tags: [olcf, ascent, batch] - extends: - - .ascent_gcc_cuda - - .run_ascent_ci - - .cmake_build_artifacts - variables: - SCHEDULER_PARAMETERS: -P CSC331 -W 2:00 -nnodes 1 -alloc_flags smt1 - timeout: 125 minutes - before_script: - - *setup_env_ecpci - - ccache -z - - git remote add lfs https://gitlab.kitware.com/vtk/vtk-m.git - - git fetch lfs - - git-lfs install - - git-lfs pull lfs - script: - # Each Ascent (Summit) node has 172 threads (43 cores). SMT1 is needed to - # avoid L1 cache pollution among different processes. Thus, using 40 cores - # seems a reasonable choice which leaves a couple of cores for system processes. - - CTEST_MAX_PARALLELISM=40 cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake - - ctest -VV -S .gitlab/ci/ctest_configure.cmake - - GITLAB_CI_EMULATION=1 jsrun -n1 -a1 -g1 -c40 -bpacked:40 ctest -VV -S .gitlab/ci/ctest_build.cmake - after_script: - - *setup_env_ecpci - - ccache -s - - ctest -VV -S .gitlab/ci/ctest_submit_build.cmake - -test:ascent_gcc_cuda: - stage: test - tags: [olcf, ascent, batch] - extends: - - .ascent_gcc_cuda - - .run_ascent_ci - - .cmake_test_artifacts - needs: - - build:ascent_gcc_cuda - dependencies: - - build:ascent_gcc_cuda - variables: - # For tests we want to use a small number of proccesses, for some reason - # a higher parallelism number tend to results in test malfunctions. - CTEST_MAX_PARALLELISM: 4 - # We need this to skip ctest_submit from being run inside a jsrun job - GITLAB_CI_EMULATION: 1 - SCHEDULER_PARAMETERS: -P CSC331 -W 1:00 -nnodes 1 -alloc_flags gpudefault - timeout: 65 minutes - before_script: - - *setup_env_ecpci - script: - - cmake -V -P .gitlab/ci/config/gitlab_ci_setup.cmake - - jsrun -n1 -a1 -g1 -c1 ctest -VV -S .gitlab/ci/ctest_test.cmake - after_script: - - *setup_env_ecpci - - ctest -VV -S .gitlab/ci/ctest_submit_test.cmake diff --git a/.gitlab/ci/config/ecpci-amend-commit.sh b/.gitlab/ci/config/ecpci-amend-commit.sh deleted file mode 100755 index 4c52c50ff..000000000 --- a/.gitlab/ci/config/ecpci-amend-commit.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -e -# shellcheck disable=SC2155 - -##============================================================================= -## -## Copyright (c) Kitware, Inc. -## All rights reserved. -## See LICENSE.txt for details. -## -## This software is distributed WITHOUT ANY WARRANTY; without even -## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE. See the above copyright notice for more information. -## -##============================================================================= - -git rev-parse @ > ./ORIGINAL_COMMIT_SHA -git add ./ORIGINAL_COMMIT_SHA - -readonly name="$(git show --quiet --format='%cn')" -readonly email="$(git show --quiet --format='%ce')" - -git config --global user.name "$name" -git config --global user.email "$email" -git commit --amend --no-edit -a - -git rev-parse @ > "$1" diff --git a/.gitlab/ci/config/ecpci-fetch-commit-trace.py b/.gitlab/ci/config/ecpci-fetch-commit-trace.py deleted file mode 100755 index a0605cad3..000000000 --- a/.gitlab/ci/config/ecpci-fetch-commit-trace.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 - -##============================================================================= -## -## Copyright (c) Kitware, Inc. -## All rights reserved. -## See LICENSE.txt for details. -## -## This software is distributed WITHOUT ANY WARRANTY; without even -## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE. See the above copyright notice for more information. -## -##============================================================================= - -import json -import ssl -import sys -import urllib.request - - -class ecpci_url_reader: - def __init__(self, base_url, token): - self.base_url = base_url - self.token = token - - def to_string(self, url): - opener = urllib.request.build_opener( - urllib.request.HTTPSHandler( - context=ssl._create_unverified_context())) - opener.addheaders = [('PRIVATE-TOKEN', token)] - return opener.open(base_url + url).read().decode('utf-8') - - def to_json(self, url): - return json.loads(self.to_string(url)) - - -base_url = sys.argv[1] -commit = sys.argv[2] -token = sys.argv[3] - -handler = ecpci_url_reader(base_url, token) - -commit_info = handler.to_json("/repository/commits/" + commit) -last_pipeline_id = str(commit_info['last_pipeline']['id']) - -jobs = handler.to_json("/pipelines/" + last_pipeline_id + "/jobs") -build_job_id = str(jobs[1]['id']) -test_job_id = str(jobs[0]['id']) - -print("ECPCITEST CONFIGURE OUTPUT============================================") -print(handler.to_string("/jobs/" + build_job_id + "/trace")) -print("ECPCITEST CONFIGURE END===============================================") - -print("ECPCITEST TEST OUTPUT=================================================") -print(handler.to_string("/jobs/" + test_job_id + "/trace")) -print("ECPCITEST TEST OUTPUT END=============================================") diff --git a/.gitlab/ci/config/ecpci-push-branch.sh b/.gitlab/ci/config/ecpci-push-branch.sh deleted file mode 100755 index ae38b0bfc..000000000 --- a/.gitlab/ci/config/ecpci-push-branch.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -ex - -##============================================================================= -## -## Copyright (c) Kitware, Inc. -## All rights reserved. -## See LICENSE.txt for details. -## -## This software is distributed WITHOUT ANY WARRANTY; without even -## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE. See the above copyright notice for more information. -## -##============================================================================= - -git -c http.sslVerify=false push --no-verify -f "$1" "HEAD:refs/heads/${2}" diff --git a/.gitlab/ci/config/ecpci-remove-branch.sh b/.gitlab/ci/config/ecpci-remove-branch.sh deleted file mode 100755 index 4c208bc97..000000000 --- a/.gitlab/ci/config/ecpci-remove-branch.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -ex - -##============================================================================= -## -## Copyright (c) Kitware, Inc. -## All rights reserved. -## See LICENSE.txt for details. -## -## This software is distributed WITHOUT ANY WARRANTY; without even -## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE. See the above copyright notice for more information. -## -##============================================================================= - -git -c http.sslVerify=false push --delete "$1" "$2" diff --git a/.gitlab/ci/config/ecpci-wait-commit-status.sh b/.gitlab/ci/config/ecpci-wait-commit-status.sh deleted file mode 100755 index c774b2393..000000000 --- a/.gitlab/ci/config/ecpci-wait-commit-status.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -e -# shellcheck disable=SC2155 - -##============================================================================= -## -## Copyright (c) Kitware, Inc. -## All rights reserved. -## See LICENSE.txt for details. -## -## This software is distributed WITHOUT ANY WARRANTY; without even -## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE. See the above copyright notice for more information. -## -##============================================================================= - -declare -r POLL_INTERVAL_SECONDS="10" - -function fetch_commit_status() -{ - local -r url="$1" - local -r commit="$2" - - local output=$(curl --insecure --silent "${url}/repository/commits/${commit}" \ - | tr ',{}[]' '\n' \ - | grep -Po -m1 '(?<=^"status":")\w+(?=")' \ - ) - - # No status means that the pipeline has not being created yet - [ -z "$output" ] && output="empty" - echo "$output" -} - -function print_pipeline_url() -{ - local -r url="$1" - local -r commit="$2" - - local web_url=$(curl --insecure --silent "${url}" \ - | tr ',{}[]' '\n' \ - | grep -Po -m1 '(?<=^"web_url":").+(?=")' \ - ) - - local pipeline_id=$(curl --insecure --silent "${url}/repository/commits/${commit}" \ - | tr ',{}[]' '\n' \ - | grep -Po '(?<=^"id":)\d+$' \ - ) - - echo "######################################################################" - echo "ECP Pipeline: ${web_url}/-/pipelines/$pipeline_id" - echo "######################################################################" -} - -function wait_commit_pipeline_status() -{ - local -r base_url="$1" - local -r commit="$2" - local is_url_printed="no" - - while true - do - local ret="$(fetch_commit_status "$base_url" "$commit")" - - if [ "$ret" != "empty" ] && [ "$is_url_printed" == "no" ] - then - print_pipeline_url "$base_url" "$commit" - is_url_printed="yes" - fi - - case "$ret" in - success) - return 0 ;; - failed|canceled|skipped) - echo "ERROR: The pipeline exited with \`${ret}\` status" > /dev/stderr - return 1 ;; - esac - sleep "$POLL_INTERVAL_SECONDS" - done -} - -wait_commit_pipeline_status "$1" "$2" diff --git a/.gitlab/ci/config/initial_config.cmake b/.gitlab/ci/config/initial_config.cmake index ab82d6835..0ec3919e9 100644 --- a/.gitlab/ci/config/initial_config.cmake +++ b/.gitlab/ci/config/initial_config.cmake @@ -116,10 +116,6 @@ foreach(option IN LISTS options) set(CMAKE_HIP_FLAGS "-O0 " CACHE STRING "") set(CMAKE_HIP_FLAGS_RELWITHDEBINFO "-g" CACHE STRING "") - elseif(ascent STREQUAL option) - set(CMAKE_C_FLAGS "-mcpu=power9" CACHE STRING "") - set(CMAKE_CXX_FLAGS "-mcpu=power9" CACHE STRING "") - elseif(ccache STREQUAL option) find_program(CCACHE_COMMAND NAMES ccache REQUIRED) diff --git a/.gitlab/ci/ubuntu2004.yml b/.gitlab/ci/ubuntu2004.yml index c6ff59fe0..b09526b51 100644 --- a/.gitlab/ci/ubuntu2004.yml +++ b/.gitlab/ci/ubuntu2004.yml @@ -87,29 +87,3 @@ test:ubuntu2004_hip_kokkos: needs: - build:ubuntu2004_hip_kokkos timeout: 3 hours - -# This is only for merge-requests -build:ascent: - stage: build - variables: - BRANCH_NAME: "mr${CI_MERGE_REQUEST_IID}-${CI_COMMIT_REF_NAME}" - ASCENT_REST_URL: "https://code.ornl.gov/api/v4/projects/7035" - ASCENT_GIT_URL: "https://vbolea:${ECPTEST_TOKEN}@code.ornl.gov/ecpcitest/vtk-m.git" - tags: - - vtkm - - docker - - build - - linux-x86_64 - extends: - - .ubuntu2004 - - .run_only_merge_requests - script: - # A new commit is needed to symbolize a restart of a build from the same original commit - - .gitlab/ci/config/ecpci-amend-commit.sh "NEW_COMMIT_SHA" - - .gitlab/ci/config/ecpci-push-branch.sh "$ASCENT_GIT_URL" "$BRANCH_NAME" - - timeout 130m .gitlab/ci/config/ecpci-wait-commit-status.sh "$ASCENT_REST_URL" "$(cat NEW_COMMIT_SHA)" - after_script: - - .gitlab/ci/config/ecpci-remove-branch.sh "$ASCENT_GIT_URL" "$BRANCH_NAME" || true - - .gitlab/ci/config/ecpci-fetch-commit-trace.py "$ASCENT_REST_URL" "$(cat NEW_COMMIT_SHA)" "$ECPTEST_TOKEN" - timeout: 150 minutes - interruptible: true