From db5e43f574cebc32f834e098d92223a095e1925e Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Fri, 24 Jan 2025 10:16:36 +0000 Subject: [PATCH 1/8] Publish RPMs for azure linux --- .github/workflows/release-test.yml | 366 ++++++++++++++++++ .github/workflows/release.yml | 122 ++++-- CMakeLists.txt | 6 +- .../{cpack_settings.cmake => cpack_deb.cmake} | 0 cmake/cpack_rpm.cmake | 31 ++ tests/test_install_build.sh | 14 +- 6 files changed, 509 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/release-test.yml rename cmake/{cpack_settings.cmake => cpack_deb.cmake} (100%) create mode 100644 cmake/cpack_rpm.cmake diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml new file mode 100644 index 000000000000..7edfa556817b --- /dev/null +++ b/.github/workflows/release-test.yml @@ -0,0 +1,366 @@ +name: Release (test) + +on: + pull_request: + types: + - labeled + - synchronize + - opened + - reopened + workflow_dispatch: + +permissions: + contents: write + actions: read + checks: write + +jobs: + make_sbom: + name: SBOM Generation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: "Install SBOM tool" + run: | + set -ex + curl -Lo sbom-tool https://github.com/microsoft/sbom-tool/releases/latest/download/sbom-tool-linux-x64 > sbom-tool + chmod +x sbom-tool + shell: bash + - name: "Produce SBOM" + run: | + set -ex + CCF_VERSION=${{ github.ref_name }} + CCF_VERSION=${CCF_VERSION#ccf-} + ./sbom-tool generate -b . -bc . -pn CCF -ps Microsoft -nsb https://sbom.microsoft -pv $CCF_VERSION -V Error + shell: bash + - name: "Upload SBOM" + uses: actions/upload-artifact@v4 + with: + name: sbom + path: _manifest/spdx_2.2/* + + # release_notes: + # name: Release Notes + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 1 + # - name: "Check Release Notes" + # run: | + # set -ex + # python scripts/extract-release-notes.py --target-git-version + # shell: bash + # - name: "Produce Release Notes" + # run: | + # set -ex + # set -o pipefail + # python ./scripts/extract-release-notes.py --target-git-version --describe-path-changes "./samples/constitution" | tee rel-notes.md + # - name: "Upload .deb Package" + # uses: actions/upload-artifact@v4 + # with: + # name: relnotes + # path: rel-notes.md + + build_release: + name: Build Release + # needs: release_notes + if: ${{ contains(github.event.pull_request.labels.*.name, 'release-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} + strategy: + matrix: + platform: + - name: virtual + os: ubuntu + image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 + package_version: DEB + test_filter: "benchmark|unit" + - name: snp + os: ubuntu + image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 + package_version: DEB + - name: virtual + os: azure-linux + image: mcr.microsoft.com/azurelinux/base/core:3.0 + package_version: RPM + test_filter: "benchmark|unit|protocolstest|lts" + - name: snp + os: azure-linux + image: mcr.microsoft.com/azurelinux/base/core:3.0 + package_version: RPM + + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ${{ matrix.platform.image }} + options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro" + + steps: + - name: "Install dependencies" + if: ${{ matrix.platform.os == 'azure-linux' }} + shell: bash + run: | + set -ex + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + # Source control + tdnf -y install ca-certificates git + # Build tools + tdnf -y install build-essential clang cmake ninja-build which + # libc++ + tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static + # Dependencies + tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel + # Test dependencies + tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect + # Install CDDL via rubygems + tdnf -y install rubygems + gem install cddl + # For packaging and release tests + tdnf -y install rpm-build procps + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Build Release ${{ matrix.platform.name }}" + shell: bash + run: | + set -ex + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_version }} -DCMAKE_BUILD_TYPE=Release .. + ninja -v | tee build.log + + - name: "Install Extended Testing Tools" + if: ${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }} + run: | + set -ex + sudo apt-get -y update + sudo apt install ansible -y + cd getting_started/setup_vm + ansible-playbook ccf-extended-testing.yml + shell: bash + + - name: "Test ${{ matrix.platform.name }}" + if: "${{ matrix.platform.name == 'virtual' }}" + run: | + set -ex + cd build + rm -rf /github/home/.cache + mkdir -p /github/home/.cache + export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15) + # Unit tests + ./tests.sh --output-on-failure -L unit -j$(nproc --all) + # Suite tests + ./tests.sh --timeout 600 --output-on-failure -L "suite" + # Most tests + ./tests.sh --timeout 360 --output-on-failure -LE "suite|${{ matrix.platform.test_filter }}" + shell: bash + + - name: "Upload logs for ${{ matrix.platform.os }}-${{ matrix.platform.name }}" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.platform.os }}-${{ matrix.platform.name }} + path: | + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + build/workspace/*.ledger/* + if-no-files-found: ignore + + - name: "Make .deb Package" + if: "${{ matrix.platform.os == 'ubuntu' }}" + id: make_deb + run: | + set -ex + set -o pipefail + cd build + cmake -L .. 2>/dev/null | grep CMAKE_INSTALL_PREFIX: | cut -d = -f 2 > /tmp/install_prefix + cpack -V -G DEB + INITIAL_PKG=`ls *.deb` + CCF_GITHUB_PKG=${INITIAL_PKG//\~/_} + if [[ "$INITIAL_PKG" != "$CCF_GITHUB_PKG" ]]; then + mv $INITIAL_PKG $CCF_GITHUB_PKG + fi + echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT + shell: bash + + - name: "Make .rpm Package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + id: make_rpm + run: | + set -ex + set -o pipefail + cd build + cmake -L .. 2>/dev/null | grep CMAKE_INSTALL_PREFIX: | cut -d = -f 2 > /tmp/install_prefix + cpack -V -G RPM + INITIAL_PKG=`ls *.rpm` + CCF_GITHUB_PKG=${INITIAL_PKG//\~/_} + if [[ "$INITIAL_PKG" != "$CCF_GITHUB_PKG" ]]; then + mv $INITIAL_PKG $CCF_GITHUB_PKG + fi + echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT + shell: bash + + - name: "Install CCF Debian package" + if: "${{ matrix.platform.os == 'ubuntu' }}" + run: | + set -ex + cd build + sudo apt -y install ./${{ steps.make_deb.outputs.name }} + shell: bash + + - name: "Install CCF RPM package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + run: | + set -ex + cd build + tdnf -y install ./${{ steps.make_rpm.outputs.name }} + shell: bash + + - name: "Test Installed CCF" + if: "${{ matrix.platform.name == 'virtual' }}" + run: | + set -ex + set -o pipefail + cd build + cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./test_install.sh {}" + shell: bash + + - name: "Recovery Benchmark for Installed CCF" + if: "${{ matrix.platform.name == 'virtual' }}" + run: | + set -ex + set -o pipefail + cd build + cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./recovery_benchmark.sh {}" + shell: bash + + - name: "Test Building a Sample Against Installed CCF" + run: | + set -ex + ./tests/test_install_build.sh -DCOMPILE_TARGET=${{ matrix.platform.name }} + shell: bash + + - name: "Upload .deb Package" + if: "${{ matrix.platform.os == 'ubuntu' }}" + uses: actions/upload-artifact@v4 + with: + name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} + path: build/${{ steps.make_deb.outputs.name }} + + - name: "Upload .rpm Package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + uses: actions/upload-artifact@v4 + with: + name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} + path: build/${{ steps.make_rpm.outputs.name }} + + - name: "Upload Compatibility Report" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" + uses: actions/upload-artifact@v4 + with: + name: compatibility + path: build/compatibility_report.json + + - name: "Upload TLS Report" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" + uses: actions/upload-artifact@v4 + with: + name: tls + path: build/tls_report.html + + - name: "Build Python Wheel" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" + id: build_wheel + run: | + set -ex + cd python + python3 -m venv env + source ./env/bin/activate + pip install wheel build + python -m build --wheel + WHL=`ls dist/*.whl` + echo "name=$WHL" >> $GITHUB_OUTPUT + shell: bash + + - name: "Upload Python Wheel" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" + uses: actions/upload-artifact@v4 + with: + name: wheel + path: python/${{ steps.build_wheel.outputs.name }} + + - name: "Build TS Package" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" + id: build_tstgz + run: | + set -ex + cd js/ccf-app + CCF_VERSION=$(<../../build/VERSION_LONG) + CCF_VERSION=${CCF_VERSION#ccf-} + echo "Setting npm package version to ${CCF_VERSION}" + npm version $CCF_VERSION + npm pack + PKG=`ls *.tgz` + echo "name=$PKG" >> $GITHUB_OUTPUT + shell: bash + + - name: "Upload TS Package" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" + uses: actions/upload-artifact@v4 + with: + name: tstgz + path: js/ccf-app/${{ steps.build_tstgz.outputs.name }} + + create_release: + needs: + - build_release + - make_sbom + name: Create Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Download Packages + uses: actions/download-artifact@v4 + with: + path: pkg + pattern: pkg-* + merge-multiple: true + - name: Download Release Notes + uses: actions/download-artifact@v4 + with: + name: relnotes + - name: Download Compatibility Report + uses: actions/download-artifact@v4 + with: + name: compatibility + - name: Download TLS Report + uses: actions/download-artifact@v4 + with: + name: tls + - name: Download Python Wheel + uses: actions/download-artifact@v4 + with: + path: wheel + name: wheel + - name: Download TS Package + uses: actions/download-artifact@v4 + with: + path: tstgz + name: tstgz + - name: Download SBOM + uses: actions/download-artifact@v4 + with: + path: sbom + name: sbom + - run: | + echo "Creating release will be done here" + shell: bash + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5a7692797958..dcdc58fd6b9a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,37 +62,75 @@ jobs: path: rel-notes.md build_release: - needs: release_notes name: Build Release + needs: release_notes strategy: matrix: platform: - name: virtual - image: default - nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + os: ubuntu + image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 + package_version: DEB + test_filter: "benchmark|unit" + - name: snp + os: ubuntu + image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 + package_version: DEB + - name: virtual + os: azure-linux + image: mcr.microsoft.com/azurelinux/base/core:3.0 + package_version: RPM + test_filter: "benchmark|unit|protocolstest|lts" - name: snp - image: default - nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - runs-on: ${{ matrix.platform.nodes }} + os: azure-linux + image: mcr.microsoft.com/azurelinux/base/core:3.0 + package_version: RPM + + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: - image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-14-01-2025 - options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro ${{ matrix.platform.container_options }}" + image: ${{ matrix.platform.image }} + options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro" + steps: + - name: "Install dependencies" + if: ${{ matrix.platform.os == 'azure-linux' }} + shell: bash + run: | + set -ex + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + # Source control + tdnf -y install ca-certificates git + # Build tools + tdnf -y install build-essential clang cmake ninja-build which + # libc++ + tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static + # Dependencies + tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel + # Test dependencies + tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect + # Install CDDL via rubygems + tdnf -y install rubygems + gem install cddl + # For packaging and release tests + tdnf -y install rpm-build procps + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: "Build Release ${{ matrix.platform.name }}" + shell: bash run: | set -ex git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} ${{ matrix.platform.cmake_options }} -DCLIENT_PROTOCOLS_TEST=ON .. + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_version }} -DCMAKE_BUILD_TYPE=Release .. ninja -v | tee build.log - shell: bash - name: "Install Extended Testing Tools" + if: ${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }} run: | set -ex sudo apt-get -y update @@ -100,9 +138,9 @@ jobs: cd getting_started/setup_vm ansible-playbook ccf-extended-testing.yml shell: bash - if: ${{ matrix.platform.name != 'snp' }} - name: "Test ${{ matrix.platform.name }}" + if: "${{ matrix.platform.name == 'virtual' }}" run: | set -ex cd build @@ -114,23 +152,23 @@ jobs: # Suite tests ./tests.sh --timeout 600 --output-on-failure -L "suite" # Most tests - ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|unit|suite" + ./tests.sh --timeout 360 --output-on-failure -LE "suite|${{ matrix.platform.test_filter }}" shell: bash - if: "${{ matrix.platform.name != 'snp' }}" - - name: "Upload logs for ${{ matrix.platform.name }}" + - name: "Upload logs for ${{ matrix.platform.os }}-${{ matrix.platform.name }}" + if: success() || failure() uses: actions/upload-artifact@v4 with: - name: logs-${{ matrix.platform.name }} + name: logs-${{ matrix.platform.os }}-${{ matrix.platform.name }} path: | build/workspace/*/*.config.json build/workspace/*/out build/workspace/*/err build/workspace/*.ledger/* if-no-files-found: ignore - if: success() || failure() - name: "Make .deb Package" + if: "${{ matrix.platform.os == 'ubuntu' }}" id: make_deb run: | set -ex @@ -146,30 +184,56 @@ jobs: echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT shell: bash + - name: "Make .rpm Package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + id: make_rpm + run: | + set -ex + set -o pipefail + cd build + cmake -L .. 2>/dev/null | grep CMAKE_INSTALL_PREFIX: | cut -d = -f 2 > /tmp/install_prefix + cpack -V -G RPM + INITIAL_PKG=`ls *.rpm` + CCF_GITHUB_PKG=${INITIAL_PKG//\~/_} + if [[ "$INITIAL_PKG" != "$CCF_GITHUB_PKG" ]]; then + mv $INITIAL_PKG $CCF_GITHUB_PKG + fi + echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT + shell: bash + - name: "Install CCF Debian package" + if: "${{ matrix.platform.os == 'ubuntu' }}" run: | set -ex cd build sudo apt -y install ./${{ steps.make_deb.outputs.name }} shell: bash + - name: "Install CCF RPM package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + run: | + set -ex + cd build + tdnf -y install ./${{ steps.make_rpm.outputs.name }} + shell: bash + - name: "Test Installed CCF" + if: "${{ matrix.platform.name == 'virtual' }}" run: | set -ex set -o pipefail cd build cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./test_install.sh {}" shell: bash - if: "${{ matrix.platform.name != 'snp' }}" - name: "Recovery Benchmark for Installed CCF" + if: "${{ matrix.platform.name == 'virtual' }}" run: | set -ex set -o pipefail cd build cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./recovery_benchmark.sh {}" shell: bash - if: "${{ matrix.platform.name != 'snp' }}" - name: "Test Building a Sample Against Installed CCF" run: | @@ -178,26 +242,35 @@ jobs: shell: bash - name: "Upload .deb Package" + if: "${{ matrix.platform.os == 'ubuntu' }}" uses: actions/upload-artifact@v4 with: - name: pkg-${{ matrix.platform.name }} + name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} path: build/${{ steps.make_deb.outputs.name }} + - name: "Upload .rpm Package" + if: "${{ matrix.platform.os == 'azure-linux' }}" + uses: actions/upload-artifact@v4 + with: + name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} + path: build/${{ steps.make_rpm.outputs.name }} + - name: "Upload Compatibility Report" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: compatibility path: build/compatibility_report.json - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Upload TLS Report" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: tls path: build/tls_report.html - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Build Python Wheel" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" id: build_wheel run: | set -ex @@ -209,16 +282,16 @@ jobs: WHL=`ls dist/*.whl` echo "name=$WHL" >> $GITHUB_OUTPUT shell: bash - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Upload Python Wheel" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: wheel path: python/${{ steps.build_wheel.outputs.name }} - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Build TS Package" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" id: build_tstgz run: | set -ex @@ -231,14 +304,13 @@ jobs: PKG=`ls *.tgz` echo "name=$PKG" >> $GITHUB_OUTPUT shell: bash - if: "${{ matrix.platform.name == 'virtual' }}" - name: "Upload TS Package" + if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" uses: actions/upload-artifact@v4 with: name: tstgz path: js/ccf-app/${{ steps.build_tstgz.outputs.name }} - if: "${{ matrix.platform.name == 'virtual' }}" create_release: needs: diff --git a/CMakeLists.txt b/CMakeLists.txt index 0926711cac87..59a4252c5fda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,11 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) ) endif() -include(${CCF_DIR}/cmake/cpack_settings.cmake) +if(PACKAGE_TYPE STREQUAL "DEB") + include(${CCF_DIR}/cmake/cpack_deb.cmake) +elseif(PACKAGE_TYPE STREQUAL "RPM") + include(${CCF_DIR}/cmake/cpack_rpm.cmake) +endif() message(STATUS "CMAKE_INSTALL_PREFIX is '${CMAKE_INSTALL_PREFIX}'") diff --git a/cmake/cpack_settings.cmake b/cmake/cpack_deb.cmake similarity index 100% rename from cmake/cpack_settings.cmake rename to cmake/cpack_deb.cmake diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake new file mode 100644 index 000000000000..ec528694e738 --- /dev/null +++ b/cmake/cpack_rpm.cmake @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +set(CPACK_PACKAGE_NAME "${CCF_PROJECT}") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Confidential Consortium Framework") +set(CPACK_PACKAGE_CONTACT "https://github.com/Microsoft/CCF") +set(CPACK_RESOURCE_FILE_LICENSE "${CCF_DIR}/LICENSE") +set(CPACK_PACKAGE_VERSION ${CCF_RELEASE_VERSION}) +set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) + +set(CPACK_RPM_PACKAGE_VERSION "${CCF_RELEASE_VERSION}") + +if(CCF_VERSION_SUFFIX) + set(CPACK_RPM_PACKAGE_VERSION + "${CPACK_RPM_PACKAGE_VERSION}~${CCF_VERSION_SUFFIX}" + ) +endif() + +message(STATUS "RPM package version: ${CPACK_RPM_PACKAGE_VERSION}") + +set(CCF_RPM_DEPENDENCIES + "libuv >= 1.34.2, openssl >= 3.3.0, nghttp2 >= 1.40.0, curl >= 7.68.0, libcxxabi >= 18.1.2" +) + +message(STATUS "RPM package dependencies: ${CCF_RPM_DEPENDENCIES}") + +set(CPACK_RPM_PACKAGE_REQUIRES "${CCF_RPM_DEPENDENCIES}") + +set(CPACK_RPM_FILE_NAME RPM-DEFAULT) + +include(CPack) diff --git a/tests/test_install_build.sh b/tests/test_install_build.sh index 4252b16accb2..9ba59b0d6a15 100755 --- a/tests/test_install_build.sh +++ b/tests/test_install_build.sh @@ -6,13 +6,19 @@ set -ex mkdir -p build_against_install cd build_against_install -CC=$(command -v clang-15 || true) -CXX=$(command -v clang++-15 || true) - -if [ "$CC" = "" ] || [ "$CXX" = "" ]; then +CC=$(which clang || true) +CXX=$(which clang++ || true) + +if [ "$CC" = "" ] || [ "$CXX" = "" ]; then + CC=$(command -v clang-15 || true) + CXX=$(command -v clang++-15 || true) +fi + +if [ "$CC" = "" ] || [ "$CXX" = "" ]; then CC=$(command -v clang-11) CXX=$(command -v clang++-11) fi CC=$CC CXX=$CXX cmake -GNinja "$@" ../samples/apps/logging/ + ninja \ No newline at end of file From 926299ef381b37f9cd7f0ec6f3badd6861ece9d5 Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 30 Jan 2025 12:54:54 +0000 Subject: [PATCH 2/8] Update - review --- .azure_pipelines_snp.yml | 2 +- .github/workflows/ci.yml | 25 +++++------------------- .github/workflows/long-test.yml | 21 +++----------------- .github/workflows/release-test.yml | 31 ++++++++---------------------- .github/workflows/release.yml | 31 +++++++----------------------- tests/test_install_build.sh | 5 ----- 6 files changed, 24 insertions(+), 91 deletions(-) diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml index 8500fa89700c..40b82cf60b9e 100644 --- a/.azure_pipelines_snp.yml +++ b/.azure_pipelines_snp.yml @@ -23,7 +23,7 @@ resources: containers: - container: virtual image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE jobs: - template: .azure-pipelines-templates/deploy_aci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d47126e58954..f116f7011603 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,11 +37,11 @@ jobs: - name: virtual image: default nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - name: snp image: default nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE runs-on: ${{ matrix.platform.nodes }} container: image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-14-01-2025 @@ -107,29 +107,14 @@ jobs: runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: image: mcr.microsoft.com/azurelinux/base/core:3.0 - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: - name: "Install dependencies" + shell: bash run: | set -ex - gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY - tdnf -y update - # Source control - tdnf -y install ca-certificates git - # Build tools - tdnf -y install build-essential clang cmake ninja-build which - # libc++ - tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static - # Dependencies - tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel - # Test dependencies - tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect - # Install CDDL via rubygems - tdnf -y install rubygems - gem install cddl - shell: bash - + ./scripts/install-azure-linux-deps.sh - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index a66e774cce0c..6fe91388e405 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -285,29 +285,14 @@ jobs: runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: image: mcr.microsoft.com/azurelinux/base/core:3.0 - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: - name: "Install dependencies" + shell: bash run: | set -ex - gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY - tdnf -y update - # Source control - tdnf -y install ca-certificates git - # Build tools - tdnf -y install build-essential clang cmake ninja-build which - # libc++ - tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static - # Dependencies - tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel - # Test dependencies - tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect - # Install CDDL via rubygems - tdnf -y install rubygems - gem install cddl - shell: bash - + ./scripts/install-azure-linux-deps.sh - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 7edfa556817b..20273347d8f1 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -74,26 +74,26 @@ jobs: - name: virtual os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_version: DEB + package_format: DEB test_filter: "benchmark|unit" - name: snp os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_version: DEB + package_format: DEB - name: virtual os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_version: RPM + package_format: RPM test_filter: "benchmark|unit|protocolstest|lts" - name: snp os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_version: RPM + package_format: RPM runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: image: ${{ matrix.platform.image }} - options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro" + options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" steps: - name: "Install dependencies" @@ -101,24 +101,9 @@ jobs: shell: bash run: | set -ex - gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY - tdnf -y update - # Source control - tdnf -y install ca-certificates git - # Build tools - tdnf -y install build-essential clang cmake ninja-build which - # libc++ - tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static - # Dependencies - tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel - # Test dependencies - tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect - # Install CDDL via rubygems - tdnf -y install rubygems - gem install cddl + ./scripts/install-azure-linux-deps.sh # For packaging and release tests - tdnf -y install rpm-build procps - + tdnf -y install rpm-build - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -130,7 +115,7 @@ jobs: git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_version }} -DCMAKE_BUILD_TYPE=Release .. + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_format }} -DCMAKE_BUILD_TYPE=Release .. ninja -v | tee build.log - name: "Install Extended Testing Tools" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dcdc58fd6b9a..d3d672ee1038 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,26 +70,26 @@ jobs: - name: virtual os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_version: DEB + package_format: DEB test_filter: "benchmark|unit" - name: snp os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_version: DEB + package_format: DEB - name: virtual os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_version: RPM + package_format: RPM test_filter: "benchmark|unit|protocolstest|lts" - name: snp os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_version: RPM + package_format: RPM runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: image: ${{ matrix.platform.image }} - options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro" + options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" steps: - name: "Install dependencies" @@ -97,24 +97,7 @@ jobs: shell: bash run: | set -ex - gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY - tdnf -y update - # Source control - tdnf -y install ca-certificates git - # Build tools - tdnf -y install build-essential clang cmake ninja-build which - # libc++ - tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static - # Dependencies - tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel - # Test dependencies - tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect - # Install CDDL via rubygems - tdnf -y install rubygems - gem install cddl - # For packaging and release tests - tdnf -y install rpm-build procps - + ./scripts/install-azure-linux-deps.sh - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -126,7 +109,7 @@ jobs: git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_version }} -DCMAKE_BUILD_TYPE=Release .. + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_format }} -DCMAKE_BUILD_TYPE=Release .. ninja -v | tee build.log - name: "Install Extended Testing Tools" diff --git a/tests/test_install_build.sh b/tests/test_install_build.sh index 9ba59b0d6a15..546c0f31803e 100755 --- a/tests/test_install_build.sh +++ b/tests/test_install_build.sh @@ -14,11 +14,6 @@ if [ "$CC" = "" ] || [ "$CXX" = "" ]; then CXX=$(command -v clang++-15 || true) fi -if [ "$CC" = "" ] || [ "$CXX" = "" ]; then - CC=$(command -v clang-11) - CXX=$(command -v clang++-11) -fi - CC=$CC CXX=$CXX cmake -GNinja "$@" ../samples/apps/logging/ ninja \ No newline at end of file From 18489f865f6aa4adbe4618288b15f78a0f578704 Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 30 Jan 2025 13:00:55 +0000 Subject: [PATCH 3/8] Missing script --- scripts/install-azure-linux-deps.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 scripts/install-azure-linux-deps.sh diff --git a/scripts/install-azure-linux-deps.sh b/scripts/install-azure-linux-deps.sh new file mode 100755 index 000000000000..edd1de9530c6 --- /dev/null +++ b/scripts/install-azure-linux-deps.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +set -ex + +gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY +tdnf -y update + +# Source control +tdnf -y install ca-certificates git + +# Build tools +tdnf -y install build-essential clang cmake ninja-build which + +# libc++ +tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static + +# Dependencies +tdnf -y install openssl-devel libuv-devel nghttp2-devel curl-devel + +# Test dependencies +tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect procps + +# Install CDDL via rubygems +tdnf -y install rubygems +gem install cddl From cce769c66ec67ba00c7136b1f334875bbc0c88e1 Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 30 Jan 2025 13:10:03 +0000 Subject: [PATCH 4/8] Reorder fix --- .github/workflows/ci.yml | 7 ++++--- .github/workflows/long-test.yml | 7 ++++--- .github/workflows/release-test.yml | 7 ++++--- .github/workflows/release.yml | 7 ++++--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f116f7011603..1917a3406471 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,14 +110,15 @@ jobs: options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Install dependencies" shell: bash run: | set -ex ./scripts/install-azure-linux-deps.sh - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: "Build Debug" run: | diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 6fe91388e405..17d6973cf5b9 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -288,14 +288,15 @@ jobs: options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Install dependencies" shell: bash run: | set -ex ./scripts/install-azure-linux-deps.sh - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: "Build Debug (Long Test)" run: | diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 20273347d8f1..05295dbe3f5e 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -96,6 +96,10 @@ jobs: options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Install dependencies" if: ${{ matrix.platform.os == 'azure-linux' }} shell: bash @@ -104,9 +108,6 @@ jobs: ./scripts/install-azure-linux-deps.sh # For packaging and release tests tdnf -y install rpm-build - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: "Build Release ${{ matrix.platform.name }}" shell: bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d3d672ee1038..1581f70500ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,15 +92,16 @@ jobs: options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Install dependencies" if: ${{ matrix.platform.os == 'azure-linux' }} shell: bash run: | set -ex ./scripts/install-azure-linux-deps.sh - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: "Build Release ${{ matrix.platform.name }}" shell: bash From d8d618bacb2be6873e9696b98f2b1f342c86b9e8 Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 30 Jan 2025 13:29:37 +0000 Subject: [PATCH 5/8] Try fix checkout --- .github/workflows/ci.yml | 7 +++++++ .github/workflows/long-test.yml | 7 +++++++ .github/workflows/release-test.yml | 8 ++++++++ .github/workflows/release.yml | 8 ++++++++ scripts/install-azure-linux-deps.sh | 3 --- 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1917a3406471..67d070e70615 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,6 +110,13 @@ jobs: options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: + - name: "Checkout dependencies" + shell: bash + run: | + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + tdnf -y install ca-certificates git + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 17d6973cf5b9..8d435e90eb3b 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -288,6 +288,13 @@ jobs: options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE steps: + - name: "Checkout dependencies" + shell: bash + run: | + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + tdnf -y install ca-certificates git + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 05295dbe3f5e..44159545f0cf 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -96,6 +96,14 @@ jobs: options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" steps: + - name: "Checkout dependencies" + if: ${{ matrix.platform.os == 'azure-linux' }} + shell: bash + run: | + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + tdnf -y install ca-certificates git + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1581f70500ec..d9527e72f6c3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,6 +92,14 @@ jobs: options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" steps: + - name: "Checkout dependencies" + if: ${{ matrix.platform.os == 'azure-linux' }} + shell: bash + run: | + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + tdnf -y install ca-certificates git + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/scripts/install-azure-linux-deps.sh b/scripts/install-azure-linux-deps.sh index edd1de9530c6..f610712c2919 100755 --- a/scripts/install-azure-linux-deps.sh +++ b/scripts/install-azure-linux-deps.sh @@ -4,9 +4,6 @@ set -ex -gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY -tdnf -y update - # Source control tdnf -y install ca-certificates git From 0ef1933d49ee3a957b7ec88604e8bab2eca32a70 Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 30 Jan 2025 15:31:42 +0000 Subject: [PATCH 6/8] Unite cpack --- .github/workflows/release-test.yml | 2 +- .github/workflows/release.yml | 6 +--- CMakeLists.txt | 6 +--- cmake/cpack_rpm.cmake | 31 ------------------- .../{cpack_deb.cmake => cpack_settings.cmake} | 26 ++++++++++++++++ 5 files changed, 29 insertions(+), 42 deletions(-) delete mode 100644 cmake/cpack_rpm.cmake rename cmake/{cpack_deb.cmake => cpack_settings.cmake} (65%) diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 44159545f0cf..232567f7f046 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -124,7 +124,7 @@ jobs: git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_format }} -DCMAKE_BUILD_TYPE=Release .. + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DCMAKE_BUILD_TYPE=Release .. ninja -v | tee build.log - name: "Install Extended Testing Tools" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d9527e72f6c3..51cea619765c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,21 +70,17 @@ jobs: - name: virtual os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_format: DEB test_filter: "benchmark|unit" - name: snp os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_format: DEB - name: virtual os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_format: RPM test_filter: "benchmark|unit|protocolstest|lts" - name: snp os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_format: RPM runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: @@ -118,7 +114,7 @@ jobs: git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build - CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DPACKAGE_TYPE=${{ matrix.platform.package_format }} -DCMAKE_BUILD_TYPE=Release .. + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DCMAKE_BUILD_TYPE=Release .. ninja -v | tee build.log - name: "Install Extended Testing Tools" diff --git a/CMakeLists.txt b/CMakeLists.txt index 59a4252c5fda..0926711cac87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,11 +36,7 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) ) endif() -if(PACKAGE_TYPE STREQUAL "DEB") - include(${CCF_DIR}/cmake/cpack_deb.cmake) -elseif(PACKAGE_TYPE STREQUAL "RPM") - include(${CCF_DIR}/cmake/cpack_rpm.cmake) -endif() +include(${CCF_DIR}/cmake/cpack_settings.cmake) message(STATUS "CMAKE_INSTALL_PREFIX is '${CMAKE_INSTALL_PREFIX}'") diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake deleted file mode 100644 index ec528694e738..000000000000 --- a/cmake/cpack_rpm.cmake +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. - -set(CPACK_PACKAGE_NAME "${CCF_PROJECT}") -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Confidential Consortium Framework") -set(CPACK_PACKAGE_CONTACT "https://github.com/Microsoft/CCF") -set(CPACK_RESOURCE_FILE_LICENSE "${CCF_DIR}/LICENSE") -set(CPACK_PACKAGE_VERSION ${CCF_RELEASE_VERSION}) -set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) - -set(CPACK_RPM_PACKAGE_VERSION "${CCF_RELEASE_VERSION}") - -if(CCF_VERSION_SUFFIX) - set(CPACK_RPM_PACKAGE_VERSION - "${CPACK_RPM_PACKAGE_VERSION}~${CCF_VERSION_SUFFIX}" - ) -endif() - -message(STATUS "RPM package version: ${CPACK_RPM_PACKAGE_VERSION}") - -set(CCF_RPM_DEPENDENCIES - "libuv >= 1.34.2, openssl >= 3.3.0, nghttp2 >= 1.40.0, curl >= 7.68.0, libcxxabi >= 18.1.2" -) - -message(STATUS "RPM package dependencies: ${CCF_RPM_DEPENDENCIES}") - -set(CPACK_RPM_PACKAGE_REQUIRES "${CCF_RPM_DEPENDENCIES}") - -set(CPACK_RPM_FILE_NAME RPM-DEFAULT) - -include(CPack) diff --git a/cmake/cpack_deb.cmake b/cmake/cpack_settings.cmake similarity index 65% rename from cmake/cpack_deb.cmake rename to cmake/cpack_settings.cmake index 9626f3c2a936..557927c81aee 100644 --- a/cmake/cpack_deb.cmake +++ b/cmake/cpack_settings.cmake @@ -8,6 +8,8 @@ set(CPACK_RESOURCE_FILE_LICENSE "${CCF_DIR}/LICENSE") set(CPACK_PACKAGE_VERSION ${CCF_RELEASE_VERSION}) set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) +# DEB-specific settings + set(CPACK_DEBIAN_PACKAGE_VERSION "${CCF_RELEASE_VERSION}") if(CCF_VERSION_SUFFIX) @@ -32,6 +34,30 @@ endif() list(JOIN CCF_DEB_DEPENDENCIES ", " CPACK_DEBIAN_PACKAGE_DEPENDS) +message(STATUS "DEB package dependencies: ${CCF_DEB_DEPENDENCIES}") + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) +# RPM-specific settings + +set(CPACK_RPM_PACKAGE_VERSION "${CCF_RELEASE_VERSION}") + +if(CCF_VERSION_SUFFIX) + set(CPACK_RPM_PACKAGE_VERSION + "${CPACK_RPM_PACKAGE_VERSION}~${CCF_VERSION_SUFFIX}" + ) +endif() + +message(STATUS "RPM package version: ${CPACK_RPM_PACKAGE_VERSION}") + +set(CCF_RPM_DEPENDENCIES + "libuv >= 1.34.2, openssl >= 3.3.0, nghttp2 >= 1.40.0, curl >= 7.68.0, libcxxabi >= 18.1.2" +) + +message(STATUS "RPM package dependencies: ${CCF_RPM_DEPENDENCIES}") + +set(CPACK_RPM_PACKAGE_REQUIRES "${CCF_RPM_DEPENDENCIES}") + +set(CPACK_RPM_FILE_NAME RPM-DEFAULT) + include(CPack) From 62bbfe6c420e564cb49cee4a3c4bcd2a34d0611b Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 30 Jan 2025 16:15:05 +0000 Subject: [PATCH 7/8] Update missing dep rpm --- .github/workflows/release-test.yml | 4 ---- .github/workflows/release.yml | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 232567f7f046..9cdb96c64d3b 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -74,21 +74,17 @@ jobs: - name: virtual os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_format: DEB test_filter: "benchmark|unit" - name: snp os: ubuntu image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - package_format: DEB - name: virtual os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_format: RPM test_filter: "benchmark|unit|protocolstest|lts" - name: snp os: azure-linux image: mcr.microsoft.com/azurelinux/base/core:3.0 - package_format: RPM runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] container: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 51cea619765c..0114eb71a9d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,6 +106,8 @@ jobs: run: | set -ex ./scripts/install-azure-linux-deps.sh + # For packaging and release tests + tdnf -y install rpm-build - name: "Build Release ${{ matrix.platform.name }}" shell: bash From d33d38efac50ec09515ed15ca815f949c8e7f15d Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Thu, 30 Jan 2025 16:15:56 +0000 Subject: [PATCH 8/8] remove test file --- .github/workflows/release-test.yml | 356 ----------------------------- 1 file changed, 356 deletions(-) delete mode 100644 .github/workflows/release-test.yml diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml deleted file mode 100644 index 9cdb96c64d3b..000000000000 --- a/.github/workflows/release-test.yml +++ /dev/null @@ -1,356 +0,0 @@ -name: Release (test) - -on: - pull_request: - types: - - labeled - - synchronize - - opened - - reopened - workflow_dispatch: - -permissions: - contents: write - actions: read - checks: write - -jobs: - make_sbom: - name: SBOM Generation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: "Install SBOM tool" - run: | - set -ex - curl -Lo sbom-tool https://github.com/microsoft/sbom-tool/releases/latest/download/sbom-tool-linux-x64 > sbom-tool - chmod +x sbom-tool - shell: bash - - name: "Produce SBOM" - run: | - set -ex - CCF_VERSION=${{ github.ref_name }} - CCF_VERSION=${CCF_VERSION#ccf-} - ./sbom-tool generate -b . -bc . -pn CCF -ps Microsoft -nsb https://sbom.microsoft -pv $CCF_VERSION -V Error - shell: bash - - name: "Upload SBOM" - uses: actions/upload-artifact@v4 - with: - name: sbom - path: _manifest/spdx_2.2/* - - # release_notes: - # name: Release Notes - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 1 - # - name: "Check Release Notes" - # run: | - # set -ex - # python scripts/extract-release-notes.py --target-git-version - # shell: bash - # - name: "Produce Release Notes" - # run: | - # set -ex - # set -o pipefail - # python ./scripts/extract-release-notes.py --target-git-version --describe-path-changes "./samples/constitution" | tee rel-notes.md - # - name: "Upload .deb Package" - # uses: actions/upload-artifact@v4 - # with: - # name: relnotes - # path: rel-notes.md - - build_release: - name: Build Release - # needs: release_notes - if: ${{ contains(github.event.pull_request.labels.*.name, 'release-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} - strategy: - matrix: - platform: - - name: virtual - os: ubuntu - image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - test_filter: "benchmark|unit" - - name: snp - os: ubuntu - image: ghcr.io/microsoft/ccf/ci/default:build-14-01-2025 - - name: virtual - os: azure-linux - image: mcr.microsoft.com/azurelinux/base/core:3.0 - test_filter: "benchmark|unit|protocolstest|lts" - - name: snp - os: azure-linux - image: mcr.microsoft.com/azurelinux/base/core:3.0 - - runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] - container: - image: ${{ matrix.platform.image }} - options: "--user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE" - - steps: - - name: "Checkout dependencies" - if: ${{ matrix.platform.os == 'azure-linux' }} - shell: bash - run: | - gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY - tdnf -y update - tdnf -y install ca-certificates git - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: "Install dependencies" - if: ${{ matrix.platform.os == 'azure-linux' }} - shell: bash - run: | - set -ex - ./scripts/install-azure-linux-deps.sh - # For packaging and release tests - tdnf -y install rpm-build - - - name: "Build Release ${{ matrix.platform.name }}" - shell: bash - run: | - set -ex - git config --global --add safe.directory /__w/CCF/CCF - mkdir build - cd build - CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=${{ matrix.platform.name }} -DCLIENT_PROTOCOLS_TEST=ON -DCMAKE_BUILD_TYPE=Release .. - ninja -v | tee build.log - - - name: "Install Extended Testing Tools" - if: ${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }} - run: | - set -ex - sudo apt-get -y update - sudo apt install ansible -y - cd getting_started/setup_vm - ansible-playbook ccf-extended-testing.yml - shell: bash - - - name: "Test ${{ matrix.platform.name }}" - if: "${{ matrix.platform.name == 'virtual' }}" - run: | - set -ex - cd build - rm -rf /github/home/.cache - mkdir -p /github/home/.cache - export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15) - # Unit tests - ./tests.sh --output-on-failure -L unit -j$(nproc --all) - # Suite tests - ./tests.sh --timeout 600 --output-on-failure -L "suite" - # Most tests - ./tests.sh --timeout 360 --output-on-failure -LE "suite|${{ matrix.platform.test_filter }}" - shell: bash - - - name: "Upload logs for ${{ matrix.platform.os }}-${{ matrix.platform.name }}" - if: success() || failure() - uses: actions/upload-artifact@v4 - with: - name: logs-${{ matrix.platform.os }}-${{ matrix.platform.name }} - path: | - build/workspace/*/*.config.json - build/workspace/*/out - build/workspace/*/err - build/workspace/*.ledger/* - if-no-files-found: ignore - - - name: "Make .deb Package" - if: "${{ matrix.platform.os == 'ubuntu' }}" - id: make_deb - run: | - set -ex - set -o pipefail - cd build - cmake -L .. 2>/dev/null | grep CMAKE_INSTALL_PREFIX: | cut -d = -f 2 > /tmp/install_prefix - cpack -V -G DEB - INITIAL_PKG=`ls *.deb` - CCF_GITHUB_PKG=${INITIAL_PKG//\~/_} - if [[ "$INITIAL_PKG" != "$CCF_GITHUB_PKG" ]]; then - mv $INITIAL_PKG $CCF_GITHUB_PKG - fi - echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT - shell: bash - - - name: "Make .rpm Package" - if: "${{ matrix.platform.os == 'azure-linux' }}" - id: make_rpm - run: | - set -ex - set -o pipefail - cd build - cmake -L .. 2>/dev/null | grep CMAKE_INSTALL_PREFIX: | cut -d = -f 2 > /tmp/install_prefix - cpack -V -G RPM - INITIAL_PKG=`ls *.rpm` - CCF_GITHUB_PKG=${INITIAL_PKG//\~/_} - if [[ "$INITIAL_PKG" != "$CCF_GITHUB_PKG" ]]; then - mv $INITIAL_PKG $CCF_GITHUB_PKG - fi - echo "name=$CCF_GITHUB_PKG" >> $GITHUB_OUTPUT - shell: bash - - - name: "Install CCF Debian package" - if: "${{ matrix.platform.os == 'ubuntu' }}" - run: | - set -ex - cd build - sudo apt -y install ./${{ steps.make_deb.outputs.name }} - shell: bash - - - name: "Install CCF RPM package" - if: "${{ matrix.platform.os == 'azure-linux' }}" - run: | - set -ex - cd build - tdnf -y install ./${{ steps.make_rpm.outputs.name }} - shell: bash - - - name: "Test Installed CCF" - if: "${{ matrix.platform.name == 'virtual' }}" - run: | - set -ex - set -o pipefail - cd build - cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./test_install.sh {}" - shell: bash - - - name: "Recovery Benchmark for Installed CCF" - if: "${{ matrix.platform.name == 'virtual' }}" - run: | - set -ex - set -o pipefail - cd build - cat /tmp/install_prefix | xargs -i bash -c "PYTHON_PACKAGE_PATH=../python ./recovery_benchmark.sh {}" - shell: bash - - - name: "Test Building a Sample Against Installed CCF" - run: | - set -ex - ./tests/test_install_build.sh -DCOMPILE_TARGET=${{ matrix.platform.name }} - shell: bash - - - name: "Upload .deb Package" - if: "${{ matrix.platform.os == 'ubuntu' }}" - uses: actions/upload-artifact@v4 - with: - name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} - path: build/${{ steps.make_deb.outputs.name }} - - - name: "Upload .rpm Package" - if: "${{ matrix.platform.os == 'azure-linux' }}" - uses: actions/upload-artifact@v4 - with: - name: pkg-${{ matrix.platform.os }}-${{ matrix.platform.name }} - path: build/${{ steps.make_rpm.outputs.name }} - - - name: "Upload Compatibility Report" - if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" - uses: actions/upload-artifact@v4 - with: - name: compatibility - path: build/compatibility_report.json - - - name: "Upload TLS Report" - if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" - uses: actions/upload-artifact@v4 - with: - name: tls - path: build/tls_report.html - - - name: "Build Python Wheel" - if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" - id: build_wheel - run: | - set -ex - cd python - python3 -m venv env - source ./env/bin/activate - pip install wheel build - python -m build --wheel - WHL=`ls dist/*.whl` - echo "name=$WHL" >> $GITHUB_OUTPUT - shell: bash - - - name: "Upload Python Wheel" - if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" - uses: actions/upload-artifact@v4 - with: - name: wheel - path: python/${{ steps.build_wheel.outputs.name }} - - - name: "Build TS Package" - if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" - id: build_tstgz - run: | - set -ex - cd js/ccf-app - CCF_VERSION=$(<../../build/VERSION_LONG) - CCF_VERSION=${CCF_VERSION#ccf-} - echo "Setting npm package version to ${CCF_VERSION}" - npm version $CCF_VERSION - npm pack - PKG=`ls *.tgz` - echo "name=$PKG" >> $GITHUB_OUTPUT - shell: bash - - - name: "Upload TS Package" - if: "${{ matrix.platform.os == 'ubuntu' && matrix.platform.name == 'virtual' }}" - uses: actions/upload-artifact@v4 - with: - name: tstgz - path: js/ccf-app/${{ steps.build_tstgz.outputs.name }} - - create_release: - needs: - - build_release - - make_sbom - name: Create Release - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: Download Packages - uses: actions/download-artifact@v4 - with: - path: pkg - pattern: pkg-* - merge-multiple: true - - name: Download Release Notes - uses: actions/download-artifact@v4 - with: - name: relnotes - - name: Download Compatibility Report - uses: actions/download-artifact@v4 - with: - name: compatibility - - name: Download TLS Report - uses: actions/download-artifact@v4 - with: - name: tls - - name: Download Python Wheel - uses: actions/download-artifact@v4 - with: - path: wheel - name: wheel - - name: Download TS Package - uses: actions/download-artifact@v4 - with: - path: tstgz - name: tstgz - - name: Download SBOM - uses: actions/download-artifact@v4 - with: - path: sbom - name: sbom - - run: | - echo "Creating release will be done here" - shell: bash - env: - GH_TOKEN: ${{ github.token }}