Use orjson for faster JSONL output
#6261
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Build | |
| on: [push, pull_request] | |
| env: | |
| PY_VER: "3.11" | |
| CMAKE_VERSION: "~3.25.0" | |
| FORCE_COLOR: "YES" | |
| KART_S3_TEST_DATA_POINT_CLOUD: "s3://kart-bring-your-own-data-poc/auckland-small-laz1.2/*.laz" | |
| KART_S3_TEST_DATA_RASTER: "s3://kart-bring-your-own-data-poc/erorisk_si/*.tif" | |
| AWS_NO_SIGN_REQUEST: "1" | |
| jobs: | |
| type-checking: | |
| name: Run Type Checking | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| cache: 'pip' # caching pip dependencies | |
| - run: pip install -r requirements/requirements.txt -r requirements/test.txt -r requirements/dev.txt | |
| - name: Run mypy | |
| run: mypy | |
| # Increase rate limits for public registry | |
| amazon-ecr-auth: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: 'arn:aws:iam::276514628126:role/ci-kart' | |
| aws-region: us-east-1 | |
| mask-aws-account-id: 'false' | |
| role-session-name: kart-ci-gha | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| registry-type: public | |
| mask-password: 'false' | |
| outputs: | |
| registry: ${{ steps.login-ecr.outputs.registry }} | |
| docker_username: ${{ steps.login-ecr.outputs.docker_username_public_ecr_aws }} | |
| docker_password: ${{ steps.login-ecr.outputs.docker_password_public_ecr_aws }} | |
| # | |
| # Linux development build on modern Ubuntu | |
| # | |
| Linux-dev: | |
| name: "Linux dev" | |
| runs-on: ubuntu-22.04 | |
| needs: amazon-ecr-auth | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.cache/ccache | |
| CCACHE_COMPRESS: "1" | |
| KART_POSTGIS_URL: "postgresql://postgres:@localhost:5432/postgres" | |
| KART_SQLSERVER_URL: "mssql://sa:PassWord1@localhost:1433/master" | |
| KART_MYSQL_URL: "mysql://root:PassWord1@localhost:3306" | |
| services: | |
| postgis: | |
| # https://github.com/postgis/docker-postgis/issues/216 | |
| image: ghcr.io/baosystems/postgis:15 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 2s | |
| --health-timeout 2s | |
| --health-retries 5 | |
| -e POSTGRES_HOST_AUTH_METHOD=trust | |
| ports: | |
| - 5432:5432 | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server | |
| options: >- | |
| -e ACCEPT_EULA=Y | |
| -e SA_PASSWORD=PassWord1 | |
| ports: | |
| - 1433:1433 | |
| mysql: | |
| image: public.ecr.aws/docker/library/mysql:latest | |
| credentials: | |
| username: ${{ needs.amazon-ecr-auth.outputs.docker_username }} | |
| password: ${{ needs.amazon-ecr-auth.outputs.docker_password }} | |
| options: >- | |
| -e MYSQL_ROOT_PASSWORD=PassWord1 | |
| ports: | |
| - 3306:3306 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/[email protected] # workaround for node 20 issue | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: ccache" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: vendor-ccache-ubuntu-22.04-cmake-vcpkg | |
| - name: "setup: misc" | |
| run: | | |
| sudo apt-get update -q -y | |
| sudo apt-get install -q -y --no-install-recommends autoconf-archive ccache | |
| mkdir -p ${{ env.CCACHE_DIR }} | |
| echo "/usr/lib/ccache" >> $GITHUB_PATH | |
| - name: "setup: vcpkg" | |
| uses: lukka/[email protected] | |
| with: | |
| vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| doNotCache: false # do-cache | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-linux | |
| - name: "app: build" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-linux | |
| buildPreset: ci-linux | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| run: | | |
| ./build/kart --version | |
| - name: "test: create no_postgis environment for tests" | |
| shell: bash | |
| run: | | |
| sudo apt-get -y install postgresql | |
| psql postgresql://postgres:@localhost:5432/postgres -c 'CREATE DATABASE no_postgis;' | |
| echo "KART_NO_POSTGIS_URL=postgresql://postgres:@localhost:5432/no_postgis" >> $GITHUB_ENV | |
| - name: "test: install database drivers for tests" | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - | |
| curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null | |
| sudo apt-get update -q -y | |
| sudo ACCEPT_EULA=Y apt-get install -q -y msodbcsql17 | |
| - name: "test: install fienode for reflink testing" | |
| run: | | |
| curl -L https://github.com/pwaller/fienode/releases/download/v1.0/fienode-linux-amd64 --output fienode | |
| chmod +x fienode | |
| mv fienode /usr/local/bin/ | |
| - name: "test: unit tests" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-linux | |
| testPreset: ci-linux | |
| - name: "test: dependency license check" | |
| run: | | |
| cmake --build build --target py-license-check | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: assemble" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-linux | |
| buildPreset: ci-bundle-linux | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart --version | |
| - name: "bundle: e2e tests" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-linux | |
| testPreset: ci-e2e-linux | |
| # | |
| # Install | |
| # | |
| - name: "install" | |
| shell: bash | |
| run: | | |
| cmake --install ./build --verbose | |
| /opt/kart/kart --version | |
| command -v kart | |
| kart --version | |
| - name: "install: e2e tests" | |
| shell: bash | |
| run: | | |
| time tests/scripts/e2e-1.sh | |
| # | |
| # Linux build for installers | |
| # Runs on centos 7 for maximum compatibility | |
| # | |
| Linux-Package: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - base: ubuntu-22.04 | |
| label: centos7/amd64 | |
| image: quay.io/pypa/manylinux2014_x86_64 | |
| arch: amd64 | |
| arch_triplet: x64-linux | |
| build_parallel: "" | |
| - base: buildjet-4vcpu-ubuntu-2204-arm | |
| label: centos7/arm64 | |
| image: quay.io/pypa/manylinux2014_aarch64 | |
| arch: arm64 | |
| arch_triplet: arm64-linux | |
| build_parallel: 1 | |
| name: Linux ${{ matrix.os.arch }} | |
| runs-on: ${{ matrix.os.base }} | |
| needs: amazon-ecr-auth | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| services: | |
| postgis: | |
| # https://github.com/postgis/docker-postgis/issues/216 | |
| image: ghcr.io/baosystems/postgis:15 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 2s | |
| --health-timeout 2s | |
| --health-retries 5 | |
| -e POSTGRES_HOST_AUTH_METHOD=trust | |
| ports: | |
| - 5432:5432 | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server | |
| options: >- | |
| -e ACCEPT_EULA=Y | |
| -e SA_PASSWORD=PassWord1 | |
| ports: | |
| - 1433:1433 | |
| mysql: | |
| image: public.ecr.aws/docker/library/mysql:latest | |
| credentials: | |
| username: ${{ needs.amazon-ecr-auth.outputs.docker_username }} | |
| password: ${{ needs.amazon-ecr-auth.outputs.docker_password }} | |
| options: >- | |
| -e MYSQL_ROOT_PASSWORD=PassWord1 | |
| ports: | |
| - 3306:3306 | |
| env: | |
| ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| KART_POSTGIS_URL: "postgresql://postgres:@postgis:5432/postgres" | |
| SQLSERVER_URL: "mssql://sa:PassWord1@sqlserver:1433/master?TrustServerCertificate=yes" | |
| KART_MYSQL_URL: "mysql://root:PassWord1@mysql:3306" | |
| ARCH: "${{ matrix.os.arch }}" | |
| ARCH_TRIPLET: "${{ matrix.os.arch_triplet }}" | |
| GITHUB_WORKSPACE: "/__w/kart/kart" # FIXME? | |
| PKG_CONFIG: "/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/tools/pkgconf/pkgconf" | |
| PKG_CONFIG_PATH: "/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/lib/pkgconfig" | |
| OPTIONS_FOR_CMAKE: >- | |
| [ | |
| '-DPKG_CONFIG_EXECUTABLE=/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/tools/pkgconf/pkgconf', | |
| '-DCPACK_DEBIAN_PACKAGE_ARCHITECTURE=${{ matrix.os.arch }}' | |
| ] | |
| container: | |
| image: ${{ matrix.os.image }} | |
| env: | |
| ACLOCAL_PATH: "/usr/local/share/aclocal:/usr/share/aclocal" | |
| PATH: "/opt/python/cp311-cp311/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/github/home/.cargo/bin:/opt/mssql-tools/bin" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/[email protected] # workaround for node 20 issue | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: rust" | |
| run: | | |
| curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | sh -s -- -y | |
| . "$HOME/.cargo/env" | |
| rustc --version | |
| - name: "setup: misc" | |
| shell: bash | |
| run: | | |
| echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV | |
| git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| yum install -y autoconf-archive perl-IPC-Cmd rpm-build unixODBC zip | |
| echo "installing golang..." | |
| curl -qL "https://go.dev/dl/go1.19.2.linux-${ARCH}.tar.gz" | tar xz -C /usr/local/ | |
| ln -sf /usr/local/go/bin/go /usr/local/go/bin/gofmt /usr/local/bin/ | |
| if [ "${ARCH}" == "arm64" ]; then | |
| echo "VCPKG_FORCE_SYSTEM_BINARIES=1" >> $GITHUB_ENV | |
| fi | |
| - name: "setup: vcpkg" | |
| uses: lukka/[email protected] | |
| with: | |
| vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| doNotCache: false # do-cache | |
| - name: "setup: pkgconf" | |
| working-directory: /__w/_temp | |
| shell: bash | |
| run: | | |
| ${GITHUB_WORKSPACE}/vcpkg-vendor/vcpkg/vcpkg install pkgconf | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| shell: bash | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/[email protected] | |
| env: | |
| LD_LIBRARY_PATH: "${{ env.LD_LIBRARY_PATH }}:${{ env.GITHUB_WORKSPACE }}/build/vcpkg_installed/${{ env.ARCH_TRIPLET }}/lib" | |
| with: | |
| configurePreset: ci-linux | |
| configurePresetAdditionalArgs: ${{ env.OPTIONS_FOR_CMAKE }} | |
| - name: "app: build" | |
| shell: bash | |
| run: | | |
| cmake --build --preset=ci-linux --verbose --parallel ${{ matrix.os.build_parallel }} | |
| - name: "vendor: save archive" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: vendor-linux-${{ env.ARCH }}-py${{ env.PY_VER }} | |
| path: build/vcpkg-vendor/kart-vendor.*.tar.gz | |
| if-no-files-found: error | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/kart --version | |
| - name: "test: create no_postgis environment for tests" | |
| shell: bash | |
| run: | | |
| yum install -y postgresql | |
| psql postgresql://postgres:@postgis:5432/postgres -c 'CREATE DATABASE no_postgis;' | |
| echo "KART_NO_POSTGIS_URL=postgresql://postgres:@postgis:5432/no_postgis" >> $GITHUB_ENV | |
| - name: "test: install MSSQL database drivers for tests" | |
| if: matrix.os.arch != 'arm64' | |
| shell: bash | |
| run: | | |
| curl https://packages.microsoft.com/config/rhel/7/prod.repo > /etc/yum.repos.d/mssql-release.repo | |
| ACCEPT_EULA=Y yum install -y msodbcsql18 mssql-tools | |
| echo "KART_SQLSERVER_URL=${SQLSERVER_URL}" >> $GITHUB_ENV | |
| - name: "test: unit tests" | |
| shell: bash | |
| run: | | |
| ctest --preset=ci-linux | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: assemble" | |
| shell: bash | |
| run: | | |
| cmake --build --preset=ci-bundle-linux --verbose | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart --version | |
| - name: "bundle: e2e tests" | |
| shell: bash | |
| run: | | |
| ctest --preset=ci-e2e-linux | |
| - name: "package: TGZ & DEB & RPM" | |
| shell: bash | |
| run: | | |
| cpack --preset=ci-linux | |
| - name: "package: save TGZ" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.os.arch }}-tgz | |
| path: build/dist/Kart-*.tar.gz | |
| if-no-files-found: error | |
| - name: "package: save DEB" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.os.arch }}-deb | |
| path: build/dist/kart_*.deb | |
| if-no-files-found: error | |
| - name: "package: save RPM" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.os.arch }}-rpm | |
| path: build/dist/kart-*.rpm | |
| if-no-files-found: error | |
| - name: "release" | |
| uses: softprops/action-gh-release@v1 | |
| if: "steps.version.outputs.is_release == 1" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| draft: true | |
| files: | | |
| build/dist/Kart-*.tar.gz | |
| build/dist/kart_*.deb | |
| build/dist/kart-*.rpm | |
| # | |
| # Linux Package tests | |
| # Install on a wide variety of Linux distributions and run E2E tests | |
| # | |
| Linux-Package-Tests: | |
| needs: [Linux-Package, amazon-ecr-auth] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - label: amd64 | |
| runner: ubuntu-22.04 | |
| - label: arm64 | |
| runner: buildjet-2vcpu-ubuntu-2204-arm | |
| os: | |
| - label: "tgz-ubuntu-22.04" | |
| image: "public.ecr.aws/ubuntu/ubuntu:jammy" | |
| type: "tgz" | |
| - label: "ubuntu-22.04" | |
| image: "public.ecr.aws/ubuntu/ubuntu:jammy" | |
| type: "deb" | |
| - label: "ubuntu-20.04" | |
| image: "public.ecr.aws/ubuntu/ubuntu:focal" | |
| type: "deb" | |
| - label: "ubuntu-18.04" | |
| image: "public.ecr.aws/ubuntu/ubuntu:bionic" | |
| type: "deb" | |
| - label: "debian-stable" | |
| image: "public.ecr.aws/docker/library/debian:stable-slim" | |
| type: "deb" | |
| - label: "debian-oldstable" | |
| image: "public.ecr.aws/docker/library/debian:oldstable-slim" | |
| type: "deb" | |
| - label: "amazonlinux-2023" | |
| image: "public.ecr.aws/amazonlinux/amazonlinux:2023" | |
| type: "rpm" | |
| - label: "fedora-latest" | |
| image: "public.ecr.aws/docker/library/fedora:latest" | |
| type: "rpm" | |
| - label: "almalinux-9" | |
| image: "public.ecr.aws/docker/library/almalinux:9" | |
| type: "rpm" | |
| - label: "almalinux-8" | |
| image: "public.ecr.aws/docker/library/almalinux:8" | |
| type: "rpm" | |
| runs-on: ${{ matrix.arch.runner }} | |
| name: "Install-check: ${{ matrix.os.label }}/${{ matrix.arch.label }}" | |
| continue-on-error: ${{ !!matrix.os.xfail }} | |
| env: | |
| ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| container: | |
| image: ${{ matrix.os.image }} | |
| credentials: | |
| username: ${{ needs.amazon-ecr-auth.outputs.docker_username }} | |
| password: ${{ needs.amazon-ecr-auth.outputs.docker_password }} | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| steps: | |
| - name: setup | |
| shell: bash | |
| run: | | |
| case "${{ matrix.os.type }}" in | |
| deb | tgz) | |
| apt update -q | |
| apt install -q -y --no-install-recommends sqlite3 | |
| ;; | |
| rpm) | |
| yum install -y sqlite tar | |
| ;; | |
| *) | |
| exit 1 | |
| ;; | |
| esac | |
| - name: get kart installer | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: Kart-linux-${{ matrix.arch.label }}-${{ matrix.os.type }} | |
| - name: install kart | |
| shell: bash | |
| run: | | |
| case "${{ matrix.os.type }}" in | |
| deb) | |
| apt install -y ./kart_*.deb | |
| ;; | |
| rpm) | |
| yum install -y ./kart-*.rpm | |
| ;; | |
| tgz) | |
| tar -xz -C /opt --strip-components=1 -f ./Kart-*.tar.gz | |
| ln -sf /opt/kart/kart /usr/local/bin/kart | |
| ;; | |
| *) | |
| exit 1 | |
| ;; | |
| esac | |
| - name: smoke test | |
| shell: bash | |
| run: | | |
| /opt/kart/kart --version | |
| command -v kart | |
| kart --version | |
| - uses: actions/checkout@v3 | |
| - name: e2e test (no helper) | |
| env: | |
| KART_USE_HELPER: "0" | |
| shell: bash | |
| run: | | |
| time tests/scripts/e2e-1.sh | |
| - name: e2e test (helper) | |
| env: | |
| KART_USE_HELPER: "1" | |
| shell: bash | |
| run: | | |
| time tests/scripts/e2e-1.sh | |
| # | |
| # Linux development build using vendor archive & host python | |
| # | |
| Linux-dev-vendor: | |
| name: "Linux dev+vendor" | |
| needs: ["Linux-Package"] | |
| runs-on: ubuntu-22.04 | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.cache/ccache | |
| CCACHE_COMPRESS: "1" | |
| PY_VER_ID: "cp311-cp311" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - name: "setup: python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PY_VER }} | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements/*.txt' | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/[email protected] # workaround for node 20 issue | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: ccache" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: vendor-ccache-ubuntu-22.04-cmake-vcpkg | |
| - name: "setup: misc" | |
| run: | | |
| sudo apt-get update -q -y | |
| sudo apt-get install -q -y --no-install-recommends autoconf-archive ccache unixodbc file sqlite3 | |
| mkdir -p ${{ env.CCACHE_DIR }} | |
| echo "/usr/lib/ccache" >> $GITHUB_PATH | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| - name: "app: get vendor archive" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: vendor-linux-amd64-py${{ env.PY_VER }} | |
| - name: "app: configuration & vendor dependencies" | |
| shell: bash | |
| run: | | |
| cmake -B ./build -S . \ | |
| -DUSE_VCPKG=NO \ | |
| -DVENDOR_ARCHIVE=./kart-vendor.${PY_VER_ID}-linux_x86_64.tar.gz \ | |
| -DBUILD_TESTING=YES | |
| - name: "app: build" | |
| shell: bash | |
| run: | | |
| cmake --build ./build --verbose | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/kart --version | |
| - name: "bundle: assemble" | |
| shell: bash | |
| run: | | |
| cmake --build ./build --target bundle --verbose | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart --version | |
| - name: "bundle: e2e tests" | |
| shell: bash | |
| working-directory: ./build | |
| run: | | |
| ctest -V -L e2e | |
| # | |
| # macOS | |
| # | |
| macOS: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - id: macos-13 | |
| label: "13/x86_64" | |
| arch_triplet: x64-osx | |
| - id: "ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1" | |
| label: "13/arm64" | |
| arch_triplet: arm64-osx | |
| runs-on: ${{ matrix.os.id }} | |
| name: macOS ${{ matrix.os.label }} | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.cache/ccache | |
| CCACHE_COMPRESS: "1" | |
| ARCH_TRIPLET: ${{ matrix.os.arch_triplet }} | |
| HOMEBREW_CACHE: ${{ github.workspace }}/.cache/brew | |
| HOMEBREW_NO_INSTALL_CLEANUP: "1" | |
| MACOS_CODESIGN_ID: ${{ secrets.MACOS_CODESIGN_ID }} | |
| MACOS_PKGSIGN_ID: ${{ secrets.MACOS_PKGSIGN_ID }} | |
| MACOS_NOTARIZE_KEYCHAIN_PROFILE: "NOTARIZE_AUTH" | |
| MACOS_NOTARIZE_KEYCHAIN_FILENAME: "signing_temp" | |
| MACOS_NOTARIZE_KEYCHAIN: "~/Library/Keychains/signing_temp.keychain-db" | |
| # X.Y version needs to match PY_VER: | |
| PY_VER_INSTALLER: "https://www.python.org/ftp/python/3.11.6/python-3.11.6-macos11.pkg" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # | |
| # setup | |
| # | |
| - uses: maxim-lobanov/setup-xcode@v1 | |
| with: | |
| xcode-version: latest-stable | |
| # TODO: cache | |
| - name: "setup: python" | |
| run: | | |
| curl -sSfL -o ${{runner.temp}}/py.pkg "${PY_VER_INSTALLER}" | |
| sudo installer -pkg ${{runner.temp}}/py.pkg -target / | |
| sudo ln -sf python3 /Library/Frameworks/Python.framework/Versions/${PY_VER}/bin/python | |
| export PATH="/Library/Frameworks/Python.framework/Versions/${PY_VER}/bin:$PATH" | |
| which python3 | |
| python3 -V --version | |
| which python | |
| python -V --version | |
| echo /Library/Frameworks/Python.framework/Versions/${PY_VER}/bin >> $GITHUB_PATH | |
| - name: "setup: golang" | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '^1.21.1' | |
| - name: "setup: rust" | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: "setup: homebrew cache" | |
| id: cache-brew | |
| uses: actions/cache@v3 | |
| with: | |
| path: .cache/brew | |
| key: brew-cmake-${{ matrix.os.id }}-vcpkg | |
| - name: "setup: cmake & ninja" | |
| uses: lukka/get-cmake@latest | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| - name: "setup: ccache" | |
| uses: actions/cache@v3 | |
| if: steps.cache-vendor-dist.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: vendor-ccache-${{ matrix.os.id }}-vcpkg | |
| # The brew install command sometimes fails on the first try but succeeds on the second :-/ | |
| - name: "setup: misc" | |
| run: | | |
| PACKAGES="autoconf automake ccache libtool openssl pandoc pkg-config" | |
| brew install -q $PACKAGES || brew install -q $PACKAGES | |
| mkdir -p ${{ env.CCACHE_DIR }} | |
| echo "$(brew --prefix)/opt/ccache/libexec" >> $GITHUB_PATH | |
| - name: "setup: vcpkg" | |
| uses: lukka/run-vcpkg@v11 | |
| with: | |
| vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| doNotCache: false # do-cache | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| env: | |
| MACOS_NOTARIZE_TEAMID: ${{ secrets.MACOS_NOTARIZE_TEAMID }} | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| if [[ -n "$MACOS_CODESIGN_ID" ]]; then | |
| echo "MACOS_SIGN_BUNDLE=ON" >> $GITHUB_ENV | |
| else | |
| echo "MACOS_SIGN_BUNDLE=OFF" >> $GITHUB_ENV | |
| fi | |
| if (( $IS_RELEASE )) && [[ -n "$MACOS_PKGSIGN_ID" ]]; then | |
| echo "MACOS_SIGN_PKG=ON" >> $GITHUB_ENV | |
| else | |
| echo "MACOS_SIGN_PKG=OFF" >> $GITHUB_ENV | |
| fi | |
| if (( $IS_RELEASE )) && [[ -n "$MACOS_NOTARIZE_TEAMID" ]]; then | |
| echo "MACOS_NOTARIZE=ON" >> $GITHUB_ENV | |
| else | |
| echo "MACOS_NOTARIZE=OFF" >> $GITHUB_ENV | |
| fi | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-macos | |
| - name: "app: build" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-macos | |
| buildPreset: ci-macos | |
| - name: "vendor: save archive" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: vendor-macos-${{ runner.arch }}-py${{ env.PY_VER }} | |
| path: build/vcpkg-vendor/kart-vendor.*.tar.gz | |
| if-no-files-found: error | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| run: | | |
| ./build/kart --version | |
| - name: "test: install clone_checker for reflink testing" | |
| run: | | |
| git clone https://github.com/dyorgio/apfs-clone-checker.git | |
| cd apfs-clone-checker | |
| gcc clone_checker.c -o clone_checker | |
| chmod +x clone_checker | |
| sudo mv clone_checker /usr/local/bin/ | |
| - name: "test: unit tests" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-macos | |
| testPreset: ci-macos | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: setup app signing certificate" | |
| id: keychain | |
| uses: apple-actions/import-codesign-certs@v1 | |
| if: env.MACOS_SIGN_BUNDLE == 'ON' | |
| with: | |
| # This action auto generates the keychain password (only works if the keychain name is "signing_temp"). | |
| keychain: ${{ env.MACOS_NOTARIZE_KEYCHAIN_FILENAME }} | |
| p12-file-base64: ${{ secrets.MACOS_APP_CERT }} | |
| p12-password: ${{ secrets.MACOS_CERT_PW }} | |
| - name: "bundle: setup notarization auth" | |
| if: env.MACOS_NOTARIZE == 'ON' | |
| env: | |
| MACOS_NOTARIZE_TEAMID: ${{ secrets.MACOS_NOTARIZE_TEAMID }} | |
| MACOS_NOTARIZE_USER: ${{ secrets.MACOS_NOTARIZE_USER }} | |
| MACOS_NOTARIZE_PW: ${{ secrets.MACOS_NOTARIZE_PW }} | |
| run: | | |
| xcrun notarytool store-credentials "$MACOS_NOTARIZE_KEYCHAIN_PROFILE" \ | |
| --apple-id "$MACOS_NOTARIZE_USER" \ | |
| --team-id "$MACOS_NOTARIZE_TEAMID" \ | |
| --password "$MACOS_NOTARIZE_PW" \ | |
| --keychain "$MACOS_NOTARIZE_KEYCHAIN" | |
| - name: "bundle: assemble" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-macos | |
| buildPreset: ci-bundle-macos | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/Kart.app/Contents/MacOS/kart --version | |
| - name: "bundle: e2e tests" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-macos | |
| testPreset: ci-e2e-macos | |
| - name: "package: setup installer signing certificate" | |
| uses: apple-actions/import-codesign-certs@v1 | |
| if: env.MACOS_SIGN_PKG == 'ON' | |
| with: | |
| create-keychain: false | |
| keychain-password: ${{ steps.keychain.outputs.keychain-password }} | |
| p12-file-base64: ${{ secrets.MACOS_INSTALLER_CERT }} | |
| p12-password: ${{ secrets.MACOS_CERT_PW }} | |
| - name: "package: ZIP & PKG" | |
| shell: bash | |
| run: | | |
| cpack --preset=ci-macos | |
| - name: "package: save ZIP" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-macOS-${{ runner.arch }}-bundle | |
| path: build/dist/Kart-*.zip | |
| if-no-files-found: error | |
| - name: "package: save PKG" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-macOS-${{ runner.arch }}-installer | |
| path: build/dist/Kart-*.pkg | |
| if-no-files-found: error | |
| - name: "package: tests" | |
| run: | | |
| sudo installer -pkg build/dist/Kart-*.pkg -dumplog -target / | |
| readlink $(which kart) | |
| # make sure good sqlite is in path | |
| export PATH="$(pwd)/build/vcpkg_installed/${{ env.ARCH_TRIPLET }}/tools:$PATH" | |
| tests/scripts/e2e-1.sh | |
| - name: "release" | |
| uses: softprops/action-gh-release@v1 | |
| if: "steps.version.outputs.is_release == 1" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| draft: true | |
| files: | | |
| build/dist/Kart-*.zip | |
| build/dist/Kart-*.pkg | |
| # | |
| # Windows | |
| # | |
| Windows: | |
| strategy: | |
| matrix: | |
| os: [{id: windows-2019, label: "2019/x64"}] | |
| runs-on: ${{ matrix.os.id }} | |
| name: Windows ${{ matrix.os.label }} | |
| env: | |
| NINJA_VERSION: "~1.10.2" # workaround for python logging output buffering noise | |
| SIGN_AZURE_CERTIFICATE: ${{ secrets.WIN_SIGN_AZURE_CERTIFICATE }} | |
| # We want to run on external PRs, but not on our own internal PRs as they'll be run | |
| # by the push to the branch. | |
| # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7 | |
| if: > | |
| ( | |
| github.event_name == 'push' | |
| || github.event.pull_request.head.repo.full_name != github.repository | |
| ) | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # Setup | |
| - name: "msvc setup" | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| - name: "Harden vcvarsall.bat" | |
| shell: pwsh | |
| run: | | |
| cd 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build' | |
| mv vcvarsall.bat orig_vcvarsall.bat | |
| echo @' | |
| @if not "%VSCMD_DEBUG%" GEQ "3" echo off | |
| FOR /F "tokens=*" %%g IN ('powershell -Command "($Env:PATH.split(';') | Select -Unique) -Join ';'"') do (SET PATH=%%g) | |
| '@ > vcvarsall.bat | |
| type orig_vcvarsall.bat >> vcvarsall.bat | |
| - name: "setup: cmake" | |
| uses: lukka/get-cmake@latest | |
| with: | |
| cmakeVersion: "${{ env.CMAKE_VERSION }}" | |
| ninjaVersion: "${{ env.NINJA_VERSION }}" | |
| - name: "setup: python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{ env.PY_VER }}" | |
| architecture: x64 | |
| cache: 'pip' | |
| cache-dependency-path: 'requirements/*.txt' | |
| - name: "setup: vcpkg" | |
| uses: lukka/run-vcpkg@v11 | |
| with: | |
| vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg" | |
| vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json" | |
| doNotCache: false # do-cache | |
| - name: "setup: misc" | |
| shell: pwsh | |
| run: | | |
| & dotnet tool install --global AzureSignTool --version 3.0.0 | |
| & choco install sqlite.shell | |
| # | |
| # App Build | |
| # | |
| - name: "app: version" | |
| id: version | |
| shell: bash | |
| run: | | |
| if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then | |
| VER=$(echo ${{ github.ref_name }} | sed 's/^v//') | |
| IS_RELEASE=1 | |
| else | |
| VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION) | |
| IS_RELEASE=0 | |
| fi | |
| echo "App Version: $VER" | |
| echo "Is Release? $IS_RELEASE" | |
| echo "value=$VER" >> $GITHUB_OUTPUT | |
| echo "KART_VERSION=$VER" >> $GITHUB_ENV | |
| echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT | |
| if (( $IS_RELEASE )) && [[ -n "$SIGN_AZURE_CERTIFICATE" ]]; then | |
| echo "WIN_SIGN_BUNDLE=ON" >> $GITHUB_ENV | |
| echo "WIN_SIGN_INSTALLER=ON" >> $GITHUB_ENV | |
| else | |
| echo "WIN_SIGN_BUNDLE=OFF" >> $GITHUB_ENV | |
| echo "WIN_SIGN_INSTALLER=OFF" >> $GITHUB_ENV | |
| fi | |
| - name: "app: configuration & vendor dependencies" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-windows | |
| - name: "app: build" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-windows | |
| buildPreset: ci-windows | |
| - name: "vendor: save archive" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: vendor-windows-${{ runner.arch }}-py${{ env.PY_VER }} | |
| path: build/vcpkg-vendor/kart-vendor.*.zip | |
| if-no-files-found: error | |
| # | |
| # App tests & checks | |
| # | |
| - name: "test: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/kart.cmd --version | |
| - name: "test: unit tests" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-windows | |
| testPreset: ci-windows | |
| # | |
| # Packaging | |
| # | |
| - name: "bundle: assemble" | |
| uses: lukka/[email protected] | |
| env: | |
| SIGN_AZURE_VAULT: ${{ secrets.WIN_SIGN_AZURE_VAULT }} | |
| SIGN_AZURE_CLIENTID: ${{ secrets.WIN_SIGN_AZURE_CLIENTID }} | |
| SIGN_AZURE_CLIENTSECRET: ${{ secrets.WIN_SIGN_AZURE_CLIENTSECRET }} | |
| SIGN_AZURE_TENANTID: ${{ secrets.WIN_SIGN_AZURE_TENANTID }} | |
| with: | |
| configurePreset: ci-windows | |
| buildPreset: ci-bundle-windows | |
| - name: "bundle: smoke test" | |
| shell: bash | |
| run: | | |
| ./build/pyinstaller/dist/kart/kart.exe --version | |
| - name: "bundle: e2e tests" | |
| uses: lukka/[email protected] | |
| with: | |
| configurePreset: ci-windows | |
| testPreset: ci-e2e-windows | |
| - name: "package: ZIP & MSI" | |
| env: | |
| SIGN_AZURE_VAULT: ${{ secrets.WIN_SIGN_AZURE_VAULT }} | |
| SIGN_AZURE_CLIENTID: ${{ secrets.WIN_SIGN_AZURE_CLIENTID }} | |
| SIGN_AZURE_CLIENTSECRET: ${{ secrets.WIN_SIGN_AZURE_CLIENTSECRET }} | |
| SIGN_AZURE_TENANTID: ${{ secrets.WIN_SIGN_AZURE_TENANTID }} | |
| shell: bash | |
| run: | | |
| if ! cpack --preset=ci-windows; then | |
| cat build/dist/_CPack_Packages/win64/WIX/wix.log | |
| exit 1 | |
| fi | |
| - name: "package: save ZIP" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-windows-${{ runner.arch }}-bundle | |
| path: build/dist/Kart-*.zip | |
| if-no-files-found: error | |
| - name: "package: save MSI" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Kart-windows-${{ runner.arch }}-installer | |
| path: build/dist/Kart-*.msi | |
| if-no-files-found: error | |
| - name: "package: tests" | |
| shell: pwsh | |
| run: | | |
| $ErrorView = 'NormalView' | |
| $msi = Resolve-Path -Path ".\build\dist\Kart-*.msi" | Select-Object -ExpandProperty Path | |
| $sqlitePath = Resolve-Path -Path ".\build\vcpkg_installed\x64-windows\tools" | Select-Object -ExpandProperty Path | |
| Write-Host "Installing $msi ..." -ForegroundColor Green | |
| & "msiexec.exe" /I $msi /quiet /norestart /l* install.log | Out-Default | |
| if (!$?) { | |
| Get-Content install.log | |
| throw "MSI install failed" | |
| } | |
| Write-Host "Refreshing system path..." -ForegroundColor Green | |
| $env:Path = [System.Environment]::GetEnvironmentVariable("Path", "Machine") + ";" +[System.Environment]::GetEnvironmentVariable("Path", "User") + ";" + $sqlitePath | |
| Write-Host "Checking Kart runs..." -ForegroundColor Green | |
| & "kart" --version | Out-Default | |
| if (!$?) {throw "Smoke test Failed"} | |
| Write-Host "Running Kart E2E tests..." -ForegroundColor Green | |
| & "tests\scripts\e2e-1.ps1" | Out-Default | |
| if (!$?) {throw "E2E test Failed"} | |
| - name: "release" | |
| uses: softprops/action-gh-release@v1 | |
| if: "steps.version.outputs.is_release == 1" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| draft: true | |
| files: | | |
| build/dist/Kart-*.zip | |
| build/dist/Kart-*.msi |