Merge pull request #1039 from Aiven-Open/jjaakola-aiven-fastapi

feat: Karapace modernization with uvicorn, FastAPI and OpenTelemetry

.coveragerc

@@ -1,4 +1,8 @@
 [run]
 branch = true
 relative_files = true
-source = src/karapace
+source = src
+disable_warnings = module-not-measured, no-data-collected
+
+[report]
+skip_empty = true

.dockerignore

@@ -4,13 +4,12 @@
 
 # Include source directories and files required for building.
 !go
-!src
+!src/karapace
 !requirements/*.txt
 !README.rst
 !LICENSE
 !pyproject.toml
 !setup.py
-!container/start.sh
 !container/healthcheck.py
 
 # Ignore some files in source directories.

.github/workflows/container-smoke-test.yml

@@ -8,6 +8,9 @@ on:
 jobs:
   smoke-test-container:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.10', '3.11', '3.12' ]
     env:
       BUILDKIT_PROGRESS: plain
     steps:
@@ -17,23 +20,26 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install requirements
-        run: make install
-
       - name: Resolve Karapace version
-        run: |
-          source ./venv/bin/activate
-          KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)")
-          echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV
+        run: echo KARAPACE_VERSION="$(git describe --tags | cut -d '-' -f -2 | sed 's/-/.dev/g')" >> $GITHUB_ENV
 
-      - name: Build container
-        run: docker build --build-arg KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} --file=container/Dockerfile .
+      - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+      - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
 
       - name: Run container
-        run: docker compose --file=container/compose.yml up --build --wait --detach
-
-      - name: Smoke test registry
-        run: bin/smoke-test-registry.sh
+        run: make start-karapace-docker-resources
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+          RUNNER_UID: ${{ env.RUNNER_UID }}
+          RUNNER_GID: ${{ env.RUNNER_GID }}
+
+      - name: Smoke test schema registry
+        run: bin/smoke-test-schema-registry.sh
+        env:
+          KARAPACE_PORT: 8081
 
       - name: Smoke test REST proxy
-        run: bin/smoke-test-rest.sh
+        run: bin/smoke-test-rest-proxy.sh
+        env:
+          KARAPACE_PORT: 8082

.github/workflows/container.yml

@@ -78,5 +78,6 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             KARAPACE_VERSION=${{ steps.ctx.outputs.version }}
+            PYTHON_VERSION=3.10.11
           file: container/Dockerfile
           platforms: "linux/amd64,linux/arm64"

.github/workflows/lint.yml

@@ -37,17 +37,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      # Need fetch-depth 0 to fetch tags, see https://github.com/actions/checkout/issues/701
+      with:
+        fetch-depth: 0
+
     - uses: actions/setup-python@v5
       with:
-        cache: pip
         python-version: '3.12'
-    - name: Install libsnappy-dev
-      run: sudo apt install libsnappy-dev
-    - name: Install requirements and typing requirements
-      run: |
-        python -m venv venv
-        source ./venv/bin/activate
-        pip --require-virtualenv install .[typing]
-    - run: |
-        source ./venv/bin/activate
-        mypy src
+
+    - name: Resolve Karapace version
+      run: echo KARAPACE_VERSION="$(git describe --tags | cut -d '-' -f -2 | sed 's/-/.dev/g')" >> $GITHUB_ENV
+
+    - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+    - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
+
+    - name: Run mypy
+      run: make type-check-mypy-in-docker
+      env:
+        PYTHON_VERSION: 3.12
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}

.github/workflows/tests.yml

@@ -20,34 +20,80 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
+        python-version: [ '3.10', '3.11', '3.12' ]
     env:
       PYTEST_ADDOPTS: >-
         --log-dir=/tmp/ci-logs
         --log-file=/tmp/ci-logs/pytest.log
         --showlocals
     steps:
     - uses: actions/checkout@v4
+      # Need fetch-depth 0 to fetch tags, see https://github.com/actions/checkout/issues/701
+      with:
+        fetch-depth: 0
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        cache: pip
         python-version: ${{ matrix.python-version }}
 
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
         go-version: '1.21.0'
 
-    - run: make install-dev
-    - run: make unit-tests
+    - name: Resolve Karapace version
+      run: echo KARAPACE_VERSION="$(git describe --tags | cut -d '-' -f -2 | sed 's/-/.dev/g')" >> $GITHUB_ENV
+
+    - run: echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
+    - run: echo "RUNNER_GID=$(id -g)" >> $GITHUB_ENV
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Cache docker layers
+      uses: docker/build-push-action@v4
       env:
-        COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
+        PYTHON_VERSION: ${{ matrix.python-version }}
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        PR_TAG: aiven-open/karapace-pr:${{ env.KARAPACE_VERSION }}
+      with:
+        context: .
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        tags: ${{ env.PR_TAG }}
+        push: false
+        build-args: |
+          KARAPACE_VERSION=${{ env.KARAPACE_VERSION }}
+          PYTHON_VERSION=${{ matrix.python-version }}
+        file: container/Dockerfile
+        platforms: linux/amd64
+
+    - run: make unit-tests-in-docker
+      env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
+        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4"
+
+    - run: make e2e-tests-in-docker
+      env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
         PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4"
-    - run: make integration-tests
+
+    - run: make integration-tests-in-docker
       env:
-        COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
+        PYTHON_VERSION: ${{ matrix.python-version }}
+        KARAPACE_VERSION: ${{ env.KARAPACE_VERSION }}
+        RUNNER_UID: ${{ env.RUNNER_UID }}
+        RUNNER_GID: ${{ env.RUNNER_GID }}
+        COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
         PYTEST_ARGS: "--cov=karapace --cov-append --random-order --numprocesses 4"
 
     - name: Archive logs
@@ -56,6 +102,7 @@ jobs:
       with:
         name: karapace-integration-test-logs-${{ matrix.python-version }}
         path: /tmp/ci-logs
+
     - name: Archive coverage file
       uses: actions/upload-artifact@v4
       with:

.gitignore

@@ -10,13 +10,15 @@
 __pycache__/
 /build/
 /dist/
-/karapace.egg-info/
+src/karapace.egg-info/
 /karapace-rpm-src.tar
 /kafka_*.tgz
 /kafka_*/
 venv
-/karapace/version.py
+*.so
+src/karapace/version.py
 .run
 .python-version
 .hypothesis/
 .DS_Store
+*.coverage.*

.pre-commit-config.yaml

@@ -30,36 +30,6 @@ repos:
   hooks:
   - id: shfmt
 
-- repo: https://github.com/asottile/pyupgrade
-  rev: v3.4.0
-  hooks:
-  - id: pyupgrade
-    args: [ --py39-plus ]
-
-- repo: https://github.com/pycqa/autoflake
-  rev: v2.1.1
-  hooks:
-    - id: autoflake
-      args:
-        - --in-place
-        - --remove-all-unused-imports
-        - --ignore-init-module-imports
-
-- repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
-  hooks:
-  - id: isort
-
-- repo: https://github.com/psf/black
-  rev: 23.3.0
-  hooks:
-  - id: black
-
-- repo: https://github.com/PyCQA/flake8
-  rev: 7.1.0
-  hooks:
-  - id: flake8
-
 - repo: https://github.com/hadolint/hadolint
   rev: v2.12.0
   hooks:
@@ -70,9 +40,12 @@ repos:
     # https://github.com/hadolint/hadolint/issues/497
     - --ignore=DL3042
 
-- repo: https://github.com/PyCQA/pylint
-  # Note: pre-commit autoupdate changes to an alpha version. Instead, manually find the
-  # latest stable version here: https://github.com/pylint-dev/pylint/releases
-  rev: v3.2.6
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.8.6
   hooks:
-  - id: pylint
+    # Run the linter.
+    - id: ruff
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format