Cache venv (#1)

* Cache venv * Cache venv * Fix test * Fix checksum step * Don't have trailing hyphen if no dependency specified * Fix syntax * Fix syntax for real * Try quotes * Try echoing * split up steps * Fix * Try single quote * Use yaml thing that removes newlines * Use shasum * Use Python * Fix output syntax * Missing quote * Sort and formatting * Bump setup-python version --------- Co-authored-by: Jay Qi <[email protected]>
drivendataorg · Feb 27, 2024 · 6f4bcb8 · 6f4bcb8
1 parent a02d47f
commit 6f4bcb8
Show file tree

Hide file tree

Showing 4 changed files with 248 additions and 30 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -0,0 +1,179 @@
+name: benchmarks
+
+on: 
+  pull_request:
+  push: 
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  tests-cache-packages:
+    name: Test uv, cache packages (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.12"]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python with uv (local source)
+        uses: ./
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: packages
+          cache-dependency-path: >-
+            test_assets/requirements.txt
+
+      - name: Check active Python
+        run: |
+          echo "Python: $(which python)"
+          echo "Python version: $(python --version)"
+
+      - name: Install dependencies
+        run: |
+          uv pip install -r test_assets/requirements.txt
+
+      - name: Test some stuff
+        run: |
+          python -c "import cloudpathlib; print('cloudpathlib:', cloudpathlib.__version__)"
+          nbautoexport --version
+          mkdocs --version
+          pytest --version
+
+  tests-cache-venv:
+    name: Test uv, cache venv (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.12"]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python with uv (local source)
+        uses: ./
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: venv
+          cache-dependency-path: >-
+            test_assets/requirements.txt
+
+      - name: Check active Python
+        run: |
+          echo "Python: $(which python)"
+          echo "Python version: $(python --version)"
+
+      - name: Install dependencies
+        run: |
+          uv pip install -r test_assets/requirements.txt
+
+      - name: Test some stuff
+        run: |
+          python -c "import cloudpathlib; print('cloudpathlib:', cloudpathlib.__version__)"
+          nbautoexport --version
+          mkdocs --version
+          pytest --version
+          
+  tests-no-cache:
+    name: Test uv no cache (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.12"]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python with uv (local source)
+        uses: ./
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Check active Python
+        run: |
+          echo "Python: $(which python)"
+          echo "Python version: $(python --version)"
+
+      - name: Install dependencies
+        run: |
+          uv pip install -r test_assets/requirements.txt
+
+      - name: Test some stuff
+        run: |
+          python -c "import cloudpathlib; print('cloudpathlib:', cloudpathlib.__version__)"
+          nbautoexport --version
+          mkdocs --version
+          pytest --version
+
+  tests-pip-cache:
+    name: Test pip, cache (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.12"]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: >-
+            test_assets/requirements.txt
+
+      - name: Install dependencies
+        run: |
+          pip install -r test_assets/requirements.txt
+
+      - name: Test some stuff
+        run: |
+          python -c "import cloudpathlib; print('cloudpathlib:', cloudpathlib.__version__)"
+          nbautoexport --version
+          mkdocs --version
+          pytest --version
+        
+
+  tests-pip-no-cache:
+    name: Test pip, no cache (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.12"]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install -r test_assets/requirements.txt
+
+      - name: Test some stuff
+        run: |
+          python -c "import cloudpathlib; print('cloudpathlib:', cloudpathlib.__version__)"
+          nbautoexport --version
+          mkdocs --version
+          pytest --version
+        
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -7,8 +7,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  tests:
-    name: Test uv with cache (${{ matrix.os }}, Python ${{ matrix.python-version }})
+  tests-cache-packages:
+    name: Test uv, cache packages (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -24,7 +24,8 @@ jobs:
         uses: ./
         with:
           python-version: ${{ matrix.python-version }}
-          cache-dependency-path: |
+          cache: packages
+          cache-dependency-path: >-
             test_assets/requirements.txt
 
       - name: Check active Python
@@ -43,8 +44,8 @@ jobs:
           mkdocs --version
           pytest --version
 
-  tests-no-cache:
-    name: Test uv no cache (${{ matrix.os }}, Python ${{ matrix.python-version }})
+  tests-cache-venv:
+    name: Test uv, cache packages (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -60,7 +61,9 @@ jobs:
         uses: ./
         with:
           python-version: ${{ matrix.python-version }}
-          cache: false
+          cache: packages
+          cache-dependency-path: >-
+            test_assets/requirements.txt
 
       - name: Check active Python
         run: |
@@ -77,9 +80,10 @@ jobs:
           nbautoexport --version
           mkdocs --version
           pytest --version
+          
 
-  tests-pip:
-    name: Test pip with cache (${{ matrix.os }}, Python ${{ matrix.python-version }})
+  tests-no-cache:
+    name: Test uv no cache (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -91,22 +95,23 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Set up Python with uv (local source)
+        uses: ./
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: |
-            test_assets/requirements.txt
+
+      - name: Check active Python
+        run: |
+          echo "Python: $(which python)"
+          echo "Python version: $(python --version)"
 
       - name: Install dependencies
         run: |
-          pip install -r test_assets/requirements.txt
+          uv pip install -r test_assets/requirements.txt
 
       - name: Test some stuff
         run: |
           python -c "import cloudpathlib; print('cloudpathlib:', cloudpathlib.__version__)"
           nbautoexport --version
           mkdocs --version
           pytest --version
-        
diff --git a/action.yml b/action.yml
@@ -4,8 +4,7 @@ inputs:
   python-version:
     description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax. Reads from .python-version if unset. Passed to setup-python."
   cache:
-    description: "Whether to cache uv's cache directory. Default is true."
-    default: "true"
+    description: "If set to 'packages', will cache uv's cache directory. If set to 'venv', will cache the virtual environment."
   cache-dependency-path:
     description: "Specify dependency files hash for cache key. Supports wildcards or a list of file names."
 outputs:
@@ -30,31 +29,44 @@ runs:
       with:
         python-version: ${{ inputs.python-version }}
 
-    - name: Install uv
-      id: install-uv
+    - name: Set up uv
+      id: setup-uv
       shell: bash
       run: |
         pipx install uv
         echo "uv-path=$(which uv)" >> $GITHUB_OUTPUT
         echo "uv-cache-dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+        echo "venv-dir=$HOME/.venv" >> $GITHUB_OUTPUT
 
     - name: Generate checksum of cache dependencies
       id: cache-dependency-checksum
-      if: ${{ inputs.cache == 'true' && inputs.cache-dependency-path }}
+      if: ${{ (inputs.cache == 'packages' || inputs.cache == 'venv') && inputs.cache-dependency-path }}
       shell: bash
       run: |
-        echo "checksum=$(md5sum ${{ inputs.cache-dependency-path }}) | md5sum | awk '{ print $1 }')" >> $GITHUB_OUTPUT
+        echo "checksum-suffix=-$(python checksum.py ${{ inputs.cache-dependency-path }})" >> $GITHUB_OUTPUT
 
-    - name: Set up uv cache
-      id: setup-uv-cache
+    - name: Set up package cache
+      id: setup-package-cache
       uses: actions/cache@v4
-      if: ${{ inputs.cache == 'true' }}
+      if: ${{ inputs.cache == 'packages' }}
       with:
-        path: ${{ steps.install-uv.outputs.uv-cache-dir }}
-        key: setup-python-uv-action-${{ runner.os }}-${{ steps.cache-dependency-checksum.outputs.checksum }}
+        path: |
+          ${{ steps.setup-uv.outputs.uv-cache-dir }}
+        key: setup-python-uv-action-packages-${{ runner.os }}${{ steps.cache-dependency-checksum.outputs.checksum-suffix }}
+        save-always: true
+
+    - name: Set up venv cache
+      id: setup-venv-cache
+      uses: actions/cache@v4
+      if: ${{ inputs.cache == 'venv' }}
+      with:
+        path: |
+          ${{ steps.setup-uv.outputs.venv-dir }}
+        key: setup-python-uv-action-venv-${{ runner.os }}-python-${{ inputs.python-version }}${{ steps.cache-dependency-checksum.outputs.checksum-suffix }}
         save-always: true
 
     - name: Create virtual environment
+      if: ${{ !(inputs.cache == 'venv' && steps.setup-venv-cache.outputs.cache-hit == 'true') }}
       shell: bash
       run: |
         uv venv $HOME/.venv
@@ -63,12 +75,12 @@ runs:
       if: ${{ runner.os != 'Windows' }}
       shell: bash
       run: |
-        echo "$HOME/.venv/bin" >> $GITHUB_PATH
-        echo "VIRTUAL_ENV=$HOME/.venv" >> $GITHUB_ENV
+        echo "${{ steps.setup-uv.outputs.venv-dir }}/bin" >> $GITHUB_PATH
+        echo "VIRTUAL_ENV=${{ steps.setup-uv.outputs.venv-dir }}" >> $GITHUB_ENV
 
     - name: Activate virtual environment (Windows)
       if: ${{ runner.os == 'Windows' }}
       shell: bash
       run: |
-        echo "$HOME/.venv/Scripts" >> $GITHUB_PATH
-        echo "VIRTUAL_ENV=$HOME/.venv" >> $GITHUB_ENV
+        echo "${{ steps.setup-uv.outputs.venv-dir }}/Scripts" >> $GITHUB_PATH
+        echo "VIRTUAL_ENV=${{ steps.setup-uv.outputs.venv-dir }}" >> $GITHUB_ENV
diff --git a/checksum.py b/checksum.py
@@ -0,0 +1,22 @@
+import hashlib
+from pathlib import Path
+import sys
+import textwrap
+
+def main():
+    dep_checksums = []
+    for dep in sorted(sys.argv[1:]):
+        # In general, dep is a glob
+        for path in sorted(Path().glob(dep)):
+            dep_checksums.append((path, hashlib.md5(path.read_bytes()).hexdigest()))
+    dep_checksums_str = textwrap.indent(
+        "\n".join([f"{path} {checksum}" for path, checksum in dep_checksums]),
+        "    "
+    )
+    sys.stderr.write(f"Dependency checksums:\n{dep_checksums_str}\n")
+    overall_checksum = hashlib.md5(dep_checksums_str.encode()).hexdigest()
+    sys.stderr.write(f"Overall checksum: {overall_checksum}\n")
+    sys.stdout.write(overall_checksum)
+
+if __name__ == "__main__":
+    main()