Benchmark: include benchmark suite and automation for multiple os

.github/workflows/bench.yml

@@ -0,0 +1,50 @@
+on: [ push, pull_request ]
+
+name: Benchmark
+
+jobs:
+  benchmark:
+    name: Run benchmarks
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macOS-latest]
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - uses: actions/cache@v2
+        with:
+          path: .venv
+          key: poetry-${{ hashFiles('poetry.lock') }}      
+
+      - name: Download benchmark artifacts
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: bench.yml
+          name: benchmarks-${{ matrix.os }}
+          path: .benchmarks
+        continue-on-error: true
+
+      - name: Run pytest-benchmark
+        run: |
+          . scripts/ci_install_deps
+          poetry run pytest benchmarks \
+            --benchmark-compare
+            --benchmark-save=main-${{ matrix.os }} \
+            --benchmark-warmup=on
+
+      
+      - name: Archive benchmark results
+        if: ${{ github.ref == 'refs/heads/main'}}
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmarks-${{ matrix.os }}
+          path: .benchmarks

.gitignore

@@ -188,3 +188,5 @@ fabric.properties
 
 
 node_modules/
+
+.benchmarks

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "quillang"]
 	path = docs/quil
 	url = https://github.com/rigetti/quil.git
+[submodule "benchmarks/quilc"]
+	path = benchmarks/quilc
+	url = https://github.com/quil-lang/quilc.git

benchmarks/test_parser.py

@@ -0,0 +1,29 @@
+import pytest
+import os
+from pyquil._parser.parser import run_parser
+from pyquil.quil import Program
+
+
+def from_corpus():
+    programs = []
+    dir = os.path.join(os.path.dirname(__file__), "quilc", "tests", "good-test-files")
+    for path in os.listdir(dir):
+        filepath = os.path.join(dir, path)
+        if os.path.isfile(filepath):
+            file = open(filepath, "r")
+            program = file.read()
+            try:
+                run_parser(program)
+                programs.append((path, program))
+            except:
+                continue
+            finally:
+                file.close()
+
+    return programs
+
+
+@pytest.mark.parametrize("quil", from_corpus())
+def test_parser(benchmark, quil):
+    benchmark.group = "Parse: %s" % quil[0]
+    benchmark((lambda: Program(run_parser(quil[1]))))

pyproject.toml

@@ -52,6 +52,7 @@ pytest-timeout = "^1.4.2"
 pytest-mock = "^3.6.1"
 pytest-freezegun = "^0.4.2"
 respx = "^0.15"
+pytest-benchmark = "^3.4.1"
 
 [tool.poetry.extras]
 latex = ["ipython"]