chore: start adding missing end to end tests (#138)

*Sets the pattern for the comprehensive unittests in Issue #104 . * Adds a pipeline test project with a root directory of test_unstructured_api_tools/pipeline-test-project/ * Generates web API's test_unstructured_api_tools/pipeline-test-project/prepline_test_project/api/ from notebooks under test_unstructured_api_tools/pipeline-test-project/pipeline-notebook with make generate-test-api * Ensures the generated API's match expected in CI with make api-check-test * Adds FastAPI client tests against the generated API's in test_unstructured_api_tools/api/test_file_apis.py There are still many notebooks to add under test_unstructured_api_tools/pipeline-test-project/pipeline-notebook and many more permutations to test per #104 , but this is establishing the pattern.
Unstructured-IO · Mar 6, 2023 · 59b52c1 · 59b52c1
1 parent feef1f6
commit 59b52c1
Show file tree

Hide file tree

Showing 18 changed files with 751 additions and 38 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -56,11 +56,27 @@ jobs:
       - name: ShellCheck
         uses: ludeeus/action-shellcheck@master
 
-  test:
+  test_api_consistency:
     runs-on: ubuntu-latest
     needs: [setup, lint]
     steps:
     - uses: actions/checkout@v3
+    - uses: actions/cache@v3
+      id: virtualenv-cache
+      with:
+        path: |
+          .venv
+        key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements/*.txt') }}
+    - name: API Consistency
+      run: |
+        source .venv/bin/activate
+        make api-check-test
+
+  test:
+    runs-on: ubuntu-latest
+    needs: test_api_consistency
+    steps:
+    - uses: actions/checkout@v3
     - uses: actions/cache@v3
       id: virtualenv-cache
       with:

diff --git a/.gitignore b/.gitignore
@@ -129,3 +129,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# API test project test files
+/test_unstructured_api_tools/pipeline-test-project/tmp*
diff --git a/Makefile b/Makefile
@@ -11,7 +11,7 @@ help: Makefile
 # Install #
 ###########
 
-## install:                 installs all base, test, and dev requirements
+## install:                    installs all base, test, and dev requirements
 .PHONY: install
 install: install-base install-dev install-test
 
@@ -31,20 +31,20 @@ install-test:
 install-dev:
 	pip install -r requirements/dev.txt
 
-## pip-compile:             compiles all base/dev/test requirements
+## pip-compile:                compiles all base/dev/test requirements
 .PHONY: pip-compile
 pip-compile:
 	pip-compile --upgrade -o requirements/base.txt
 	pip-compile --upgrade requirements/dev.in
 	pip-compile --upgrade requirements/test.in
 
-## install-project-local:   install unstructured_api_tools into your local python environment
+## install-project-local:      install unstructured_api_tools into your local python environment
 .PHONY: install-project-local
 install-project-local: install
 	# MAYBE TODO: fail if already exists?
 	pip install -e .
 
-## uninstall-project-local: uninstall unstructured_api_tools from your local python environment
+## uninstall-project-local:    uninstall unstructured_api_tools from your local python environment
 .PHONY: uninstall-project-local
 uninstall-project-local:
 	pip uninstall ${PACKAGE_NAME}
@@ -53,16 +53,42 @@ uninstall-project-local:
 # Test and Lint #
 #################
 
-## test:                    runs all unittests
+## run-jupyter-test-notebooks: starts jupyter, allows execution of test notebooks
+.PHONY: run-jupyter-test-notebooks
+run-jupyter-test-notebooks:
+	PYTHONPATH=$(realpath .)/test_unstructured_api_tools/pipeline-test-project/ JUPYTER_PATH=$(realpath .)/test_unstructured_api_tools/pipeline-test-project/ jupyter-notebook --NotebookApp.token='' --NotebookApp.password=''
+
+## tidy-test-notebooks:        execute notebooks and remove metadata
+.PHONY: tidy-test-notebooks
+tidy-test-notebooks:
+	PYTHONPATH=. ./test_unstructured_api_tools/pipeline-test-project/scripts/check-and-format-notebooks.py
+
+## generate-test-api:          generates FastAPIs under ./test_unstructured_api_tools/pipeline-test-project
+.PHONY: generate-test-api
+generate-test-api:
+	# generates FastAPI API's from notebooks in the test project ./test_unstructured_api_tools/pipeline-test-project
+	PYTHONPATH=. PIPELINE_FAMILY_CONFIG=test_unstructured_api_tools/pipeline-test-project/preprocessing-pipeline-family.yaml \
+		python3 ./unstructured_api_tools/cli.py convert-pipeline-notebooks \
+		--input-directory ./test_unstructured_api_tools/pipeline-test-project/pipeline-notebooks \
+		--output-directory ./test_unstructured_api_tools/pipeline-test-project/prepline_test_project/api
+
+
+## api-check-test:             verifies auto-generated pipeline APIs match the existing ones
+.PHONY: api-check-test
+api-check-test:
+	PYTHONPATH=. PACKAGE_NAME=prepline_test_project ./test_unstructured_api_tools/pipeline-test-project/scripts/test-doc-pipeline-apis-consistent.sh
+
+
+## test:                       runs all unittests
 .PHONY: test
 test:
-	PYTHONPATH=. pytest test_${PACKAGE_NAME} --cov=${PACKAGE_NAME} --cov-report term-missing -vvv
+	PYTHONPATH=.:./test_unstructured_api_tools/pipeline-test-project pytest test_${PACKAGE_NAME} --cov=${PACKAGE_NAME} --cov-report term-missing -vvv
 
-## check:                   runs linters (includes tests)
+## check:                      runs linters (includes tests)
 .PHONY: check
 check: check-src check-tests check-version
 
-## check-src:               runs linters (source only, no tests)
+## check-src:                  runs linters (source only, no tests)
 .PHONY: check-src
 check-src:
 	black --line-length 100 ${PACKAGE_NAME} --check
@@ -71,32 +97,33 @@ check-src:
 
 .PHONY: check-tests
 check-tests:
-	black --line-length 100 test_${PACKAGE_NAME} --check
-	flake8 test_${PACKAGE_NAME}
+	black --line-length 100 test_${PACKAGE_NAME} --check --exclude test_${PACKAGE_NAME}/pipeline-test-project
+	flake8 test_${PACKAGE_NAME} --exclude test_${PACKAGE_NAME}/pipeline-test-project/prepline_test_project/api
 
-## check-scripts:           run shellcheck
+## check-scripts:              run shellcheck
 .PHONY: check-scripts
 check-scripts:
     # Fail if any of these files have warnings
 	scripts/shellcheck.sh
 
-## check-version:           run check to ensure version in CHANGELOG.md matches version in package
+## check-version:              run check to ensure version in CHANGELOG.md matches version in package
 .PHONY: check-version
 check-version:
     # Fail if syncing version would produce changes
 	scripts/version-sync.sh -c -d ${PACKAGE_NAME}
 
-## tidy:                    run black
+## tidy:                       run black
 .PHONY: tidy
 tidy:
 	black --line-length 100 ${PACKAGE_NAME}
 	black --line-length 100 test_${PACKAGE_NAME}
 
-## version-sync:            update __version__.py with most recent version from CHANGELOG.md
+## version-sync:               update __version__.py with most recent version from CHANGELOG.md
 .PHONY: version-sync
 version-sync:
 	scripts/version-sync.sh -d ${PACKAGE_NAME}
 
 .PHONY: check-coverage
 check-coverage:
+	# TODO(crag): add coverage check for test_unstructured_api_tools/pipeline-test-project/prepline_test_project/api/
 	coverage report --fail-under=95
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -22,9 +22,9 @@ defusedxml==0.7.1
     # via nbconvert
 deprecated==1.2.13
     # via limits
-fastapi==0.91.0
+fastapi==0.92.0
     # via unstructured-api-tools (setup.py)
-fastjsonschema==2.16.2
+fastjsonschema==2.16.3
     # via nbformat
 h11==0.14.0
     # via uvicorn
@@ -36,15 +36,15 @@ importlib-metadata==6.0.0
     # via
     #   jupyter-client
     #   nbconvert
-importlib-resources==5.10.2
+importlib-resources==5.12.0
     # via jsonschema
 jinja2==3.1.2
     # via
     #   nbconvert
     #   unstructured-api-tools (setup.py)
 jsonschema==4.17.3
     # via nbformat
-jupyter-client==8.0.2
+jupyter-client==8.0.3
     # via nbclient
 jupyter-core==5.2.0
     # via
@@ -62,7 +62,7 @@ markupsafe==2.1.2
     #   nbconvert
 mistune==2.0.5
     # via nbconvert
-mypy==1.0.0
+mypy==1.0.1
     # via unstructured-api-tools (setup.py)
 mypy-extensions==1.0.0
     # via mypy
@@ -84,17 +84,17 @@ pkgutil-resolve-name==1.3.10
     # via jsonschema
 platformdirs==3.0.0
     # via jupyter-core
-pydantic==1.10.4
+pydantic==1.10.5
     # via fastapi
 pygments==2.14.0
     # via nbconvert
 pyrsistent==0.19.3
     # via jsonschema
 python-dateutil==2.8.2
     # via jupyter-client
-python-dotenv==0.21.1
+python-dotenv==1.0.0
     # via uvicorn
-python-multipart==0.0.5
+python-multipart==0.0.6
     # via unstructured-api-tools (setup.py)
 pyyaml==6.0
     # via uvicorn
@@ -104,14 +104,13 @@ six==1.16.0
     # via
     #   bleach
     #   python-dateutil
-    #   python-multipart
 slowapi==0.1.7
     # via unstructured-api-tools (setup.py)
 sniffio==1.3.0
     # via anyio
-soupsieve==2.3.2.post1
+soupsieve==2.4
     # via beautifulsoup4
-starlette==0.24.0
+starlette==0.25.0
     # via fastapi
 tinycss2==1.2.1
     # via nbconvert
@@ -126,13 +125,13 @@ traitlets==5.9.0
     #   nbclient
     #   nbconvert
     #   nbformat
-types-requests==2.28.11.12
+types-requests==2.28.11.15
     # via unstructured-api-tools (setup.py)
-types-ujson==5.7.0.0
+types-ujson==5.7.0.1
     # via unstructured-api-tools (setup.py)
-types-urllib3==1.26.25.5
+types-urllib3==1.26.25.8
     # via types-requests
-typing-extensions==4.4.0
+typing-extensions==4.5.0
     # via
     #   limits
     #   mypy
@@ -150,9 +149,9 @@ webencodings==0.5.1
     #   tinycss2
 websockets==10.4
     # via uvicorn
-wrapt==1.14.1
+wrapt==1.15.0
     # via deprecated
-zipp==3.13.0
+zipp==3.15.0
     # via
     #   importlib-metadata
     #   importlib-resources

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -32,9 +32,9 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-pip-tools==6.12.2
+pip-tools==6.12.3
     # via -r requirements/dev.in
-prompt-toolkit==3.0.36
+prompt-toolkit==3.0.38
     # via ipython
 ptyprocess==0.7.0
     # via pexpect

diff --git a/requirements/test.in b/requirements/test.in
@@ -5,6 +5,9 @@ coverage
 # ref: https://github.com/psf/black/issues/2964
 click>=8.1
 flake8
+httpx
 pytest-cov
 # NOTE(mrobinson) - requests is needed for the fastapi test client
 requests
+requests_toolbelt
+nbdev