feat: Add initial files with the etl uvicorn cli (#1)

.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: make install-lint
+
+      - name: Run check-python
+        run: make check-python
+
+  shfmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: setup shfmt
+        uses: mfinelli/setup-shfmt@v3
+      - name: Run shfmt
+        run: shfmt -d .

.gitignore

@@ -17,6 +17,7 @@ eggs/
 lib/
 lib64/
 parts/
+rabbitmq_data/
 sdist/
 var/
 wheels/
@@ -25,6 +26,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+.python_packages/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -106,10 +108,8 @@ ipython_config.py
 #pdm.lock
 #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
 #   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+#   https://pdm.fming.dev/#use-with-ide
 .pdm.toml
-.pdm-python
-.pdm-build/
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
@@ -159,4 +159,10 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+
+.DS_Store
+
+# Visual studio temps
+.vs/
+.vscode/

Makefile

@@ -0,0 +1,93 @@
+PACKAGE_NAME := platform
+PROJECT_NAME := api
+PROJECT_PATH := platform_api
+ARCH=amd64
+GIT_HASH := $(shell git rev-parse --short HEAD)
+GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
+VERSION_SCRIPTS_PATH=scripts
+IMAGE_REPOSITORY := "${PACKAGE_NAME}-${PROJECT_NAME}"
+IMAGE_REGISTRY=uticplatform.azurecr.io
+
+###########
+# INSTALL #
+###########
+
+.PHONY: pip-compile
+pip-compile:
+	./scripts/pip-compile.sh
+
+.PHONY: install
+install: install-base install-lint install-test
+
+.PHONY: install-base
+install-base:
+	pip install -r requirements/base.txt
+
+.PHONY: install-lint
+install-lint:
+	pip install -r requirements/lint.txt
+
+.PHONY: install-test
+install-test:
+	pip install -r requirements/test.txt
+
+###########
+#  TIDY   #
+###########
+
+.PHONY: tidy
+tidy: tidy-black tidy-ruff tidy-autoflake tidy-shell
+
+.PHONY: tidy_shell
+tidy-shell:
+	shfmt -l -w .
+
+.PHONY: tidy-ruff
+tidy-ruff:
+	ruff check --fix-only --show-fixes .
+
+.PHONY: tidy-black
+tidy-black:
+	black .
+
+.PHONY: tidy-autoflake
+tidy-autoflake:
+	autoflake --in-place .
+
+###########
+#  CHECK  #
+###########
+
+.PHONY: check
+check: check-python check-shell
+
+.PHONY: check-python
+check-python: check-black check-flake8 check-ruff check-autoflake
+
+.PHONY: check-black
+check-black:
+	black . --check
+
+.PHONY: check-flake8
+check-flake8:
+	flake8 .
+
+.PHONY: check-ruff
+check-ruff:
+	ruff check .
+
+.PHONY: check-autoflake
+check-autoflake:
+	autoflake --check-diff .
+
+.PHONY: check-shell
+check-shell:
+	shfmt -d .
+
+###########
+#  TEST   #
+###########
+
+.PHONY: test
+test:
+	PYTHONPATH=. pytest

README.md

@@ -1 +1,47 @@
-# unstructured-platform-plugins
+# Unstructured Platform Plugins
+ ![CI](https://github.com/Unstructured-IO/unstructured-enterprise/actions/workflows/ci.yml/badge.svg?branch=main)
+
+Information about how to build custom plugins to integrate with Unstructured Platform.
+
+## Plugin Development
+Any plugin must be published in a dedicated docker image with all required dependencies that when run, exposes an api 
+on port 8000 with the required endpoints to interact with the Unstructured Platform product:
+* `/invoke`: A `POST` endpoint which gets all data to run the underlying logic in the request body and expects a json serializable response. 
+* `/schema`: A `GET` endpoint which publishes a json schema formatted response with the schema of the input and output expected by the plugin.
+* `/id`: A `GET` endpoint which publishes a string unique identifier for this instance of the plugin. Will default to a hash of the schema 
+response if one is not set explicitly.
+
+
+## Utility CLI
+When installing this repo, it also installs the cli `etl-uvicorn`. This takes a pointer to any generic python 
+function and wraps it in a FastApi application to conform to the patterns that are expected by the api hosting the 
+plugin logic. This cli extends the existing `uvicorn` cli which takes in a pointer to a fastapi instance or factory but 
+instead takes in a pointer to a python function/class which gets wrapped with a FastApi application. 
+
+## Example usage
+Wrapping a basic function
+```shell
+etl-uvicorn unstructured_platform_plugins.etl_uvicorn.example:sample_function
+```
+
+Wrapping a basic async function
+```shell
+etl-uvicorn unstructured_platform_plugins.etl_uvicorn.example:async_sample_function
+```
+
+Wrapping a class. For this to work, the class must be self instantiating. When passing a class in, a method needs to 
+be passed in as well, otherwise `__call__` is used.
+```shell
+etl-uvicorn unstructured_platform_plugins.etl_uvicorn.example:SampleClass --method sample_method
+```
+
+Wrapping an instance of a class.
+```shell
+etl-uvicorn unstructured_platform_plugins.etl_uvicorn.example:sample_class --method sample_method
+```
+
+The CLI does some validation on the wrapped function, which must have explicit inputs and outputs, meaning *args 
+and **kwargs are not supported. These will cause the cli to fail fast.
+```shell
+etl-uvicorn unstructured_platform_plugins.etl_uvicorn.example:sample_function
+```

pyproject.toml

@@ -0,0 +1,60 @@
+[tool.black]
+line-length = 100
+extend_exclude = """
+(
+    .venv
+    | venv
+    | .python_packages
+    | build
+    | migrations
+)
+"""
+
+
+[tool.ruff]
+line-length = 100
+exclude= [
+    ".venv",
+    "venv",
+    ".python_packages",
+    "build",
+    "migrations"
+]
+
+[tool.ruff.lint]
+select = [
+    "C4",       # -- flake8-comprehensions --
+    "COM",      # -- flake8-commas --
+    "E",        # -- pycodestyle errors --
+    "F",        # -- pyflakes --
+    "I",        # -- isort (imports) --
+    "PLR0402",  # -- Name compared with itself like `foo == foo` --
+    "PT",       # -- flake8-pytest-style --
+    "SIM",      # -- flake8-simplify --
+    "UP015",    # -- redundant `open()` mode parameter (like "r" is default) --
+    "UP018",    # -- Unnecessary {literal_type} call like `str("abc")`. (rewrite as a literal) --
+    "UP032",    # -- Use f-string instead of `.format()` call --
+    "UP034",    # -- Avoid extraneous parentheses --
+]
+ignore = [
+    "COM812",   # -- over aggressively insists on trailing commas where not desireable --
+    "PT005",    # -- flags mock fixtures with names intentionally matching private method name --
+    "PT011",    # -- pytest.raises({exc}) too broad, use match param or more specific exception --
+    "PT012",    # -- pytest.raises() block should contain a single simple statement --
+    "SIM117",   # -- merge `with` statements for context managers that have same scope --
+]
+
+[tool.autoflake]
+expand_star_imports=true
+ignore_pass_statements=false
+recursive=true
+quiet=true
+remove_all_unused_imports=true
+remove_unused_variables=true
+exclude= [
+    ".venv",
+    "venv",
+    ".python_packages",
+    "build",
+    "migrations"
+]

requirements/cli.in

@@ -0,0 +1,3 @@
+uvicorn
+fastapi
+click

requirements/cli.txt

@@ -0,0 +1,97 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements//cli.in
+#
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.4.0
+    # via
+    #   httpx
+    #   starlette
+    #   watchfiles
+certifi==2024.6.2
+    # via
+    #   httpcore
+    #   httpx
+click==8.1.7
+    # via
+    #   -r requirements//cli.in
+    #   typer
+    #   uvicorn
+dnspython==2.6.1
+    # via email-validator
+email-validator==2.1.1
+    # via fastapi
+fastapi==0.111.0
+    # via -r requirements//cli.in
+fastapi-cli==0.0.4
+    # via fastapi
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+httpcore==1.0.5
+    # via httpx
+httptools==0.6.1
+    # via uvicorn
+httpx==0.27.0
+    # via fastapi
+idna==3.7
+    # via
+    #   anyio
+    #   email-validator
+    #   httpx
+jinja2==3.1.4
+    # via fastapi
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+orjson==3.10.3
+    # via fastapi
+pydantic==2.7.3
+    # via fastapi
+pydantic-core==2.18.4
+    # via pydantic
+pygments==2.18.0
+    # via rich
+python-dotenv==1.0.1
+    # via uvicorn
+python-multipart==0.0.9
+    # via fastapi
+pyyaml==6.0.1
+    # via uvicorn
+rich==13.7.1
+    # via typer
+shellingham==1.5.4
+    # via typer
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+starlette==0.37.2
+    # via fastapi
+typer==0.12.3
+    # via fastapi-cli
+typing-extensions==4.12.1
+    # via
+    #   fastapi
+    #   pydantic
+    #   pydantic-core
+    #   typer
+ujson==5.10.0
+    # via fastapi
+uvicorn[standard]==0.30.1
+    # via
+    #   -r requirements//cli.in
+    #   fastapi
+uvloop==0.19.0
+    # via uvicorn
+watchfiles==0.22.0
+    # via uvicorn
+websockets==12.0
+    # via uvicorn

requirements/lint.in

@@ -0,0 +1,7 @@
+-c ./constraints.txt
+black
+flake8
+mypy
+ruff
+autoflake
+flake8-print