diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..a9980ce --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,49 @@ +name: test + +on: + push: + branches: + - '*' + pull_request: + branches: + - '*' + +jobs: + build-test-functions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: 0 + - name: Get faas-cli + run: curl -sLSf https://cli.openfaas.com | sudo sh + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ^1.15 + - name: Setup git-semver + run: GO111MODULE=on go get github.com/mdomke/git-semver/v6@v6.0.1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Get TAG + id: get_tag + run: echo ::set-output name=tag::$(git-semver | tr '+' '.') + - name: Get Repo Owner + id: get_repo_owner + run: > + echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} | + tr '[:upper:]' '[:lower:]') + - name: Debug variables + run: | + echo "repo_owner: ${{ steps.get_repo_owner.outputs.repo_owner }}" + echo "tag: ${{ steps.get_tag.outputs.tag }}" + echo "repo: ${{ github.repository }}" + echo "git-describe: $(git describe --tags --always)" + - name: Publish functions + run: > + DOCKER_BUILDKIT=1 + OWNER="${{ steps.get_repo_owner.outputs.repo_owner }}" + TAG="${{ steps.get_tag.outputs.tag }}" + faas-cli build + --parallel 2 + --disable-stack-pull diff --git a/pydatascience-test/__init__.py b/pydatascience-test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydatascience-test/core/README.md b/pydatascience-test/core/README.md new file mode 100644 index 0000000..98b6f65 --- /dev/null +++ b/pydatascience-test/core/README.md @@ -0,0 +1,3 @@ +Put common utilities and generalized logic here + +A basic implementation for reading mounted secrets is provided, as an example. diff --git a/pydatascience-test/core/__init__.py b/pydatascience-test/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydatascience-test/core/utils.py b/pydatascience-test/core/utils.py new file mode 100644 index 0000000..4f86c8e --- /dev/null +++ b/pydatascience-test/core/utils.py @@ -0,0 +1,9 @@ + + +def getSecret(secret_name: str) -> str: + """load secret value from the openfaas secret folder + Args: + secret_name (str): name of the secret + """ + with open("/var/openfaas/secrets/" + secret_name, "r") as file: + return file.read() diff --git a/pydatascience-test/handler.py b/pydatascience-test/handler.py new file mode 100644 index 0000000..dc4f305 --- /dev/null +++ b/pydatascience-test/handler.py @@ -0,0 +1,24 @@ +import json +import os + +import numpy as np + +# from .core import utils + + +function_root = os.environ.get("function_root") + +# Now pre-load the model, e.g. +# from .core import model + + +def handle(req: bytes) -> str: + """handle a request to the function + Args: + req (bytes): request body + """ + + return json.dumps({ + "echo": req, + "random": np.random.random_sample(), + }) diff --git a/pydatascience-test/handler_test.py b/pydatascience-test/handler_test.py new file mode 100644 index 0000000..0cccdd7 --- /dev/null +++ b/pydatascience-test/handler_test.py @@ -0,0 +1,14 @@ +import json +from .handler import handle + +# Test your handler here + +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +def test_handle(): + raw_output = handle("input") + output = json.loads(raw_output) + + assert output.get("echo") == "input" + assert 0.0 <= output.get("random") <= 1.0 \ No newline at end of file diff --git a/pydatascience-test/requirements.txt b/pydatascience-test/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/pydatascience-test/tox.ini b/pydatascience-test/tox.ini new file mode 100644 index 0000000..6877994 --- /dev/null +++ b/pydatascience-test/tox.ini @@ -0,0 +1,42 @@ +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +# Additionally, you can replace the content of this file with +# [tox] +# skipsdist = true + +# You can also edit, remove, or add additional test steps +# by editing, removing, or adding new testenv sections + + +# find out more about tox: https://tox.readthedocs.io/en/latest/ +[tox] +envlist = lint,test +skipsdist = true + +[testenv:test] +deps = + flask + pytest + -r ../requirements.txt + -rrequirements.txt +commands = + # run unit tests with pytest + # https://docs.pytest.org/en/stable/ + # configure by adding a pytest.ini to your handler + pytest + +[testenv:lint] +deps = + flake8 +commands = + flake8 . + +[flake8] +count = true +max-line-length = 127 +max-complexity = 10 +statistics = true +# stop the build if there are Python syntax errors or undefined names +select = E9,F63,F7,F82 +show-source = true \ No newline at end of file diff --git a/pydatascience-test/train.py b/pydatascience-test/train.py new file mode 100644 index 0000000..8f58cc1 --- /dev/null +++ b/pydatascience-test/train.py @@ -0,0 +1,3 @@ +""" +Put your model training here +""" diff --git a/pydatascience-web-test/__init__.py b/pydatascience-web-test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydatascience-web-test/core/README.md b/pydatascience-web-test/core/README.md new file mode 100644 index 0000000..98b6f65 --- /dev/null +++ b/pydatascience-web-test/core/README.md @@ -0,0 +1,3 @@ +Put common utilities and generalized logic here + +A basic implementation for reading mounted secrets is provided, as an example. diff --git a/pydatascience-web-test/core/__init__.py b/pydatascience-web-test/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydatascience-web-test/core/utils.py b/pydatascience-web-test/core/utils.py new file mode 100644 index 0000000..4f86c8e --- /dev/null +++ b/pydatascience-web-test/core/utils.py @@ -0,0 +1,9 @@ + + +def getSecret(secret_name: str) -> str: + """load secret value from the openfaas secret folder + Args: + secret_name (str): name of the secret + """ + with open("/var/openfaas/secrets/" + secret_name, "r") as file: + return file.read() diff --git a/pydatascience-web-test/handler.py b/pydatascience-web-test/handler.py new file mode 100644 index 0000000..41fde2e --- /dev/null +++ b/pydatascience-web-test/handler.py @@ -0,0 +1,34 @@ + +import json +import os + +from flask import Request +import numpy as np + +# from .core import utils + + +function_root = os.environ.get("function_root") + +# Now pre-load the model, e.g. +# from .core import model + + +def handle(req: Request): + """handle a request to the function. + + Your response is immediately passed to the caller, unmodified. + This allows you full control of the response, e.g. you can set + the status code by returning a tuple (str, int). A detailed + description of how responses are handled is found here: + + http://flask.pocoo.org/docs/1.0/quickstart/#about-responses + + Args: + req (Request): Flask request object + """ + + return json.dumps({ + "echo": req.get_data().decode('utf-8'), + "random": np.random.random_sample(), + }) diff --git a/pydatascience-web-test/handler_test.py b/pydatascience-web-test/handler_test.py new file mode 100644 index 0000000..4413e80 --- /dev/null +++ b/pydatascience-web-test/handler_test.py @@ -0,0 +1,21 @@ +import json +from .handler import handle + +import flask + +app = flask.Flask(__name__) + + +# Test your handler here + +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +def test_handle(): + with app.test_request_context('/', data="input"): + assert flask.request.path == '/' + raw_output = handle(flask.request) + output = json.loads(raw_output) + + assert output.get("echo") == "input" + assert 0.0 <= output.get("random") <= 1.0 \ No newline at end of file diff --git a/pydatascience-web-test/requirements.txt b/pydatascience-web-test/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/pydatascience-web-test/requirements_test.txt b/pydatascience-web-test/requirements_test.txt new file mode 100644 index 0000000..87c51f0 --- /dev/null +++ b/pydatascience-web-test/requirements_test.txt @@ -0,0 +1,3 @@ +tox==3.* +flake8 +pytest \ No newline at end of file diff --git a/pydatascience-web-test/tox.ini b/pydatascience-web-test/tox.ini new file mode 100644 index 0000000..c9d2139 --- /dev/null +++ b/pydatascience-web-test/tox.ini @@ -0,0 +1,42 @@ +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +# Additionally, you can replace the content of this file with +# [tox] +# skipsdist = true + +# You can also edit, remove, or add additional test steps +# by editing, removing, or adding new testenv sections + + +# find out more about tox: https://tox.readthedocs.io/en/latest/ +[tox] +envlist = lint,test +skipsdist = true + +[testenv:test] +deps = + flask + pytes + -r ../requirements.txt + -rrequirements.txt +commands = + # run unit tests with pytest + # https://docs.pytest.org/en/stable/ + # configure by adding a pytest.ini to your handler + pytest + +[testenv:lint] +deps = + flake8 +commands = + flake8 . + +[flake8] +count = true +max-line-length = 127 +max-complexity = 10 +statistics = true +# stop the build if there are Python syntax errors or undefined names +select = E9,F63,F7,F82 +show-source = true \ No newline at end of file diff --git a/pydatascience-web-test/train.py b/pydatascience-web-test/train.py new file mode 100644 index 0000000..8f58cc1 --- /dev/null +++ b/pydatascience-web-test/train.py @@ -0,0 +1,3 @@ +""" +Put your model training here +""" diff --git a/stack.yml b/stack.yml new file mode 100644 index 0000000..bb09ece --- /dev/null +++ b/stack.yml @@ -0,0 +1,15 @@ +version: 1.0 +provider: + name: openfaas + gateway: http://127.0.0.1:8080 + +functions: + pydatascience-test: + lang: pydatascience + handler: ./pydatascience-test + image: ghcr.io/${OWNER:-lucasroesler}/pydatascience-test:${TAG:-latest} + pydatascience-web-test: + lang: pydatascience-web + handler: ./pydatascience-web-test + image: pydatascience-web-test:latest + diff --git a/template/pydatascience-web/Dockerfile b/template/pydatascience-web/Dockerfile index 62ad0ea..b75c550 100644 --- a/template/pydatascience-web/Dockerfile +++ b/template/pydatascience-web/Dockerfile @@ -1,6 +1,7 @@ -ARG watchdog_version=0.7.7 -FROM openfaas/of-watchdog:${watchdog_version} as watchdog -FROM python:3-slim +ARG watchdog_version=0.8.4 +ARG python_version=3.9 +FROM --platform=${TARGETPLATFORM:-linux/amd64} ghcr.io/openfaas/of-watchdog:${watchdog_version} as watchdog +FROM --platform=${TARGETPLATFORM:-linux/amd64} python:${python_version}-slim COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog RUN chmod +x /usr/bin/fwatchdog @@ -17,9 +18,10 @@ ENV HOME /home/app ENV PATH=$HOME/conda/bin:$PATH RUN apt-get update \ - && apt-get -y install curl bzip2 ${ADDITIONAL_PACKAGE} \ - && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ - && chown app /tmp/miniconda.sh \ + && apt-get -y install curl bzip2 ${ADDITIONAL_PACKAGE} + +RUN curl -sSL "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -o /tmp/miniforge.sh \ + && chown app /tmp/miniforge.sh \ && apt-get -qq -y remove curl \ && apt-get -qq -y autoremove \ && apt-get autoclean \ @@ -29,22 +31,34 @@ RUN apt-get update \ WORKDIR /home/app/ USER app -RUN bash /tmp/miniconda.sh -bfp $HOME/conda \ +RUN bash /tmp/miniforge.sh -bfp $HOME/conda \ && conda install -y python=3 \ && conda update conda \ && conda clean --all --yes \ - && rm -rf /tmp/miniconda.sh + && rm -rf /tmp/miniforge.sh COPY requirements.txt . -RUN conda install --file requirements.txt -c ${CHANNEL} +RUN conda install --file requirements.txt -c ${CHANNEL} \ + && pip install tox-current-env COPY index.py . RUN mkdir -p function RUN touch ./function/__init__.py WORKDIR /home/app/function/ -COPY function/requirements.txt . -RUN conda install --file requirements.txt -c ${CHANNEL} +COPY function/requirements*.txt ./ +RUN conda install --file requirements.txt --file requirements_test.txt -c ${CHANNEL} + + +COPY function/ . + +ARG TEST_COMMAND="tox --current-env" +ARG TEST_ENABLED=true +RUN if [ "x$TEST_ENABLED" = "xfalse" ]; then \ + echo "skipping tests";\ + else \ + eval "$TEST_COMMAND"; \ + fi WORKDIR /home/app/ COPY function/ ./function diff --git a/template/pydatascience-web/function/handler_test.py b/template/pydatascience-web/function/handler_test.py new file mode 100644 index 0000000..c976061 --- /dev/null +++ b/template/pydatascience-web/function/handler_test.py @@ -0,0 +1,11 @@ + +from .handler import handle + +# Test your handler here + +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +def test_handle(): + # assert handle("input") == "input" + pass \ No newline at end of file diff --git a/template/pydatascience-web/function/requirements_test.txt b/template/pydatascience-web/function/requirements_test.txt new file mode 100644 index 0000000..87c51f0 --- /dev/null +++ b/template/pydatascience-web/function/requirements_test.txt @@ -0,0 +1,3 @@ +tox==3.* +flake8 +pytest \ No newline at end of file diff --git a/template/pydatascience-web/function/tox.ini b/template/pydatascience-web/function/tox.ini new file mode 100644 index 0000000..c9d2139 --- /dev/null +++ b/template/pydatascience-web/function/tox.ini @@ -0,0 +1,42 @@ +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +# Additionally, you can replace the content of this file with +# [tox] +# skipsdist = true + +# You can also edit, remove, or add additional test steps +# by editing, removing, or adding new testenv sections + + +# find out more about tox: https://tox.readthedocs.io/en/latest/ +[tox] +envlist = lint,test +skipsdist = true + +[testenv:test] +deps = + flask + pytes + -r ../requirements.txt + -rrequirements.txt +commands = + # run unit tests with pytest + # https://docs.pytest.org/en/stable/ + # configure by adding a pytest.ini to your handler + pytest + +[testenv:lint] +deps = + flake8 +commands = + flake8 . + +[flake8] +count = true +max-line-length = 127 +max-complexity = 10 +statistics = true +# stop the build if there are Python syntax errors or undefined names +select = E9,F63,F7,F82 +show-source = true \ No newline at end of file diff --git a/template/pydatascience-web/handler_test.py b/template/pydatascience-web/handler_test.py new file mode 100644 index 0000000..c976061 --- /dev/null +++ b/template/pydatascience-web/handler_test.py @@ -0,0 +1,11 @@ + +from .handler import handle + +# Test your handler here + +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +def test_handle(): + # assert handle("input") == "input" + pass \ No newline at end of file diff --git a/template/pydatascience-web/requirements.txt b/template/pydatascience-web/requirements.txt index 3c8a8b6..0916afd 100644 --- a/template/pydatascience-web/requirements.txt +++ b/template/pydatascience-web/requirements.txt @@ -2,3 +2,4 @@ flask gevent python-dateutil==2.7.3 pandas==0.24.2 +tox==3.* \ No newline at end of file diff --git a/template/pydatascience/Dockerfile b/template/pydatascience/Dockerfile index 62ad0ea..f599737 100644 --- a/template/pydatascience/Dockerfile +++ b/template/pydatascience/Dockerfile @@ -1,6 +1,7 @@ -ARG watchdog_version=0.7.7 -FROM openfaas/of-watchdog:${watchdog_version} as watchdog -FROM python:3-slim +ARG watchdog_version=0.8.4 +ARG python_version=3.8 +FROM --platform=${TARGETPLATFORM:-linux/amd64} ghcr.io/openfaas/of-watchdog:${watchdog_version} as watchdog +FROM --platform=${TARGETPLATFORM:-linux/amd64} python:${python_version}-slim COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog RUN chmod +x /usr/bin/fwatchdog @@ -17,9 +18,10 @@ ENV HOME /home/app ENV PATH=$HOME/conda/bin:$PATH RUN apt-get update \ - && apt-get -y install curl bzip2 ${ADDITIONAL_PACKAGE} \ - && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ - && chown app /tmp/miniconda.sh \ + && apt-get -y install curl bzip2 ${ADDITIONAL_PACKAGE} + +RUN curl -sSL "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -o /tmp/miniforge.sh \ + && chown app /tmp/miniforge.sh \ && apt-get -qq -y remove curl \ && apt-get -qq -y autoremove \ && apt-get autoclean \ @@ -29,25 +31,36 @@ RUN apt-get update \ WORKDIR /home/app/ USER app -RUN bash /tmp/miniconda.sh -bfp $HOME/conda \ +RUN bash /tmp/miniforge.sh -bfp $HOME/conda \ && conda install -y python=3 \ && conda update conda \ && conda clean --all --yes \ - && rm -rf /tmp/miniconda.sh + && rm -rf /tmp/miniforge.sh COPY requirements.txt . -RUN conda install --file requirements.txt -c ${CHANNEL} +RUN conda install --file requirements.txt -c ${CHANNEL} \ + && pip install tox-current-env COPY index.py . RUN mkdir -p function RUN touch ./function/__init__.py WORKDIR /home/app/function/ -COPY function/requirements.txt . -RUN conda install --file requirements.txt -c ${CHANNEL} +COPY function/requirements*.txt ./ +RUN conda install --file requirements.txt --file requirements_test.txt -c ${CHANNEL} + + +COPY function/ . + +ARG TEST_COMMAND="tox --current-env" +ARG TEST_ENABLED=true +RUN if [ "x$TEST_ENABLED" = "xfalse" ]; then \ + echo "skipping tests";\ + else \ + eval "$TEST_COMMAND"; \ + fi WORKDIR /home/app/ -COPY function/ ./function # Set environment variables ENV fprocess="python index.py" diff --git a/template/pydatascience/function/handler.py b/template/pydatascience/function/handler.py index 126c588..3dcc0a1 100644 --- a/template/pydatascience/function/handler.py +++ b/template/pydatascience/function/handler.py @@ -16,4 +16,4 @@ def handle(req: bytes) -> str: req (bytes): request body """ - return json.dumps({"echo": req.decode('utf-8')}) + return json.dumps({"echo": req) diff --git a/template/pydatascience/function/handler_test.py b/template/pydatascience/function/handler_test.py new file mode 100644 index 0000000..c976061 --- /dev/null +++ b/template/pydatascience/function/handler_test.py @@ -0,0 +1,11 @@ + +from .handler import handle + +# Test your handler here + +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +def test_handle(): + # assert handle("input") == "input" + pass \ No newline at end of file diff --git a/template/pydatascience/function/requirements_test.txt b/template/pydatascience/function/requirements_test.txt new file mode 100644 index 0000000..87c51f0 --- /dev/null +++ b/template/pydatascience/function/requirements_test.txt @@ -0,0 +1,3 @@ +tox==3.* +flake8 +pytest \ No newline at end of file diff --git a/template/pydatascience/function/tox.ini b/template/pydatascience/function/tox.ini new file mode 100644 index 0000000..6877994 --- /dev/null +++ b/template/pydatascience/function/tox.ini @@ -0,0 +1,42 @@ +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +# Additionally, you can replace the content of this file with +# [tox] +# skipsdist = true + +# You can also edit, remove, or add additional test steps +# by editing, removing, or adding new testenv sections + + +# find out more about tox: https://tox.readthedocs.io/en/latest/ +[tox] +envlist = lint,test +skipsdist = true + +[testenv:test] +deps = + flask + pytest + -r ../requirements.txt + -rrequirements.txt +commands = + # run unit tests with pytest + # https://docs.pytest.org/en/stable/ + # configure by adding a pytest.ini to your handler + pytest + +[testenv:lint] +deps = + flake8 +commands = + flake8 . + +[flake8] +count = true +max-line-length = 127 +max-complexity = 10 +statistics = true +# stop the build if there are Python syntax errors or undefined names +select = E9,F63,F7,F82 +show-source = true \ No newline at end of file