Feature add gha ci workflow and support tox based testing (#3)

* feat: Add support to tox, arm64, and a CI flow

**What**
- Add support for arm64 builds, this requires a switch to miniforge for
  conda. This is a community effort in conda to support arm, which is
  not fully suppported by Anaconda co.
- Add support for testing with tox
- Add two sample test functions that demonstrate support for numpy
- Add CI workflow using github actions to verify that the same functions
  build as expected.

Signed-off-by: Lucas Roesler <[email protected]>

.github/workflows/test.yaml

@@ -0,0 +1,49 @@
+name: test
+
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  build-test-functions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 0
+      - name: Get faas-cli
+        run: curl -sLSf https://cli.openfaas.com | sudo sh
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.15
+      - name: Setup git-semver
+        run: GO111MODULE=on go get github.com/mdomke/git-semver/[email protected]
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Get TAG
+        id: get_tag
+        run: echo ::set-output name=tag::$(git-semver | tr '+' '.')
+      - name: Get Repo Owner
+        id: get_repo_owner
+        run: >
+          echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} |
+          tr '[:upper:]' '[:lower:]')
+      - name: Debug variables
+        run: |
+          echo "repo_owner: ${{ steps.get_repo_owner.outputs.repo_owner }}"
+          echo "tag: ${{ steps.get_tag.outputs.tag }}"
+          echo "repo: ${{ github.repository }}"
+          echo "git-describe: $(git describe --tags --always)"
+      - name: Publish functions
+        run: >
+          DOCKER_BUILDKIT=1
+          OWNER="${{ steps.get_repo_owner.outputs.repo_owner }}"
+          TAG="${{ steps.get_tag.outputs.tag }}"
+          faas-cli build
+          --parallel 2
+          --disable-stack-pull

pydatascience-test/core/README.md

@@ -0,0 +1,3 @@
+Put common utilities and generalized logic here
+
+A basic implementation for reading mounted secrets is provided, as an example.

pydatascience-test/core/utils.py

@@ -0,0 +1,9 @@
+
+
+def getSecret(secret_name: str) -> str:
+    """load secret value from the openfaas secret folder
+    Args:
+        secret_name (str): name of the secret
+    """
+    with open("/var/openfaas/secrets/" + secret_name, "r") as file:
+        return file.read()

pydatascience-test/handler.py

@@ -0,0 +1,24 @@
+import json
+import os
+
+import numpy as np
+
+# from .core import utils
+
+
+function_root = os.environ.get("function_root")
+
+# Now  pre-load the model, e.g.
+# from .core import model
+
+
+def handle(req: bytes) -> str:
+    """handle a request to the function
+    Args:
+        req (bytes): request body
+    """
+
+    return json.dumps({
+        "echo": req,
+        "random": np.random.random_sample(),
+    })

pydatascience-test/handler_test.py

@@ -0,0 +1,14 @@
+import json
+from .handler import handle
+
+# Test your handler here
+
+# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml
+# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args
+
+def test_handle():
+    raw_output = handle("input")
+    output = json.loads(raw_output)
+
+    assert output.get("echo") == "input"
+    assert 0.0 <= output.get("random") <= 1.0

pydatascience-test/tox.ini

@@ -0,0 +1,42 @@
+# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml
+# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args
+
+# Additionally, you can replace the content of this file with
+#   [tox]
+#   skipsdist = true
+
+# You can also edit, remove, or add additional test steps
+# by editing, removing, or adding new testenv sections
+
+
+# find out more about tox: https://tox.readthedocs.io/en/latest/
+[tox]
+envlist = lint,test
+skipsdist = true
+
+[testenv:test]
+deps =
+  flask
+  pytest
+  -r ../requirements.txt
+  -rrequirements.txt
+commands =
+  # run unit tests with pytest
+  # https://docs.pytest.org/en/stable/
+  # configure by adding a pytest.ini to your handler
+  pytest
+
+[testenv:lint]
+deps =
+  flake8
+commands =
+  flake8 .
+
+[flake8]
+count = true
+max-line-length = 127
+max-complexity = 10
+statistics = true
+# stop the build if there are Python syntax errors or undefined names
+select = E9,F63,F7,F82
+show-source = true

pydatascience-test/train.py

@@ -0,0 +1,3 @@
+"""
+Put your model training here
+"""

pydatascience-web-test/core/README.md

@@ -0,0 +1,3 @@
+Put common utilities and generalized logic here
+
+A basic implementation for reading mounted secrets is provided, as an example.

pydatascience-web-test/core/utils.py

@@ -0,0 +1,9 @@
+
+
+def getSecret(secret_name: str) -> str:
+    """load secret value from the openfaas secret folder
+    Args:
+        secret_name (str): name of the secret
+    """
+    with open("/var/openfaas/secrets/" + secret_name, "r") as file:
+        return file.read()

pydatascience-web-test/handler.py

@@ -0,0 +1,34 @@
+
+import json
+import os
+
+from flask import Request
+import numpy as np
+
+# from .core import utils
+
+
+function_root = os.environ.get("function_root")
+
+# Now  pre-load the model, e.g.
+# from .core import model
+
+
+def handle(req: Request):
+    """handle a request to the function.
+
+    Your response is immediately passed to the caller, unmodified.
+    This allows you full control of the response, e.g. you can set
+    the status code by returning a tuple (str, int). A detailed
+    description of how responses are handled is found here:
+
+    http://flask.pocoo.org/docs/1.0/quickstart/#about-responses
+
+    Args:
+        req (Request): Flask request object
+    """
+
+    return json.dumps({
+        "echo": req.get_data().decode('utf-8'),
+        "random": np.random.random_sample(),
+    })

pydatascience-web-test/handler_test.py

@@ -0,0 +1,21 @@
+import json
+from .handler import handle
+
+import flask
+
+app = flask.Flask(__name__)
+
+
+# Test your handler here
+
+# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml
+# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args
+
+def test_handle():
+    with app.test_request_context('/', data="input"):
+        assert flask.request.path == '/'
+        raw_output = handle(flask.request)
+        output = json.loads(raw_output)
+
+        assert output.get("echo") == "input"
+        assert 0.0 <= output.get("random") <= 1.0

pydatascience-web-test/requirements_test.txt

@@ -0,0 +1,3 @@
+tox==3.*
+flake8
+pytest

pydatascience-web-test/tox.ini

@@ -0,0 +1,42 @@
+# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml
+# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args
+
+# Additionally, you can replace the content of this file with
+#   [tox]
+#   skipsdist = true
+
+# You can also edit, remove, or add additional test steps
+# by editing, removing, or adding new testenv sections
+
+
+# find out more about tox: https://tox.readthedocs.io/en/latest/
+[tox]
+envlist = lint,test
+skipsdist = true
+
+[testenv:test]
+deps =
+  flask
+  pytes
+  -r ../requirements.txt
+  -rrequirements.txt
+commands =
+  # run unit tests with pytest
+  # https://docs.pytest.org/en/stable/
+  # configure by adding a pytest.ini to your handler
+  pytest
+
+[testenv:lint]
+deps =
+  flake8
+commands =
+  flake8 .
+
+[flake8]
+count = true
+max-line-length = 127
+max-complexity = 10
+statistics = true
+# stop the build if there are Python syntax errors or undefined names
+select = E9,F63,F7,F82
+show-source = true

pydatascience-web-test/train.py

@@ -0,0 +1,3 @@
+"""
+Put your model training here
+"""

stack.yml

@@ -0,0 +1,15 @@
+version: 1.0
+provider:
+  name: openfaas
+  gateway: http://127.0.0.1:8080
+
+functions:
+  pydatascience-test:
+    lang: pydatascience
+    handler: ./pydatascience-test
+    image: ghcr.io/${OWNER:-lucasroesler}/pydatascience-test:${TAG:-latest}
+  pydatascience-web-test:
+    lang: pydatascience-web
+    handler: ./pydatascience-web-test
+    image: pydatascience-web-test:latest
+

template/pydatascience-web/Dockerfile

@@ -1,6 +1,7 @@
-ARG watchdog_version=0.7.7
-FROM openfaas/of-watchdog:${watchdog_version} as watchdog
-FROM python:3-slim
+ARG watchdog_version=0.8.4
+ARG python_version=3.9
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ghcr.io/openfaas/of-watchdog:${watchdog_version} as watchdog
+FROM --platform=${TARGETPLATFORM:-linux/amd64} python:${python_version}-slim
 
 COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog
 RUN chmod +x /usr/bin/fwatchdog
@@ -17,9 +18,10 @@ ENV HOME /home/app
 ENV PATH=$HOME/conda/bin:$PATH
 
 RUN apt-get update \
-    && apt-get -y install curl bzip2 ${ADDITIONAL_PACKAGE} \
-    && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
-    && chown app /tmp/miniconda.sh \
+    && apt-get -y install curl bzip2 ${ADDITIONAL_PACKAGE}
+
+RUN curl -sSL "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -o /tmp/miniforge.sh \
+    && chown app /tmp/miniforge.sh \
     && apt-get -qq -y remove curl \
     && apt-get -qq -y autoremove \
     && apt-get autoclean \
@@ -29,22 +31,34 @@ RUN apt-get update \
 WORKDIR /home/app/
 USER app
 
-RUN bash /tmp/miniconda.sh -bfp $HOME/conda \
+RUN bash /tmp/miniforge.sh -bfp $HOME/conda \
     && conda install -y python=3 \
     && conda update conda \
     && conda clean --all --yes \
-    && rm -rf /tmp/miniconda.sh
+    && rm -rf /tmp/miniforge.sh
 
 COPY requirements.txt   .
-RUN conda install --file requirements.txt -c ${CHANNEL}
+RUN conda install --file requirements.txt -c ${CHANNEL} \
+    && pip install tox-current-env
 COPY index.py           .
 
 RUN mkdir -p function
 RUN touch ./function/__init__.py
 
 WORKDIR /home/app/function/
-COPY function/requirements.txt	.
-RUN conda install --file requirements.txt -c ${CHANNEL}
+COPY function/requirements*.txt	./
+RUN conda install --file requirements.txt --file requirements_test.txt -c ${CHANNEL}
+
+
+COPY function/ .
+
+ARG TEST_COMMAND="tox --current-env"
+ARG TEST_ENABLED=true
+RUN if [ "x$TEST_ENABLED" = "xfalse" ]; then \
+    echo "skipping tests";\
+    else \
+    eval "$TEST_COMMAND"; \
+    fi
 
 WORKDIR /home/app/
 COPY function/ ./function

template/pydatascience-web/function/handler_test.py

@@ -0,0 +1,11 @@
+
+from .handler import handle
+
+# Test your handler here
+
+# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml
+# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args
+
+def test_handle():
+    # assert handle("input") == "input"
+    pass

template/pydatascience-web/function/requirements_test.txt

@@ -0,0 +1,3 @@
+tox==3.*
+flake8
+pytest