update the sc server as sc-k8-server

add minikube deployment instruction on minikube
convert deployment script as helm chart
update makefile to generate docker image for aws

DEVELOPER.md

@@ -124,14 +124,43 @@ Get SPU
  fluvio spu list --sc 127.0.0.1:9003
 ```
 
-## Setting up for Kubernetes local development
+## Deploying docker images to minikube
 
+Steps are:
+* First make sure to shut down sc and custom spu servers.
+* Delete Custom SPU and topics.
+* Make sure docker is running
+
 Run following script to allow host docker to access minikube docker.  Without it, you can't upload image to minikube.
 
 ```
 ./dev-tools/minikube-docker.sh 
 ```
 
+Ensure you have setup tunnel so can access SC and SPU from your machine:
+```
+sudo ./k8-util/minikube-tunnel.sh
+```
+
+Create Docker images locally:
+```
+make minikube_image
+```
+
+## Helm Install
+
+Install helm
+```
+brew install helm
+```
+
+Deploy docker images
+```
+./k8-utils/deploy.sh
+```
+
+## Release
+
 
 
 

Makefile

@@ -1,12 +1,14 @@
+VERSION := $(shell cat VERSION)
 TOOLCHAIN = "./rust-toolchain"
 RUSTV = $(shell cat ${TOOLCHAIN})
 RUST_DOCKER_IMAGE=fluvio/rust-tool:${RUSTV}
 CARGO_BUILD=build --release
 BIN_NAME=release
-PUSH=push
+MAKE_CMD=build
 GITHUB_USER=infinyon
 GITHUB_REPO=fluvio
 GITHUB_TAG=0.2.1
+DOCKER_REGISTRY=infinyon
 TARGET_LINUX=x86_64-unknown-linux-musl
 TARGET_DARWIN=x86_64-apple-darwin
 CLI_BUILD=fluvio_cli
@@ -34,27 +36,40 @@ release_cli_linux:
 	cd /tmp;tar -czvf cli-${TARGET_LINUX}-release.tar.gz $(CLI_BUILD)_${TARGET_LINUX};rm -rf $(CLI_BUILD)_${TARGET_LINUX}
 
 
+all_image:	linux-spu-server spu_image linux-sc-server sc_image
 
 # create docker images for release
-release_image:	PUSH=push_release
+release_image:	MAKE_CMD=push_release
+release_image:	all_image
 
-minikube_image:	linux-spu-server spu_image linux-sc-server sc_image
-release_image:	linux-spu-server spu_image spu_image linux-sc-server sc_image
+develop_image:	VERSION=$(shell git log -1 --pretty=format:"%H")
+develop_image: 	all_image
 
+local_image:	develop_image
+local_image:	DOCKER_REGISTRY=localhost:5000/infinyon
 
+minikube_image:	local_image
+minikube_image:	MAKE_CMD=minikube
 
-linux-sc-server:
-	cargo $(CARGO_BUILD) --bin sc-server  --target ${TARGET_LINUX}
 
-linux-spu-server:
+aws_dev_image:	develop_image
+aws_dev_image:	MAKE_CMD=push
+aws_dev_image:	DOCKER_REGISTRY=$(AWS_ECR)
+
+linux-sc-server:	install_musl
+	cargo $(CARGO_BUILD) --bin sc-k8-server  --target ${TARGET_LINUX}
+
+linux-spu-server:	install_musl
 	cargo $(CARGO_BUILD) --bin spu-server  --target ${TARGET_LINUX}
 
 
-spu_image:	install_musl linux-spu-server
-	make build BIN_NAME=$(BIN_NAME) $(PUSH) -C k8-util/docker/spu
+spu_image:	 linux-spu-server
+	echo "Building SPU image with version: ${VERSION}"
+	make build BIN_NAME=$(BIN_NAME) $(MAKE_CMD) VERSION=${VERSION} REGISTRY=${DOCKER_REGISTRY} -C k8-util/docker/spu
 
 sc_image:	install_musl linux-spu-server
-	make build BIN_NAME=$(BIN_NAME) $(PUSH) -C k8-util/docker/sc
+	echo "Building SC image with version: ${VERSION}"
+	make build BIN_NAME=$(BIN_NAME) $(MAKE_CMD) VERSION=${VERSION} REGISTRY=${DOCKER_REGISTRY} -C k8-util/docker/sc
 
 
 cargo_cache_dir:

VERSION

@@ -0,0 +1 @@
+0.2.2

dev-tools/log/debug-sc-all

@@ -2,4 +2,4 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 RELATIVE_PATH="$DIR"/../../target/debug
 
-RUST_LOG=kf=trace,sc=trace,flv=trace "$RELATIVE_PATH"/flv-sc-k8
+RUST_LOG=kf=trace,sc=trace,flv=trace "$RELATIVE_PATH"/sc-k8-server

dev-tools/log/debug-sc-client

@@ -1 +1 @@
-RUST_LOG=sc_core=debug,cluster_cache=debug,k8_client=trace,tokio_reactor=error target/debug/flv-sc-k8
+RUST_LOG=sc_core=debug,cluster_cache=debug,k8_client=trace,tokio_reactor=error target/debug/sc-k8-server

dev-tools/log/debug-sc-min

@@ -2,4 +2,4 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 RELATIVE_PATH="$DIR"/../../target/debug
 
-RUST_LOG=kf=debug,sc=debug,flv=debug "$RELATIVE_PATH"/flv-sc-k8  $@
+RUST_LOG=kf=debug,sc=debug,flv=debug "$RELATIVE_PATH"/sc-k8-server  $@

dev-tools/log/debug-sc-server

@@ -1 +1 @@
-RUST_LOG=sc_core=trace,k8_config=debug,k8_client=debug,tokio_reactor=error target/debug/flv-sc-k8
+RUST_LOG=sc_core=trace,k8_config=debug,k8_client=debug,tokio_reactor=error target/debug/sc-k8-server

k8-util/docker/sc/Dockerfile.sc

@@ -1,5 +1,5 @@
 FROM scratch
 
-COPY    sc-server  /fluvio/sc-server
+COPY    sc-k8-server  /fluvio/sc-k8-server
 
-CMD ["/fluvio/sc-server"]
+CMD ["/fluvio/sc-k8-server"]

k8-util/docker/sc/Makefile

@@ -1,27 +1,20 @@
-NAME = infinyon/fluvio-sc
-TAG = 0.2.1
+NAME = $(REGISTRY)/fluvio-sc
+TAG = $(VERSION)
 BIN_DIR = ../../../target/x86_64-unknown-linux-musl/$(BIN_NAME)
 
-all:	push
+all:	build
 
 
 copy_binaries:
 	mkdir -p target
-	cp	${BIN_DIR}/sc-server target
+	cp	${BIN_DIR}/sc-k8-server target
 
 build:	copy_binaries
 	docker build -f Dockerfile.sc  -t $(NAME):$(TAG)  ./target
 
-push_registry:
-	docker tag $(NAME):$(TAG) localhost:5000/$(NAME):$(TAG)
-	docker push localhost:5000/$(NAME):$(TAG)
+push:	build
+	docker push $(NAME):$(TAG)
 
-pull_minikube:
+minikube:	push
 	eval $$(minikube docker-env); \
-	docker pull localhost:5000/$(NAME):$(TAG); \
-	docker tag localhost:5000/$(NAME):$(TAG) $(NAME):$(TAG)
-
-push:	push_registry pull_minikube
-
-push_release:	build
-	docker push $(NAME):$(TAG)
+	docker pull $(NAME):$(TAG);

k8-util/docker/spu/Makefile

@@ -1,5 +1,5 @@
-NAME = infinyon/fluvio-spu
-TAG = 0.2.1
+NAME = $(REGISTRY)/fluvio-spu
+TAG = $(VERSION)
 BIN_DIR = ../../../target/x86_64-unknown-linux-musl/$(BIN_NAME)
 
 all: push
@@ -11,17 +11,9 @@ copy_binaries:
 build:	copy_binaries
 	docker build -f Dockerfile.spu  -t $(NAME):$(TAG)  ./target
 
+push:	build
+	docker push $(NAME):$(TAG)
 
-push_registry:
-	docker tag $(NAME):$(TAG) localhost:5000/$(NAME):$(TAG)
-	docker push localhost:5000/$(NAME):$(TAG)
-
-pull_minikube:
+minikube:	push
 	eval $$(minikube docker-env); \
-	docker pull localhost:5000/$(NAME):$(TAG); \
-	docker tag localhost:5000/$(NAME):$(TAG) $(NAME):$(TAG)
-
-push:	build push_registry pull_minikube
-
-push_release:	build
-	docker push $(NAME):$(TAG)
+	docker pull $(NAME):$(TAG);

k8-util/helm/Makefile

@@ -0,0 +1,26 @@
+VERSION := $(shell cat ../../VERSION)
+GIT_VERSION=$(shell git log -1 --pretty=format:"%H")
+REGISTRY=infinyon
+CLOUD=minikube
+SC_LOG="RUST_LOG=kf=trace,sc=trace,flv=trace"
+
+dry-run:
+	helm install --debug --dry-run goodly-guppy ./fluvio --set fluvioVersion=$(GIT_VERSION)
+
+dry-run-aws:
+	helm install --debug --dry-run goodly-guppy ./fluvio --set cloud=aws
+
+install-test:
+	helm install test ./fluvio --set fluvioVersion=$(GIT_VERSION) \
+		 --set registry=$(REGISTRY) --set cloud=$(CLOUD)
+
+install_minikube:	REGISTRY=localhost:5000/infinyon
+install_minikube:	install-test
+
+install_aws:	install-test
+install_aws:	CLOUD=aws
+install_aws:	REGISTRY=$(AWS_ECR)
+
+
+uninstall-test:
+	helm uninstall test

k8-util/helm/fluvio/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8-util/helm/fluvio/Chart.yaml

@@ -0,0 +1,21 @@
+apiVersion: v2
+name: fluvio
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 0.1.0

k8-util/sc-deployment/sc-deployment.yaml → k8-util/helm/fluvio/templates/sc-deployment.yaml

@@ -7,10 +7,12 @@ metadata:
 spec:
   containers:
     - name: flv-sc
-      image: infinyon/fluvio-sc:0.2.1
+      image: {{ .Values.registry }}/fluvio-sc:{{ .Values.fluvioVersion }}
       imagePullPolicy: IfNotPresent
       ports:
         - containerPort: 9003
       env:
         - name: RUST_LOG
-          value: sc_server=info
+          value: {{ .Values.scLog }}
+        - name:  SPU_IMAGE
+          value: {{ .Values.registry }}/fluvio-spu:{{ .Values.fluvioVersion }}

k8-util/helm/fluvio/templates/storage.yaml

@@ -0,0 +1,17 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: fluvio-spu
+  annotations:
+    {{- if eq .Values.cloud "aws" }}
+    storageclass.kubernetes.io/is-default-class: "true"
+    {{- end }}
+{{- if eq .Values.cloud "minikube" }}
+provisioner: k8s.io/minikube-hostpath
+reclaimPolicy: Retain
+{{- else if eq .Values.cloud "aws" }}
+provisioner: kubernetes.io/aws-ebs
+parameters:
+  type: gp2
+  fsType: ext4 
+{{- end }}

k8-util/helm/fluvio/values.yaml

@@ -0,0 +1,30 @@
+# Default values for fluvio.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+cloud:  minikube
+fluvioVersion: 0.0.1
+registry: infinyon
+
+scLog: info
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000

k8-util/minikube-tunnel.sh

@@ -1,2 +1,3 @@
 #!/bin/bash
+pkill tunnel
 nohup  minikube tunnel  > /tmp/tunnel.out 2> /tmp/tunnel.out &

src/nj/nj-core/src/basic.rs

@@ -81,7 +81,7 @@ impl JsEnv {
 
     pub fn create_double(&self,value: f64) -> Result<napi_value,NjError> {
 
-        let mut result: napi_value = ptr::null_mut();
+        let mut result = ptr::null_mut();
         napi_call_result!(
             crate::sys::napi_create_double(
                 self.0,
@@ -94,7 +94,7 @@ impl JsEnv {
 
     pub fn create_int64(&self,value: i64) -> Result<napi_value,NjError> {
 
-        let mut result: napi_value = ptr::null_mut();
+        let mut result = ptr::null_mut();
         napi_call_result!(
             crate::sys::napi_create_int64(
                 self.0,
@@ -107,7 +107,7 @@ impl JsEnv {
 
     pub fn create_int32(&self,value: i32) -> Result<napi_value,NjError> {
 
-        let mut result: napi_value = ptr::null_mut();
+        let mut result = ptr::null_mut();
         napi_call_result!(
             crate::sys::napi_create_int32(
                 self.0,

src/sc-k8/Cargo.toml

@@ -4,6 +4,10 @@ edition = "2018"
 version = "0.2.1"
 authors = ["fluvio.io"]
 
+[[bin]]
+name = "sc-k8-server"
+path = "src/main.rs"
+doc = false
 
 [dependencies]
 rand = "0.7.2"