Introduce dynamic scaling of lifecycle env

Introduce scaling of lifecycle env, via a `deploy` script that deploys N number of lifecycle namespaces, and also via a `scale` script that increases the number of replicas per namespace. Also change slow-cooker from a Job to a Deployment. Part of #43. Signed-off-by: Andrew Seigner <[email protected]>
BuoyantIO · May 24, 2018 · 2fb5421 · 2fb5421
1 parent e6719ad
commit 2fb5421
Show file tree

Hide file tree

Showing 5 changed files with 123 additions and 35 deletions.
diff --git a/lifecycle/README.md b/lifecycle/README.md
@@ -12,8 +12,8 @@ of time in a dynamically-scheduled environment in order to exercise:
 
 ## First time setup
 
-[`lifecycle.yml`](lifecycle.yml) creates a `ClusterRole`, which requires your user to have this
-ability.
+[`lifecycle.yml`](lifecycle.yml) creates a `ClusterRole`, which requires your
+user to have this ability.
 
 ```bash
 kubectl create clusterrolebinding cluster-admin-binding-$USER \
@@ -32,7 +32,15 @@ conduit dashboard --conduit-namespace conduit-lifecycle
 Deploy test framework to `lifecycle` namespace:
 
 ```bash
-cat lifecycle.yml | conduit inject --conduit-namespace conduit-lifecycle - | kubectl apply -f -
+export LIFECYCLE_NS=lifecycle
+kubectl create ns $LIFECYCLE_NS
+cat lifecycle.yml | conduit inject --conduit-namespace conduit-lifecycle - | kubectl -n $LIFECYCLE_NS apply -f -
+```
+
+Scale `bb-p2p` and `bb-terminus`:
+
+```bash
+kubectl -n $LIFECYCLE_NS scale --replicas=3 deploy/bb-p2p deploy/bb-terminus
 ```
 
 ## Observe
@@ -46,19 +54,39 @@ conduit dashboard --conduit-namespace conduit-lifecycle --show grafana
 Tail slow-cooker logs:
 
 ```bash
-kubectl -n lifecycle logs -f $(
-  kubectl -n lifecycle get po --selector=job-name=slow-cooker -o jsonpath='{.items[*].metadata.name}'
+kubectl -n $LIFECYCLE_NS logs -f $(
+  kubectl -n $LIFECYCLE_NS get po --selector=app=slow-cooker -o jsonpath='{.items[*].metadata.name}'
 ) slow-cooker
 ```
 
 Relevant Grafana dashboards to observe
 - `Conduit Deployment`, for route lifecycle and service discovery lifecycle
 - `Prometheus 2.0 Stats`, for telemetry resource lifecycle
 
-
 ## Teardown
 
 ```bash
-kubectl delete ns lifecycle
+kubectl delete ns $LIFECYCLE_NS
 kubectl delete ns conduit-lifecycle
 ```
+
+## Batch Deploy / Scale / Teardown
+
+Deploy 10 lifecycle namespaces:
+
+```bash
+conduit install --conduit-namespace conduit-lifecycle | kubectl apply -f -
+bin/deploy 10
+```
+
+Scale 10 lifecycle namespaces to 3 replicas of `bb-p2p` and `bb-terminus` each:
+
+```bash
+bin/scale 10 3
+```
+
+Teardown 10 lifecycle namespaces:
+
+```bash
+bin/teardown 10
+```
diff --git a/lifecycle/bin/deploy b/lifecycle/bin/deploy
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+
+NAMESPACES=${1:-1}
+
+if [ "$NAMESPACES" -gt 100 ]; then
+  echo "Don't deploy more than 100 namespaces"
+  exit 1
+fi
+
+echo "Deploying $NAMESPACES namespaces..."
+
+for i in `seq 1 $NAMESPACES`;
+do
+  NS=lifecycle$i
+
+  echo "\nDeploying $NS..."
+  kubectl create ns $NS
+  cat lifecycle.yml |
+    conduit inject --conduit-namespace conduit-lifecycle - |
+    kubectl -n $NS apply -f -
+done
diff --git a/lifecycle/bin/scale b/lifecycle/bin/scale
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -e
+
+NAMESPACES=${1:-1}
+REPLICAS=${2:-1}
+
+if [ "$NAMESPACES" -gt 100 ]; then
+  echo "Don't deploy more than 100 namespaces"
+  exit 1
+fi
+
+if [ "$REPLICAS" -gt 100 ]; then
+  echo "Don't scale more than 100 replicas"
+  exit 1
+fi
+
+echo "Scaling $NAMESPACES namespaces to $REPLICAS replicas..."
+
+for i in `seq 1 $NAMESPACES`;
+do
+  NS=lifecycle$i
+
+  echo "\nScaling $NS to $REPLICAS replicas..."
+  kubectl -n lifecycle$i scale --replicas=$REPLICAS deploy/bb-p2p deploy/bb-terminus
+done
diff --git a/lifecycle/bin/teardown b/lifecycle/bin/teardown
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+
+NAMESPACES=${1:-1}
+
+echo "Tearing down $NAMESPACES namespaces...\n"
+
+for i in `seq 1 $NAMESPACES`;
+do
+  kubectl delete ns lifecycle$i || true
+done
diff --git a/lifecycle/lifecycle.yml b/lifecycle/lifecycle.yml
@@ -8,24 +8,22 @@
 #         bb terminus
 #
 
-kind: Namespace
-apiVersion: v1
-metadata:
-  name: lifecycle
-
 #
 # slow_cooker
 #
 ---
-apiVersion: batch/v1
-kind: Job
+apiVersion: extensions/v1beta1
+kind: Deployment
 metadata:
+  labels:
+    app: slow-cooker
   name: slow-cooker
-  namespace: lifecycle
 spec:
+  replicas: 1
   template:
     metadata:
-      name: slow-cooker
+      labels:
+        app: slow-cooker
     spec:
       containers:
       - image: buoyantio/slow_cooker:1.1.0
@@ -42,11 +40,10 @@ spec:
             -concurrency 10 \
             -interval 30s \
             -metric-addr 0.0.0.0:9990 \
-            http://bb-p2p.lifecycle.svc.cluster.local:8080
+            http://bb-p2p:8080
         ports:
         - name: slow-cooker
           containerPort: 9990
-      restartPolicy: OnFailure
 ---
 
 #
@@ -56,7 +53,6 @@ kind: Service
 apiVersion: v1
 metadata:
   name: bb-p2p
-  namespace: lifecycle
 spec:
   clusterIP: None
   selector:
@@ -72,9 +68,8 @@ metadata:
   labels:
     app: bb-p2p
   name: bb-p2p
-  namespace: lifecycle
 spec:
-  replicas: 10
+  replicas: 1
   template:
     metadata:
       labels:
@@ -91,7 +86,7 @@ spec:
         - |
           exec \
           /out/bb point-to-point-channel \
-          --grpc-downstream-server=bb-terminus.lifecycle.svc.cluster.local:9090 \
+          --grpc-downstream-server=bb-terminus:9090 \
           --h1-server-port=8080
         ports:
         - containerPort: 8080
@@ -105,7 +100,6 @@ kind: Service
 apiVersion: v1
 metadata:
   name: bb-terminus
-  namespace: lifecycle
 spec:
   clusterIP: None
   selector:
@@ -121,9 +115,8 @@ metadata:
   labels:
     app: bb-terminus
   name: bb-terminus
-  namespace: lifecycle
 spec:
-  replicas: 10
+  replicas: 1
   template:
     metadata:
       labels:
@@ -155,9 +148,8 @@ kind: ServiceAccount
 apiVersion: v1
 metadata:
   name: redeployer
-  namespace: lifecycle
 ---
-kind: ClusterRole
+kind: Role
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
   name: lifecycle:redeployer
@@ -166,25 +158,22 @@ rules:
   resources: ["pods"]
   verbs: ["delete", "get", "list"]
 ---
-kind: ClusterRoleBinding
+kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
   name: lifecycle:redeployer
-  namespace: lifecycle
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
+  kind: Role
   name: lifecycle:redeployer
 subjects:
 - kind: ServiceAccount
   name: redeployer
-  namespace: lifecycle
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: redeployer
-  namespace: lifecycle
 data:
   redeployer: |-
     #!/bin/sh
@@ -193,17 +182,23 @@ data:
     sleep 60
 
     while true; do
-      PODS=$(kubectl -n lifecycle get po --selector=app=bb-terminus -o jsonpath='{.items[*].metadata.name}')
+      PODS=$(kubectl -n $LIFECYCLE_NS get po --selector=app=bb-terminus -o jsonpath='{.items[*].metadata.name}')
 
       SPACES=$(echo "${PODS}" | awk -F" " '{print NF-1}')
       POD_COUNT=$(($SPACES+1))
       echo "found ${POD_COUNT} pods"
 
       # restart each pod every minute
       SLEEP_TIME=$(( 60 / $POD_COUNT))
+      if [ $SLEEP_TIME = 0 ]; then
+        SLEEP_TIME=1
+      fi
+
+      # TODO: consider overriding the calculation above, to better accomodate large deployments
+      # SLEEP_TIME=30
 
       for POD in ${PODS}; do
-        kubectl -n lifecycle delete po $POD
+        kubectl -n $LIFECYCLE_NS delete po $POD
         echo "sleeping for ${SLEEP_TIME} seconds..."
         sleep $SLEEP_TIME
       done
@@ -215,7 +210,6 @@ metadata:
   labels:
     app: redeployer
   name: redeployer
-  namespace: lifecycle
 spec:
   replicas: 1
   template:
@@ -230,6 +224,11 @@ spec:
         name: redeployer
         command:
         - "/data/redeployer"
+        env:
+        - name: LIFECYCLE_NS
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         volumeMounts:
         - name: redeployer
           mountPath: /data