Delete entire pods in lifecycle environment

The lifecycle environment was testing service discovery via bb-terminus container exit. This did not play well with k8s's `CrashLoopBackoff`. Disable bb-terminus container exit in favor of a `redeployer` script, that deletes each bb-terminus pod once per minute. Fixes #42, relates to #43. Signed-off-by: Andrew Seigner <[email protected]>
BuoyantIO · May 24, 2018 · d15938f · d15938f
1 parent eab1166
commit d15938f
Showing 1 changed file with 64 additions and 2 deletions.
diff --git a/lifecycle/lifecycle.yml b/lifecycle/lifecycle.yml
@@ -141,8 +141,70 @@ spec:
           exec \
           /out/bb terminus \
           --grpc-server-port=9090 \
-          --response-text=BANANA \
-          --terminate-after=$(shuf -i 550-650 -n1) # 10 qps * 10 concurrency * 60 seconds / 10 replicas == 600
+          --response-text=BANANA
+          #--terminate-after=$(shuf -i 550-650 -n1) # 10 qps * 10 concurrency * 60 seconds / 10 replicas == 600
         ports:
         - containerPort: 9090
           name: bb-term-grpc
+---
+
+#
+# Redeploy via kubectl
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: redeploy
+  namespace: lifecycle
+data:
+  redeploy: |-
+    #!/bin/sh
+
+    # give deployment time to fully roll out
+    sleep 60
+
+    while true; do
+      PODS=$(kubectl -n lifecycle get po --selector=app=bb-terminus -o jsonpath='{.items[*].metadata.name}')
+
+      SPACES=$(echo "${PODS}" | awk -F" " '{print NF-1}')
+      POD_COUNT=$(($SPACES+1))
+      echo "found ${POD_COUNT} pods"
+
+      # restart each pod every minute
+      SLEEP_TIME=$(( 60 / $POD_COUNT))
+
+      for POD in ${PODS}; do
+        kubectl -n lifecycle delete po $POD
+        echo "sleeping for ${SLEEP_TIME} seconds..."
+        sleep $SLEEP_TIME
+      done
+    done
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: redeployer
+  name: redeployer
+  namespace: lifecycle
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: redeployer
+    spec:
+      containers:
+      - image: lachlanevenson/k8s-kubectl:v1.10.3
+        imagePullPolicy: IfNotPresent
+        name: redeployer
+        command:
+        - "/data/redeploy"
+        volumeMounts:
+        - name: redeploy
+          mountPath: /data
+      volumes:
+      - name: redeploy
+        configMap:
+          name: redeploy
+          defaultMode: 0744