new version

README.md

@@ -89,11 +89,11 @@ EOF
 ```
 
 ```bash 
-# sudo docker-compose up chess1_service
-# sudo docker-compose up chess2_service
-# sudo docker-compose up jupyterlab_service
+# sudo docker compose up chess1_service
+# sudo docker compose up chess2_service
+# sudo docker compose up jupyterlab_service
 
-sudo docker-compose up 
+sudo docker compose up 
 ```
 
 You can check if it's working going to any of the URL:
@@ -136,19 +136,31 @@ export ANSIBLE_CONFIG=${PWD}/ansible.cfg
 # check connectivity
 ansible all -m ping
 
-ansible-playbook ./ansible/setup.yml                  --limit hetzner
-ansible-playbook ./ansible/run.yml                    --limit hetzner --tags "restart"
+ansible-playbook ./ansible/setup.yml  # --tags "restart"
 
 # you can run it later...
 # OPTIONAL, you can even use without precaching (cached=arco will cache blocks on demand)
-ansible-playbook ./ansible/precache.yml               --limit hetzner 
+ansible-playbook ./ansible/precache.yml                
 
 # check docker ps
-ansible --become-user root --become all -m shell -a 'docker-compose ps' --limit hetzner 
-ansible --become-user root --become all -m shell -a 'df -h' --limit hetzner  | grep "/dev/sda1"
+ansible --become-user root --become all -m shell -a 'cd /root/deploy && docker compose ps'  
+ansible --become-user root --become all -m shell -a 'df -h'   | grep "/dev/sda1"
+
+
+# check load balancer
+VPS=$(ansible hetzner --list-hosts | tail -n +2)
+for it in ${VPS} ; do 
+  echo ${it}
+  echo "  HEALTH" $(curl -L -s -o /dev/null -w "%{http_code}" http://$it/health)
+  echo "  LAB   " $(curl -L -s -o /dev/null -w "%{http_code}" http://$it/lab)
+  echo "  CHESS1" $(curl -L -s -o /dev/null -w "%{http_code}" http://$it/chess1)
+  echo "  CHESS2" $(curl -L -s -o /dev/null -w "%{http_code}" http://$it/chess2)
+done
+
+
 
 # ansible-playbook ./ansible/benchmark.yml --verbose
-# ansible-playbook ./ansible/run.yml --limit hetzner --tags "stop"
+# ansible-playbook ./ansible/run.yml  --tags "stop"
 
 # Clean up notebooks
 for it in $(find ./notebooks/*.ipynb) ; do

ansible/run.yml

@@ -38,6 +38,12 @@
         dest: /root/deploy/notebooks/nasa/
       with_fileglob: "../notebooks/nasa/*.ipynb"
 
+    # create nginx log directory
+    - name: Creates nginx log directory
+      ansible.builtin.file:
+        path: /var/log/nginx
+        state: directory
+
     - name: Remove all containers 
       ansible.builtin.shell: |
         docker rm -v -f $(docker ps -qa) || true
@@ -57,4 +63,19 @@
         restarted: true
       tags:
         - start
-        - restart
+        - restart
+
+    - name: Pause for 10 seconds 
+      pause:
+        seconds: 10
+
+    - name: Confirm services are up
+      uri:
+        url: "http://{{ ansible_host }}/{{ item }}"
+        follow_redirects: all
+        status_code: 200
+      loop:
+        - health
+        - lab
+        - chess1
+        - chess2

docker-compose.yml

@@ -2,16 +2,23 @@ services:
 
   # Old Dashboard
   chess1_service:
+    restart: unless-stopped
     image: docker.io/nsdf/nsdf-chess:0.1
     ports:
       - "10334:10334"
     environment:
       - VISUS_CACHE=/mnt/data/visus-cache
     volumes:
       - /mnt/data:/mnt/data
+    healthcheck:
+      test: curl --fail http://localhost:10334 || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   # New Dashboard
   chess2_service:
+    restart: unless-stopped
     image: docker.io/nsdf/openvisuspy:1.0.64
     ports:
       - "10335:10335"
@@ -27,10 +34,15 @@ services:
       - /mnt/data:/mnt/data
       - ./json/dashboards.chess.json:/root/json/dashboards.json
     command: python3 -m panel serve /usr/local/lib/python3.10/site-packages/openvisuspy/app --port 10335 --address 0.0.0.0 --allow-websocket-origin='*' --use-xheaders --num-procs 0 --args /root/json/dashboards.json
-
+    healthcheck:
+      test: curl --fail http://localhost:10335 || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   # Jupyter Lab
   jupyterlab_service:
+    restart: unless-stopped
     image: docker.io/nsdf/openvisuspy:1.0.64
     ports:
       - "8888:8888"
@@ -41,11 +53,37 @@ services:
       - /mnt/data:/mnt/data
       - ./notebooks:/home/notebooks
     command: jupyter lab --allow-root --notebook-dir="/home/notebooks" --port 8888 --NotebookApp.token='${NSDF_TOKEN}' --NotebookApp.allow_origin='*' --ip '0.0.0.0'
+    healthcheck:
+      test: curl --fail http://localhost:8888 || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   # nginx
   nginx_service:
+    restart: unless-stopped
     image: nginx
     ports:
       - 80:80
     volumes:
-      - ./nginx.conf:/etc/nginx/conf.d/default.conf
+      - /var/log/nginx:/var/log/nginx
+      - ./nginx.conf:/etc/nginx/conf.d/default.conf
+    depends_on:
+      chess1_service:
+        condition: service_healthy
+      chess2_service:
+        condition: service_healthy
+      jupyterlab_service:
+        condition: service_healthy
+
+  # autoheal_service
+  # see https://sdr-enthusiasts.gitbook.io/ads-b/useful-extras/auto-restart-unhealthy-containers
+  autoheal_service:
+    image: willfarrell/autoheal:latest
+    tty: true
+    container_name: autoheal
+    restart: unless-stopped
+    environment:
+      - AUTOHEAL_CONTAINER_LABEL=all
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

nginx.conf

@@ -1,13 +1,13 @@
 upstream jupyterlab_upstream {
-  server jupyterlab_service:8888  max_fails=3  fail_timeout=30s;
+  server jupyterlab_service:8888  max_fails=0  fail_timeout=30s;
 }
 
 upstream chess1_upstream {
-  server chess1_service:10334  max_fails=3  fail_timeout=30s;
+  server chess1_service:10334  max_fails=0  fail_timeout=30s;
 }
 
 upstream chess2_upstream {
-  server chess2_service:10335  max_fails=3  fail_timeout=30s;
+  server chess2_service:10335  max_fails=0  fail_timeout=30s;
 }
 
 server {