Add cron container, with it enabled for Akeneo (#210)

* Add cron container for Akeneo * Add cron container for Akeneo * Add cron container for Akeneo * Add cron container for Akeneo * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness * Move cron container to base harness - fix test build issues * Revert "Move cron container to base harness - fix test build issues" This reverts commit 9f3da30e1a911688b9a58ee21947ce731b376485. * Move cron container to base harness - fix test build issues * trigger build * Improve the quoting of env vars * Improve the quoting of env vars * Improve env vars and logging * remove unneeded sleep Co-authored-by: g-foster <[email protected]>
inviqa · Feb 10, 2020 · 0137591 · 0137591
1 parent 5bb1b5c
commit 0137591
Show file tree

Hide file tree

Showing 15 changed files with 179 additions and 5 deletions.
diff --git a/_twig/docker-compose.yml/application.yml.twig b/_twig/docker-compose.yml/application.yml.twig
@@ -55,6 +55,9 @@
   php-fpm:
 {% if @('app.build') == 'dynamic' %}
     build: .my127ws/docker/image/php-fpm
+{% if ("cron" in @('app.services')) %}
+    image: {{ @('workspace.name') ~ '-php-fpm:dev' }}
+{% endif %}
     volumes:
       - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
       - ./.my127ws:/.my127ws

diff --git a/_twig/docker-compose.yml/service/cron.yml.twig b/_twig/docker-compose.yml/service/cron.yml.twig
@@ -0,0 +1,18 @@
+  cron:
+{% if @('app.build') == 'dynamic' %}
+    build:
+      context: ./
+      dockerfile: .my127ws/docker/image/cron/Dockerfile
+    volumes:
+      - {{ (dockersync) ? @('workspace.name') ~ '-sync:/app:nocopy' : './:/app:delegated' }}
+      - ./.my127ws/application:/home/build/application
+{% else %}
+    image: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-cron' }}
+{% endif %}
+    environment:
+{{ deep_merge_to_yaml([@('services.php-base.environment'),@('services.cron.environment')], 2, 6) | raw }}
+{% include blocks ~ 'environment.yml.twig' %}
+    networks:
+      - private
+    labels:
+      - traefik.enable=false
diff --git a/docker/image/console/root/lib/task/database/available.sh b/docker/image/console/root/lib/task/database/available.sh
@@ -20,7 +20,7 @@ function task_database_available()
 
     while ! $command &> /dev/null; do
 
-        if (( counter > 8 )); then
+        if (( counter > 120 )); then
             (>&2 echo "timeout while waiting on ${DB_PLATFORM} to become available")
             exit 1
         fi

diff --git a/docker/image/cron/Dockerfile.twig b/docker/image/cron/Dockerfile.twig
@@ -0,0 +1,29 @@
+{% if @('app.build') == 'static' %}
+FROM {{ @('docker.repository') ~ ':' ~ @('app.version') }}-php-fpm
+{% else %}
+FROM {{ @('workspace.name') ~ '-php-fpm:dev' }}
+{% endif %}
+
+# install cron
+RUN apt update
+RUN apt install -y cron sudo
+
+WORKDIR /app
+COPY .my127ws/docker/image/cron/root /
+RUN chmod +x /cron-run-with-env.sh
+
+{% if @('app.build') == 'static' %}
+RUN bash /fix_app_permissions.sh
+{% else %}
+VOLUME /app
+{% endif %}
+ENV APP_MODE {{ @('app.mode') }}
+
+{% if @('app.build') == 'static' %}
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["sleep", "infinity"]
+{% else %}
+ENTRYPOINT ["/entrypoint.dynamic.sh"]
+CMD ["sleep", "infinity"]
+{% endif %}
diff --git a/docker/image/cron/root/cron-run-with-env.sh b/docker/image/cron/root/cron-run-with-env.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+IFS=$'\n'
+env_vars="$(cat /app/env.sh)"
+/usr/bin/env - "$env_vars" sudo --preserve-env -u www-data "$@" > /proc/1/fd/1 2> /proc/1/fd/2
diff --git a/docker/image/cron/root/crontab.twig b/docker/image/cron/root/crontab.twig
@@ -0,0 +1,3 @@
+{% for cronjob in @('backend.cron.jobs') -%}
+{{ cronjob|raw }}
+{% endfor %}
diff --git a/docker/image/cron/root/entrypoint.sh b/docker/image/cron/root/entrypoint.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# this is used to load env vars in crontab commands
+env > /app/env.sh
+
+# run
+crontab /crontab
+exec cron -f -L 15
diff --git a/harness/attributes/common.yml b/harness/attributes/common.yml
@@ -77,6 +77,8 @@ attributes.default:
       steps: []
     migrate:
       steps: []
+    cron:
+      jobs: []
 
   frontend:
     path: /app

diff --git a/harness/attributes/docker-base.yml b/harness/attributes/docker-base.yml
@@ -33,6 +33,8 @@ attributes:
         POSTGRES_PASSWORD: = @('database.pass')
         POSTGRES_USER: = @('database.user')
         PGDATA: /var/lib/postgresql/data/pgdata
+    cron:
+      environment: []
   pipeline:
     base:
       services:

diff --git a/harness/config/confd.yml b/harness/config/confd.yml
@@ -22,12 +22,15 @@ confd('harness:/'):
   - { src: docker/image/php-fpm/root/fix_app_permissions.sh }
   - { src: docker/image/php-fpm/root/usr/local/etc/php/php.ini }
   - { src: docker/image/php-fpm/root/usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini }
+  - { src: docker/image/cron/Dockerfile }
+  - { src: docker/image/cron/root/crontab }
   - { src: application/overlay/Jenkinsfile }
   - { src: application/overlay/auth.json }
   - { src: application/overlay/.dockerignore, dst: workspace:/.dockerignore }
   - { src: application/skeleton/README.md }
   - { src: docker-compose.yml, dst: workspace:/docker-compose.yml }
   - { src: docker-sync.yml, dst: workspace:/docker-sync.yml }
+  - { src: harness/scripts/enable.sh }
   - { src: helm/app/values.yaml }
   - { src: helm/app/Chart.yaml }
   - { src: helm/qa/requirements.yaml }

diff --git a/harness/config/pipeline.yml b/harness/config/pipeline.yml
@@ -9,6 +9,10 @@ command('app build console'): |
   #!bash(workspace:/)|@
   passthru docker build --pull -t @('docker.repository'):@('app.version')-console -f .my127ws/docker/image/console/Dockerfile .
 
+command('app build cron'): |
+  #!bash(workspace:/)|@
+  passthru docker build -t @('docker.repository'):@('app.version')-cron -f .my127ws/docker/image/cron/Dockerfile .
+
 command('app build php-fpm'): |
   #!bash(harness:/docker/image/php-fpm)|@
   passthru docker build -t @('docker.repository'):@('app.version')-php-fpm .

diff --git a/harness/scripts/enable.sh → harness/scripts/enable.sh.twig b/harness/scripts/enable.sh → harness/scripts/enable.sh.twig
@@ -35,9 +35,15 @@ dynamic()
         passthru docker-sync stop
     fi
 
-    passthru docker-compose pull
-    passthru docker-compose build --pull
-    passthru docker-compose up -d
+    {% if ("cron" in @('app.services')) %}
+        passthru "docker-compose config --services | grep -v php-fpm | xargs docker-compose pull"
+        passthru "docker-compose config --services | grep -v cron | xargs docker-compose build --pull"
+        passthru docker-compose build cron
+    {% else %}
+        passthru docker-compose pull
+        passthru docker-compose build --pull
+        passthru docker-compose up -d
+    {% endif %}
 
     passthru docker-compose exec -T -u build console app build
     passthru docker-compose exec -T -u build console app init

diff --git a/helm/app/_twig/values.yaml/resources.yml.twig b/helm/app/_twig/values.yaml/resources.yml.twig
@@ -10,6 +10,7 @@
       postgres: {{ @('resources.requests.memory.postgres') }}
       redis: {{ @('resources.requests.memory.redis') }}
       redis_session: {{ @('resources.requests.memory.redis_session') }}
+      cron: {{ @('resources.requests.memory.cron') }}
   limits:
     memory:
       console: {{ @('resources.limits.memory.console') }}
@@ -21,4 +22,5 @@
       mysql: {{ @('resources.limits.memory.mysql') }}
       postgres: {{ @('resources.limits.memory.postgres') }}
       redis: {{ @('resources.limits.memory.redis') }}
-      redis_session: {{ @('resources.limits.memory.redis_session') }}
+      redis_session: {{ @('resources.limits.memory.redis_session') }}
+      cron: {{ @('resources.limits.memory.cron') }}
diff --git a/helm/app/templates/application/cron/deployment.yaml b/helm/app/templates/application/cron/deployment.yaml
@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.resourcePrefix }}cron
+  labels:
+    app.service: {{ .Values.resourcePrefix }}cron
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.service: {{ .Values.resourcePrefix }}cron
+  template:
+    metadata:
+      labels:
+        app.service: {{ .Values.resourcePrefix }}cron
+    spec:
+      {{- if not (eq "" (include "application.volumes.wwwDataPaths" .)) }}
+      initContainers:
+      - name: cron-volume-permissions
+        image: busybox
+        command: 
+        - "/bin/chown"
+        - "-R"
+        - "www-data"
+        {{- include "application.volumes.wwwDataPaths" . | indent 8 }}
+        volumeMounts:
+        {{- if not (and (eq "" (include "application.volumeMounts.backend" .)) (eq "" (include "application.volumeMounts.all" .)) ) }}
+        {{- include "application.volumeMounts.backend" . | indent 8 }}
+        {{- include "application.volumeMounts.all" . | indent 8 }}
+        {{- else }} []
+        {{- end }}
+      {{- end }}
+      containers:
+      - name: cron
+        env:
+        {{- range $key, $value := index .Values.docker.services "php-base" "environment" }}
+        {{- $tp := typeOf $value }}
+        - name: {{ $key | quote }}
+        {{- if eq $tp "string" }}
+          value: {{ tpl $value $ | quote }}
+        {{- else}}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
+        {{- range $key, $value := index .Values.docker.services "cron" "environment" }}
+        {{- $tp := typeOf $value }}
+        - name: {{ $key | quote }}
+        {{- if eq $tp "string" }}
+          value: {{ tpl $value $ | quote }}
+        {{- else}}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
+        {{- range $key, $value := .Values.environment }}
+        - name: {{ $key | quote }}
+          value: {{ $value | quote }}
+        {{- end }}
+        image: {{ .Values.docker.image.cron }}
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 9000
+        resources:
+          limits:
+            memory: {{ .Values.resources.limits.memory.cron }}
+          requests:
+            memory: {{ .Values.resources.requests.memory.cron }}
+        readinessProbe:
+          tcpSocket:
+            port: 9000
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        volumeMounts:
+        {{- if not (and (eq "" (include "application.volumeMounts.backend" .)) (eq "" (include "application.volumeMounts.all" .)) ) }}
+        {{- include "application.volumeMounts.backend" . | indent 8 }}
+        {{- include "application.volumeMounts.all" . | indent 8 }}
+        {{- else }} []
+        {{- end }}
+
+      imagePullSecrets:
+      - name: {{ .Values.resourcePrefix }}docker-config
+      restartPolicy: Always
+      volumes:
+      {{- if not (and (eq "" (include "application.volumes.backend" .)) (eq "" (include "application.volumes.all" .)) ) }}
+      {{- include "application.volumes.backend" . | indent 6 }}
+      {{- include "application.volumes.all" . | indent 6 }}
+      {{- else }} []
+      {{- end }}
+status: {}
diff --git a/helm/app/values.yaml.twig b/helm/app/values.yaml.twig
@@ -10,6 +10,7 @@ docker:
     fpm: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-php-fpm' }}
     nginx: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-nginx' }}
     elasticsearch: {{ @('elasticsearch.image') ~ ':' ~ @('elasticsearch.tag') }}
+    cron: {{ @('docker.repository') ~ ':' ~ @('app.version') ~ '-cron' }}
   services:
 {{ deep_merge_to_yaml([@('services'), @('pipeline.base.services')], 2, 4) | raw }}
 
@@ -20,6 +21,7 @@ service:
   postgres: {{ ("postgres" in @('app.services')) ? 'true' : 'false' }}
   redis: {{ ("redis" in @('app.services')) ? 'true' : 'false' }}
   redis_session: {{ ("redis-session" in @('app.services')) ? 'true' : 'false' }}
+  cron: {{ ("cron" in @('app.services')) ? 'true' : 'false' }}
 
 resources:
 {% include blocks ~ 'resources.yml.twig' %}