- switched to single zammad pod to get rid of nfs container in kubernetes

- added nginx configmap
- switched to statefulset for es
- removed nginx & nfs container builds
- switched to postgresql 10.1 (manual update needed)

containers/zammad-nfs/Dockerfile

@@ -1,22 +0,0 @@
-FROM cpuguy83/nfs-server
-MAINTAINER Zanmmad <info@zammad.org>
-
-ENV NFS_DIR /exports
-ENV TMPFS_SIZE 256M
-
-LABEL org.label-schema.build-date="$BUILD_DATE" \
-      org.label-schema.name="Zammad" \
-      org.label-schema.license="AGPL-3.0" \
-      org.label-schema.description="Docker container for Zammad - NFS Server Container for Kubernetes" \
-      org.label-schema.url="https://zammad.org" \
-      org.label-schema.vcs-url="https://github.com/zammad/zammad" \
-      org.label-schema.vcs-type="Git" \
-      org.label-schema.vendor="Zammad" \
-      org.label-schema.schema-version="1.2" \
-      org.label-schema.docker.cmd="sysctl -w vm.max_map_count=262144;docker-compose up"
-
-# docker init
-COPY containers/zammad-nfs/docker-entrypoint.sh /
-RUN chmod +x /docker-entrypoint.sh
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["zammad-nfs"]

containers/zammad-nfs/docker-entrypoint.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [ "$1" = 'zammad-nfs' ]; then
-  echo "creating nfs dir"
-
-  test -d ${NFS_DIR}/data || mkdir -p ${NFS_DIR}/data
-  test -d ${NFS_DIR}/uploads || mkdir -p ${NFS_DIR}/uploads
-  chmod -R 777 ${NFS_DIR}
-
-  echo "create nfs exports"
-  echo "# NFS Export for Zammad" > /etc/exports
-  echo "${NFS_DIR} *(rw,sync,no_subtree_check,fsid=0,no_root_squash)" >> /etc/exports
-  echo "${NFS_DIR}/data *(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports
-  echo "${NFS_DIR}/uploads *(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports
-
-  exec runsvdir /etc/sv
-fi

containers/zammad-nginx/Dockerfile

@@ -1,19 +0,0 @@
-FROM nginx:alpine
-MAINTAINER Zammad <info@zammad.org>
-ARG BUILD_DATE
-
-LABEL org.label-schema.build-date="$BUILD_DATE" \
-      org.label-schema.name="Zammad" \
-      org.label-schema.license="AGPL-3.0" \
-      org.label-schema.description="Docker container for Zammad - Nginx" \
-      org.label-schema.url="https://zammad.org" \
-      org.label-schema.vcs-url="https://github.com/zammad/zammad" \
-      org.label-schema.vcs-type="Git" \
-      org.label-schema.vendor="Zammad" \
-      org.label-schema.schema-version="1.2" \
-      org.label-schema.docker.cmd="sysctl -w vm.max_map_count=262144;docker-compose up"
-
-ADD containers/zammad-nginx/nginx-zammad.conf /etc/nginx/conf.d/zammad.conf
-
-RUN rm /etc/nginx/conf.d/default.conf
-

containers/zammad-nginx/nginx-zammad.conf

@@ -1,49 +0,0 @@
-#
-# this is the nginx config for zammad
-#
-
-server_tokens off;
-
-upstream zammad-railsserver {
-    server zammad-railsserver:3000;
-}
-
-upstream zammad-websocket {
-    server zammad-websocket:6042;
-}
-
-server {
-    listen 80;
-
-    server_name _;
-
-    root /home/zammad/public;
-
-    client_max_body_size 50M;
-
-    location ~ ^/(assets/|robots.txt|humans.txt|favicon.ico) {
-        expires max;
-    }
-
-    location /ws {
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "Upgrade";
-        proxy_set_header CLIENT_IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_read_timeout 86400;
-        proxy_pass http://zammad-websocket;
-    }
-
-    location / {
-        proxy_set_header Host $http_host;
-        proxy_set_header CLIENT_IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_read_timeout 180;
-        proxy_pass http://zammad-railsserver;
-
-        gzip on;
-        gzip_types text/plain text/xml text/css image/svg+xml application/javascript application/x-javascript application/json application/xml;
-        gzip_proxied any;
-    }
-}

containers/zammad-postgresql/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:9.6-alpine
+FROM postgres:10.1-alpine
 MAINTAINER Zammad <info@zammad.org>
 ARG BUILD_DATE
 

containers/zammad/Dockerfile

@@ -24,7 +24,7 @@ LABEL org.label-schema.build-date="$BUILD_DATE" \
       org.label-schema.docker.cmd="sysctl -w vm.max_map_count=262144;docker-compose up"
 
 # install dependencies, GOSU & zammad
-RUN BUILD_DEPENDENCIES="git build-essential libffi-dev libpq5 libpq-dev nfs-common nginx rsync" \
+RUN BUILD_DEPENDENCIES="git build-essential libffi-dev libpq5 libpq-dev nginx rsync" \
     GOSU_DEPENDENCIES="ca-certificates wget" \
     set -ex \
 	  && apt-get update && apt-get install -y --force-yes --no-install-recommends ${BUILD_DEPENDENCIES} ${GOSU_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* \
@@ -46,6 +46,7 @@ RUN BUILD_DEPENDENCIES="git build-essential libffi-dev libpq5 libpq-dev nfs-comm
     && bundle exec rake assets:precompile \
     && sed -e 's#.*adapter: postgresql#  adapter: postgresql#g' -e 's#.*username:.*#  username: postgres#g' -e 's#.*password:.*#  password: \n  host: zammad-postgresql\n#g' < config/database.yml.pkgr > config/database.yml \
     && rm -r tmp/cache \
+    && sed -i -e "s/.*config.cache_store.*file_store.*cache_file_store.*/    config.cache_store = :dalli_store, 'zammad-memcached:11211'\n    config.session_store = :dalli_store, 'zammad-memcached:11211'/" config/application.rb \
     && sed -e 's#server localhost:3000#server zammad-railsserver:3000#g' -e 's#localhost:6042#zammad-websocket:6042#g' -e 's#server_name localhost#server_name _#g' -e 's#.*\(access\|error\)_log.*log;##g' < contrib/nginx/zammad.conf > /etc/nginx/sites-enabled/default \
     && chown -R ${ZAMMAD_USER}:${ZAMMAD_USER} ${ZAMMAD_TMP_DIR}
 

containers/zammad/docker-entrypoint.sh

@@ -9,14 +9,6 @@ function check_zammad_ready {
   done
 }
 
-function mount_nfs {
-  if [ -n "$(env|grep KUBERNETES)" ]; then
-    test -d ${ZAMMAD_DIR} || mkdir -p ${ZAMMAD_DIR}
-    mount -t nfs4 zammad-nfs:/data /opt/zammad
-    chown ${ZAMMAD_USER}:${ZAMMAD_USER} ${ZAMMAD_DIR}
-  fi
-}
-
 # zammad init
 if [ "$1" = 'zammad-init' ]; then
   until (echo > /dev/tcp/zammad-postgresql/5432) &> /dev/null; do
@@ -24,17 +16,12 @@ if [ "$1" = 'zammad-init' ]; then
     sleep 5
   done
 
-  mount_nfs
-
   # install / update zammad
-  rsync -a --delete --exclude 'storage/fs/*' --exclude 'public/assets/images/*' ${ZAMMAD_TMP_DIR}/ ${ZAMMAD_DIR}
+  rsync -av --delete --exclude 'storage/fs/*' --exclude 'public/assets/images/*' ${ZAMMAD_TMP_DIR}/ ${ZAMMAD_DIR}
-  rsync -a ${ZAMMAD_TMP_DIR}/public/assets/images/ ${ZAMMAD_DIR}/public/assets/images
+  rsync -av ${ZAMMAD_TMP_DIR}/public/assets/images/ ${ZAMMAD_DIR}/public/assets/images
 
   cd ${ZAMMAD_DIR}
 
-  # enable memcached
-  sed -i -e "s/.*config.cache_store.*file_store.*cache_file_store.*/    config.cache_store = :dalli_store, 'zammad-memcached:11211'\n    config.session_store = :dalli_store, 'zammad-memcached:11211'/" config/application.rb
-
   echo "initialising / updating database..."
   # db mirgrate
   set +e
@@ -70,12 +57,6 @@ fi
 
 # zammad nginx
 if [ "$1" = 'zammad-nginx' ]; then
-  mount_nfs
-
-  if [ -n "$(env|grep KUBERNETES)" ]; then
-    sed -i -e 's#server zammad-\(railsserver\|websocket\):#server zammad:#g' /etc/nginx/sites-enabled/default
-  fi
-
   until [ -f "${ZAMMAD_DIR}/${ZAMMAD_READY_FILE}" ] && [ -n "$(grep zammad-railsserver < ${ZAMMAD_DIR}/${ZAMMAD_READY_FILE})" ] && [ -n "$(grep zammad-scheduler < ${ZAMMAD_DIR}/${ZAMMAD_READY_FILE})" ] && [ -n "$(grep zammad-websocket < ${ZAMMAD_DIR}/${ZAMMAD_READY_FILE})" ] ; do
     echo "waiting for all zammad services to start..."
     sleep 5
@@ -91,8 +72,6 @@ fi
 
 # zammad-railsserver
 if [ "$1" = 'zammad-railsserver' ]; then
-  mount_nfs
-
   check_zammad_ready
 
   cd ${ZAMMAD_DIR}
@@ -107,8 +86,6 @@ fi
 
 # zammad-scheduler
 if [ "$1" = 'zammad-scheduler' ]; then
-  mount_nfs
-
   check_zammad_ready
 
   cd ${ZAMMAD_DIR}
@@ -123,8 +100,6 @@ fi
 
 # zammad-websocket
 if [ "$1" = 'zammad-websocket' ]; then
-  mount_nfs
-
   check_zammad_ready
 
   cd ${ZAMMAD_DIR}

hooks/config

@@ -3,4 +3,4 @@
 # build hooks config
 #
 
-DOCKER_IMAGES="zammad zammad-elasticsearch zammad-memcached zammad-nfs zammad-nginx zammad-postgresql"
+DOCKER_IMAGES="zammad zammad-elasticsearch zammad-memcached zammad-postgresql"

kubernetes/30_configmap_nginx.yaml

@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-configmap
+  namespace: zammad
+data:
+  config: |-
+    #
+    # kubernetes nginx config for zammad
+    #
+
+    server_tokens off;
+
+    upstream zammad-railsserver {
+        server localhost:3000;
+    }
+
+    upstream zammad-websocket {
+        server localhost:6042;
+    }
+
+    server {
+        listen 80;
+
+        server_name _;
+
+        root /opt/zammad/public;
+
+        client_max_body_size 50M;
+
+        location ~ ^/(assets/|robots.txt|humans.txt|favicon.ico) {
+            expires max;
+        }
+
+        location /ws {
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "Upgrade";
+            proxy_set_header CLIENT_IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_read_timeout 86400;
+            proxy_pass http://zammad-websocket;
+        }
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header CLIENT_IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_read_timeout 180;
+            proxy_pass http://zammad-railsserver;
+
+            gzip on;
+            gzip_types text/plain text/xml text/css image/svg+xml application/javascript application/x-javascript application/json application/xml;
+            gzip_proxied any;
+        }
+    }

kubernetes/40_deployment_zammad.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: zammad
 spec:
   replicas: 1
+  revisionHistoryLimit: 1
   template:
     metadata:
       labels:
@@ -12,47 +13,51 @@ spec:
         component: services
     spec:
       terminationGracePeriodSeconds: 10
-      # volumes:
+      volumes:
-      # - name: cache-volume
+        - name: zammad
-      #   emptyDir: {}
+          persistentVolumeClaim:
-      # - name: data
+            claimName: zammad
-      #   nfs:
+        - name: nginx-configmap
-      #     # FIXME: use the nfs service IP instead of 'zammad-nfs'
+          configMap:
-      #     server: zammad-nfs
+             name: nginx-configmap
-      #     path: /data
+             items:
-      # - name: uploads
+              - key: config
-      #   nfs:
+                path: default
-      #     # FIXME: use nfs service  IP instead of 'zammad-nfs'
-      #     server: zammad-nfs
-      #     path: /uploads
 
       initContainers:
       - name: zammad-init
         image: monotek/zammad-docker-compose:zammad
         args: [ "zammad-init" ]
         imagePullPolicy: Always
-        securityContext:
+        volumeMounts:
-          privileged: true
+         - name: zammad
-        # volumeMounts:
+           mountPath: /opt/zammad
-        # - name: cache-volume
-        #   mountPath: /opt/zammad/tmp
-        # - name: data
-        #   mountPath: /opt/zammad/storage
-        # - name: uploads
-        #   mountPath: /opt/zammad/assets/uploads
 
       containers:
-      - name: zammad-scheduler
+      - name: zammad-nginx
         image: monotek/zammad-docker-compose:zammad
-        args: [ "zammad-scheduler" ]
+        args: [ "zammad-nginx" ]
+        ports:
+        - containerPort: 80
+          name: nginx
         imagePullPolicy: Always
-        securityContext:
+        volumeMounts:
-          privileged: true
+         - name: zammad
-        # volumeMounts:
+           mountPath: /opt/zammad
-        # - name: cache-volume
+         - name: nginx-configmap
-        #   mountPath: /opt/zammad/tmp
+           mountPath: /etc/nginx/sites-enabled
-        # - name: data
+        readinessProbe:
-        #   mountPath: /opt/zammad/storage
+          httpGet:
+            path: /
+            port: 80
+          initialDelaySeconds: 60
+          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 80
+          initialDelaySeconds: 60
+          periodSeconds: 10
 
       - name: zammad-railsserver
         image: monotek/zammad-docker-compose:zammad
@@ -61,15 +66,29 @@ spec:
         ports:
         - name: railsserver
           containerPort: 3000
-        securityContext:
+        volumeMounts:
-          privileged: true
+         - name: zammad
-        # volumeMounts:
+           mountPath: /opt/zammad
-        # - name: cache-volume
+        readinessProbe:
-        #   mountPath: /opt/zammad/tmp
+          httpGet:
-        # - name: data
+            path: /
-        #   mountPath: /opt/zammad/storage
+            port: 3000
-        # - name: uploads
+          initialDelaySeconds: 90
-        #   mountPath: /opt/zammad/assets/uploads
+          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 3000
+          initialDelaySeconds: 90
+          periodSeconds: 10
+
+      - name: zammad-scheduler
+        image: monotek/zammad-docker-compose:zammad
+        args: [ "zammad-scheduler" ]
+        imagePullPolicy: Always
+        volumeMounts:
+         - name: zammad
+           mountPath: /opt/zammad
 
       - name: zammad-websocket
         image: monotek/zammad-docker-compose:zammad
@@ -78,10 +97,16 @@ spec:
         ports:
         - name: websocket
           containerPort: 6042
-        securityContext:
+        volumeMounts:
-          privileged: true  
+         - name: zammad
-        # volumeMounts:
+           mountPath: /opt/zammad
-        # - name: cache-volume
+        readinessProbe:
-        #   mountPath: /opt/zammad/tmp
+          tcpSocket:
-        # - name: data
+            port: 6042
-        #   mountPath: /opt/zammad/storage
+          initialDelaySeconds: 60
+          periodSeconds: 10
+        livenessProbe:
+          tcpSocket:
+            port: 6042
+          initialDelaySeconds: 60
+          periodSeconds: 10

kubernetes/41_deployment_nginx.yaml

@@ -1,33 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: zammad-nginx
-  namespace: zammad
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: nginx
-        component: webserver
-    spec:
-      terminationGracePeriodSeconds: 10
-#      volumes:
-#      - name: uploads
-#        nfs:
-#          # FIXME: use nfs service  IP instead of 'zammad-nfs'
-#          server: zammad-nfs
-#          path: /uploads
-      containers:
-      - name: zammad-nginx
-        image: monotek/zammad-docker-compose:zammad
-        args: [ "zammad-nginx" ]
-        ports:
-        - containerPort: 80
-          name: nginx
-        imagePullPolicy: Always
-        securityContext:
-          privileged: true
-#        volumeMounts:
-#        - name: uploads
-#          mountPath: /opt/zammad/assets/uploads

kubernetes/42_deployment_memcached.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: zammad
 spec:
   replicas: 1
+  revisionHistoryLimit: 1
   template:
     metadata:
       labels:
@@ -21,3 +22,13 @@ spec:
         - containerPort: 11211
           name: memcached
         imagePullPolicy: Always
+        readinessProbe:
+          tcpSocket:
+            port: 11211
+          initialDelaySeconds: 20
+          periodSeconds: 10
+        livenessProbe:
+          tcpSocket:
+            port: 11211
+          initialDelaySeconds: 10
+          periodSeconds: 10

kubernetes/43_deployment_postgesql.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: zammad
 spec:
   replicas: 1
+  revisionHistoryLimit: 1
   template:
     metadata:
       labels:
@@ -27,3 +28,13 @@ spec:
         volumeMounts:
         - name: zammad-postgresql
           mountPath: /var/lib/postgresql
+        readinessProbe:
+          tcpSocket:
+            port: 5432
+          initialDelaySeconds: 60
+          periodSeconds: 10
+        livenessProbe:
+          tcpSocket:
+            port: 5432
+          initialDelaySeconds: 60
+          periodSeconds: 10

kubernetes/45_deployment_nfs.yaml

@@ -1,33 +0,0 @@
-apiVersion: apps/v1beta1
-kind: Deployment
-metadata:
-  name: zammad-nfs
-  namespace: zammad
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        component: networkstorage
-        app: nfs
-    spec:
-      terminationGracePeriodSeconds: 10
-      volumes:
-      - name: zammad
-        persistentVolumeClaim:
-          claimName: zammad
-
-      containers:
-      - name: zammad-nfs
-        image: monotek/zammad-docker-compose:zammad-nfs
-        imagePullPolicy: Always
-        ports:
-        - name: nfsserver
-          containerPort: 2049
-        - name: rpc
-          containerPort: 111
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: zammad
-          mountPath: /exports

kubernetes/50_statefulset_elasticsearch.yaml

@@ -1,9 +1,13 @@
 apiVersion: apps/v1beta1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: zammad-elasticsearch
   namespace: zammad
+  labels:
+    app: elasticsearch
+    component: searchindex
 spec:
+  serviceName: zammad-elasticsearch
   replicas: 1
   template:
     metadata:
@@ -16,21 +20,31 @@ spec:
       - name: zammad-elasticsearch
         persistentVolumeClaim:
           claimName: zammad-elasticsearch
-
       initContainers:
       - name: sysctl
         image: busybox
         command: ["sysctl", "-w", "vm.max_map_count=262144"]
         securityContext:
           privileged: true
-
       containers:
       - name: zammad-elasticsearch
-        image: monotek/zammad-docker-compose:zammad-elasticsearch
+        image: zammad/zammad-docker-compose:zammad-elasticsearch
         ports:
-        - name: elasticsearch
+        - containerPort: 9200
-          containerPort: 9200
+          name: elasticsearch
         imagePullPolicy: Always
         volumeMounts:
-        - name: zammad-elasticsearch
+        - mountPath: /usr/share/elasticsearch/data
-          mountPath: /usr/share/elasticsearch/data
+          name: zammad-elasticsearch
+        readinessProbe:
+          httpGet:
+            path: /_cluster/health?local=true
+            port: 9200
+          initialDelaySeconds: 60
+          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /_cluster/health?local=true
+            port: 9200
+          initialDelaySeconds: 60
+          periodSeconds: 10

kubernetes/80_svc.yaml

@@ -9,10 +9,8 @@ metadata:
     component: services
 spec:
   ports:
-  - name: railsserver
+  - name: nginx
-    port: 3000
+    port: 80
-  - name: websocket
-    port: 6042
   selector:
     app: zammad
     component: services
@@ -54,44 +52,6 @@ spec:
     component: cache
   type: NodePort
 
----
-kind: Service
-apiVersion: v1
-metadata:
-  name: zammad-nfs
-  namespace: zammad
-  labels:
-    app: nfs
-    component: networkstorage
-spec:
-  ports:
-  - name: nfsserver
-    port: 2049
-  - name: rpc
-    port: 111
-  selector:
-    app: nfs
-    component: networkstorage
-  type: NodePort
-
----
-kind: Service
-apiVersion: v1
-metadata:
-  name: zammad-nginx
-  namespace: zammad
-  labels:
-    app: nginx
-    component: webserver
-spec:
-  ports:
-  - name: nginx
-    port: 80
-  selector:
-    app: nginx
-    component: webserver
-  type: NodePort
-
 ---
 kind: Service
 apiVersion: v1

kubernetes/90_ingress.yaml

@@ -4,8 +4,8 @@ metadata:
   name: zammad.example.com
   namespace: zammad
   labels:
-    component: webserver
     app: zammad
+    component: services
     host: zammad.example.com
   annotations:
     ingress.kubernetes.io/ssl-redirect: 'true'
@@ -22,5 +22,5 @@ spec:
         paths:
           - path: /
             backend:
-              serviceName: zammad-nginx
+              serviceName: zammad
               servicePort: 80

kubernetes/Readme.md

@@ -5,6 +5,7 @@ beta and not ready for production.
 
 ## Prerequisites
 
+- Kubernetes 1.8.x Cluster with at least on node, 2 CPUs and 4 GB of RAM
 - Change the ingress to your needs
 
 
@@ -29,21 +30,13 @@ beta and not ready for production.
 
 ### Install on Google Kubernetes Engine
 * connect to cluster via gcloud command
+* kubectl proxy
 * kubectl apply -f .
-
+* open dashboard in browser
+  * http://127.0.0.1:8001/ui
 
 
 ## If you want to help to improve the Kuberntes deployments here are some todos:
-* fix the nfs mount from entrypoint, which is currently used because of:
-  * https://github.com/kubernetes/kubernetes/issues/8735
-  * otherwise you have to manually:
-    * kubectl apply -f 00_namespace.yaml -f 80_svc.yaml
-    * kubectl --namespace=zammad describe services zammad-nfs
-      * use the NFS server IP to update
-        * 40_deployment_zammad.yaml
-        * 41_deployment_nginx.yaml
-    * kubectl apply -f 10_pvc.yaml -f 40_deployment_zammad.yaml -f 41_deployment_nginx.yaml -f 42_deployment_memcached.yaml -f 43_deployment_postgesql.yaml -f 44_deployment_elasticsearch.yaml -f 45_deployment_nfs.yaml -f 90_ingress.yaml
-* create config map for nginx
 * create a zammad helm chart
 * document steps to use existing helm charts for elasticsearch, postgresql and so on
 * add cpu & mem limits