- switched to single zammad pod to get rid of nfs container in kubernetes

- added nginx configmap
- switched to statefulset for es
- removed nginx & nfs container builds
- switched to postgresql 10.1 (manual update needed)
This commit is contained in:
André Bauer 2017-12-02 12:58:46 +01:00
parent adf3e241b8
commit ce474aabef
18 changed files with 186 additions and 315 deletions

View File

@ -1,22 +0,0 @@
FROM cpuguy83/nfs-server
MAINTAINER Zanmmad <info@zammad.org>
ENV NFS_DIR /exports
ENV TMPFS_SIZE 256M
LABEL org.label-schema.build-date="$BUILD_DATE" \
org.label-schema.name="Zammad" \
org.label-schema.license="AGPL-3.0" \
org.label-schema.description="Docker container for Zammad - NFS Server Container for Kubernetes" \
org.label-schema.url="https://zammad.org" \
org.label-schema.vcs-url="https://github.com/zammad/zammad" \
org.label-schema.vcs-type="Git" \
org.label-schema.vendor="Zammad" \
org.label-schema.schema-version="1.2" \
org.label-schema.docker.cmd="sysctl -w vm.max_map_count=262144;docker-compose up"
# docker init
COPY containers/zammad-nfs/docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["zammad-nfs"]

View File

@ -1,19 +0,0 @@
#!/bin/bash
set -e
if [ "$1" = 'zammad-nfs' ]; then
echo "creating nfs dir"
test -d ${NFS_DIR}/data || mkdir -p ${NFS_DIR}/data
test -d ${NFS_DIR}/uploads || mkdir -p ${NFS_DIR}/uploads
chmod -R 777 ${NFS_DIR}
echo "create nfs exports"
echo "# NFS Export for Zammad" > /etc/exports
echo "${NFS_DIR} *(rw,sync,no_subtree_check,fsid=0,no_root_squash)" >> /etc/exports
echo "${NFS_DIR}/data *(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports
echo "${NFS_DIR}/uploads *(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports
exec runsvdir /etc/sv
fi

View File

@ -1,19 +0,0 @@
FROM nginx:alpine
MAINTAINER Zammad <info@zammad.org>
ARG BUILD_DATE
LABEL org.label-schema.build-date="$BUILD_DATE" \
org.label-schema.name="Zammad" \
org.label-schema.license="AGPL-3.0" \
org.label-schema.description="Docker container for Zammad - Nginx" \
org.label-schema.url="https://zammad.org" \
org.label-schema.vcs-url="https://github.com/zammad/zammad" \
org.label-schema.vcs-type="Git" \
org.label-schema.vendor="Zammad" \
org.label-schema.schema-version="1.2" \
org.label-schema.docker.cmd="sysctl -w vm.max_map_count=262144;docker-compose up"
ADD containers/zammad-nginx/nginx-zammad.conf /etc/nginx/conf.d/zammad.conf
RUN rm /etc/nginx/conf.d/default.conf

View File

@ -1,49 +0,0 @@
#
# this is the nginx config for zammad
#
server_tokens off;
upstream zammad-railsserver {
server zammad-railsserver:3000;
}
upstream zammad-websocket {
server zammad-websocket:6042;
}
server {
listen 80;
server_name _;
root /home/zammad/public;
client_max_body_size 50M;
location ~ ^/(assets/|robots.txt|humans.txt|favicon.ico) {
expires max;
}
location /ws {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header CLIENT_IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 86400;
proxy_pass http://zammad-websocket;
}
location / {
proxy_set_header Host $http_host;
proxy_set_header CLIENT_IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 180;
proxy_pass http://zammad-railsserver;
gzip on;
gzip_types text/plain text/xml text/css image/svg+xml application/javascript application/x-javascript application/json application/xml;
gzip_proxied any;
}
}

View File

@ -1,4 +1,4 @@
FROM postgres:9.6-alpine
FROM postgres:10.1-alpine
MAINTAINER Zammad <info@zammad.org>
ARG BUILD_DATE

View File

@ -24,7 +24,7 @@ LABEL org.label-schema.build-date="$BUILD_DATE" \
org.label-schema.docker.cmd="sysctl -w vm.max_map_count=262144;docker-compose up"
# install dependencies, GOSU & zammad
RUN BUILD_DEPENDENCIES="git build-essential libffi-dev libpq5 libpq-dev nfs-common nginx rsync" \
RUN BUILD_DEPENDENCIES="git build-essential libffi-dev libpq5 libpq-dev nginx rsync" \
GOSU_DEPENDENCIES="ca-certificates wget" \
set -ex \
&& apt-get update && apt-get install -y --force-yes --no-install-recommends ${BUILD_DEPENDENCIES} ${GOSU_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* \
@ -46,6 +46,7 @@ RUN BUILD_DEPENDENCIES="git build-essential libffi-dev libpq5 libpq-dev nfs-comm
&& bundle exec rake assets:precompile \
&& sed -e 's#.*adapter: postgresql# adapter: postgresql#g' -e 's#.*username:.*# username: postgres#g' -e 's#.*password:.*# password: \n host: zammad-postgresql\n#g' < config/database.yml.pkgr > config/database.yml \
&& rm -r tmp/cache \
&& sed -i -e "s/.*config.cache_store.*file_store.*cache_file_store.*/ config.cache_store = :dalli_store, 'zammad-memcached:11211'\n config.session_store = :dalli_store, 'zammad-memcached:11211'/" config/application.rb \
&& sed -e 's#server localhost:3000#server zammad-railsserver:3000#g' -e 's#localhost:6042#zammad-websocket:6042#g' -e 's#server_name localhost#server_name _#g' -e 's#.*\(access\|error\)_log.*log;##g' < contrib/nginx/zammad.conf > /etc/nginx/sites-enabled/default \
&& chown -R ${ZAMMAD_USER}:${ZAMMAD_USER} ${ZAMMAD_TMP_DIR}

View File

@ -9,14 +9,6 @@ function check_zammad_ready {
done
}
function mount_nfs {
if [ -n "$(env|grep KUBERNETES)" ]; then
test -d ${ZAMMAD_DIR} || mkdir -p ${ZAMMAD_DIR}
mount -t nfs4 zammad-nfs:/data /opt/zammad
chown ${ZAMMAD_USER}:${ZAMMAD_USER} ${ZAMMAD_DIR}
fi
}
# zammad init
if [ "$1" = 'zammad-init' ]; then
until (echo > /dev/tcp/zammad-postgresql/5432) &> /dev/null; do
@ -24,17 +16,12 @@ if [ "$1" = 'zammad-init' ]; then
sleep 5
done
mount_nfs
# install / update zammad
rsync -a --delete --exclude 'storage/fs/*' --exclude 'public/assets/images/*' ${ZAMMAD_TMP_DIR}/ ${ZAMMAD_DIR}
rsync -a ${ZAMMAD_TMP_DIR}/public/assets/images/ ${ZAMMAD_DIR}/public/assets/images
rsync -av --delete --exclude 'storage/fs/*' --exclude 'public/assets/images/*' ${ZAMMAD_TMP_DIR}/ ${ZAMMAD_DIR}
rsync -av ${ZAMMAD_TMP_DIR}/public/assets/images/ ${ZAMMAD_DIR}/public/assets/images
cd ${ZAMMAD_DIR}
# enable memcached
sed -i -e "s/.*config.cache_store.*file_store.*cache_file_store.*/ config.cache_store = :dalli_store, 'zammad-memcached:11211'\n config.session_store = :dalli_store, 'zammad-memcached:11211'/" config/application.rb
echo "initialising / updating database..."
# db mirgrate
set +e
@ -70,12 +57,6 @@ fi
# zammad nginx
if [ "$1" = 'zammad-nginx' ]; then
mount_nfs
if [ -n "$(env|grep KUBERNETES)" ]; then
sed -i -e 's#server zammad-\(railsserver\|websocket\):#server zammad:#g' /etc/nginx/sites-enabled/default
fi
until [ -f "${ZAMMAD_DIR}/${ZAMMAD_READY_FILE}" ] && [ -n "$(grep zammad-railsserver < ${ZAMMAD_DIR}/${ZAMMAD_READY_FILE})" ] && [ -n "$(grep zammad-scheduler < ${ZAMMAD_DIR}/${ZAMMAD_READY_FILE})" ] && [ -n "$(grep zammad-websocket < ${ZAMMAD_DIR}/${ZAMMAD_READY_FILE})" ] ; do
echo "waiting for all zammad services to start..."
sleep 5
@ -91,8 +72,6 @@ fi
# zammad-railsserver
if [ "$1" = 'zammad-railsserver' ]; then
mount_nfs
check_zammad_ready
cd ${ZAMMAD_DIR}
@ -107,8 +86,6 @@ fi
# zammad-scheduler
if [ "$1" = 'zammad-scheduler' ]; then
mount_nfs
check_zammad_ready
cd ${ZAMMAD_DIR}
@ -123,8 +100,6 @@ fi
# zammad-websocket
if [ "$1" = 'zammad-websocket' ]; then
mount_nfs
check_zammad_ready
cd ${ZAMMAD_DIR}

View File

@ -3,4 +3,4 @@
# build hooks config
#
DOCKER_IMAGES="zammad zammad-elasticsearch zammad-memcached zammad-nfs zammad-nginx zammad-postgresql"
DOCKER_IMAGES="zammad zammad-elasticsearch zammad-memcached zammad-postgresql"

View File

@ -0,0 +1,56 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: nginx-configmap
namespace: zammad
data:
config: |-
#
# kubernetes nginx config for zammad
#
server_tokens off;
upstream zammad-railsserver {
server localhost:3000;
}
upstream zammad-websocket {
server localhost:6042;
}
server {
listen 80;
server_name _;
root /opt/zammad/public;
client_max_body_size 50M;
location ~ ^/(assets/|robots.txt|humans.txt|favicon.ico) {
expires max;
}
location /ws {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header CLIENT_IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 86400;
proxy_pass http://zammad-websocket;
}
location / {
proxy_set_header Host $http_host;
proxy_set_header CLIENT_IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 180;
proxy_pass http://zammad-railsserver;
gzip on;
gzip_types text/plain text/xml text/css image/svg+xml application/javascript application/x-javascript application/json application/xml;
gzip_proxied any;
}
}

View File

@ -5,6 +5,7 @@ metadata:
namespace: zammad
spec:
replicas: 1
revisionHistoryLimit: 1
template:
metadata:
labels:
@ -12,47 +13,51 @@ spec:
component: services
spec:
terminationGracePeriodSeconds: 10
# volumes:
# - name: cache-volume
# emptyDir: {}
# - name: data
# nfs:
# # FIXME: use the nfs service IP instead of 'zammad-nfs'
# server: zammad-nfs
# path: /data
# - name: uploads
# nfs:
# # FIXME: use nfs service IP instead of 'zammad-nfs'
# server: zammad-nfs
# path: /uploads
volumes:
- name: zammad
persistentVolumeClaim:
claimName: zammad
- name: nginx-configmap
configMap:
name: nginx-configmap
items:
- key: config
path: default
initContainers:
- name: zammad-init
image: monotek/zammad-docker-compose:zammad
args: [ "zammad-init" ]
imagePullPolicy: Always
securityContext:
privileged: true
# volumeMounts:
# - name: cache-volume
# mountPath: /opt/zammad/tmp
# - name: data
# mountPath: /opt/zammad/storage
# - name: uploads
# mountPath: /opt/zammad/assets/uploads
volumeMounts:
- name: zammad
mountPath: /opt/zammad
containers:
- name: zammad-scheduler
- name: zammad-nginx
image: monotek/zammad-docker-compose:zammad
args: [ "zammad-scheduler" ]
args: [ "zammad-nginx" ]
ports:
- containerPort: 80
name: nginx
imagePullPolicy: Always
securityContext:
privileged: true
# volumeMounts:
# - name: cache-volume
# mountPath: /opt/zammad/tmp
# - name: data
# mountPath: /opt/zammad/storage
volumeMounts:
- name: zammad
mountPath: /opt/zammad
- name: nginx-configmap
mountPath: /etc/nginx/sites-enabled
readinessProbe:
httpGet:
path: /
port: 80
initialDelaySeconds: 60
periodSeconds: 10
livenessProbe:
httpGet:
path: /
port: 80
initialDelaySeconds: 60
periodSeconds: 10
- name: zammad-railsserver
image: monotek/zammad-docker-compose:zammad
@ -61,15 +66,29 @@ spec:
ports:
- name: railsserver
containerPort: 3000
securityContext:
privileged: true
# volumeMounts:
# - name: cache-volume
# mountPath: /opt/zammad/tmp
# - name: data
# mountPath: /opt/zammad/storage
# - name: uploads
# mountPath: /opt/zammad/assets/uploads
volumeMounts:
- name: zammad
mountPath: /opt/zammad
readinessProbe:
httpGet:
path: /
port: 3000
initialDelaySeconds: 90
periodSeconds: 10
livenessProbe:
httpGet:
path: /
port: 3000
initialDelaySeconds: 90
periodSeconds: 10
- name: zammad-scheduler
image: monotek/zammad-docker-compose:zammad
args: [ "zammad-scheduler" ]
imagePullPolicy: Always
volumeMounts:
- name: zammad
mountPath: /opt/zammad
- name: zammad-websocket
image: monotek/zammad-docker-compose:zammad
@ -78,10 +97,16 @@ spec:
ports:
- name: websocket
containerPort: 6042
securityContext:
privileged: true
# volumeMounts:
# - name: cache-volume
# mountPath: /opt/zammad/tmp
# - name: data
# mountPath: /opt/zammad/storage
volumeMounts:
- name: zammad
mountPath: /opt/zammad
readinessProbe:
tcpSocket:
port: 6042
initialDelaySeconds: 60
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 6042
initialDelaySeconds: 60
periodSeconds: 10

View File

@ -1,33 +0,0 @@
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: zammad-nginx
namespace: zammad
spec:
replicas: 1
template:
metadata:
labels:
app: nginx
component: webserver
spec:
terminationGracePeriodSeconds: 10
# volumes:
# - name: uploads
# nfs:
# # FIXME: use nfs service IP instead of 'zammad-nfs'
# server: zammad-nfs
# path: /uploads
containers:
- name: zammad-nginx
image: monotek/zammad-docker-compose:zammad
args: [ "zammad-nginx" ]
ports:
- containerPort: 80
name: nginx
imagePullPolicy: Always
securityContext:
privileged: true
# volumeMounts:
# - name: uploads
# mountPath: /opt/zammad/assets/uploads

View File

@ -5,6 +5,7 @@ metadata:
namespace: zammad
spec:
replicas: 1
revisionHistoryLimit: 1
template:
metadata:
labels:
@ -21,3 +22,13 @@ spec:
- containerPort: 11211
name: memcached
imagePullPolicy: Always
readinessProbe:
tcpSocket:
port: 11211
initialDelaySeconds: 20
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 11211
initialDelaySeconds: 10
periodSeconds: 10

View File

@ -5,6 +5,7 @@ metadata:
namespace: zammad
spec:
replicas: 1
revisionHistoryLimit: 1
template:
metadata:
labels:
@ -27,3 +28,13 @@ spec:
volumeMounts:
- name: zammad-postgresql
mountPath: /var/lib/postgresql
readinessProbe:
tcpSocket:
port: 5432
initialDelaySeconds: 60
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 5432
initialDelaySeconds: 60
periodSeconds: 10

View File

@ -1,33 +0,0 @@
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: zammad-nfs
namespace: zammad
spec:
replicas: 1
template:
metadata:
labels:
component: networkstorage
app: nfs
spec:
terminationGracePeriodSeconds: 10
volumes:
- name: zammad
persistentVolumeClaim:
claimName: zammad
containers:
- name: zammad-nfs
image: monotek/zammad-docker-compose:zammad-nfs
imagePullPolicy: Always
ports:
- name: nfsserver
containerPort: 2049
- name: rpc
containerPort: 111
securityContext:
privileged: true
volumeMounts:
- name: zammad
mountPath: /exports

View File

@ -1,9 +1,13 @@
apiVersion: apps/v1beta1
kind: Deployment
kind: StatefulSet
metadata:
name: zammad-elasticsearch
namespace: zammad
labels:
app: elasticsearch
component: searchindex
spec:
serviceName: zammad-elasticsearch
replicas: 1
template:
metadata:
@ -16,21 +20,31 @@ spec:
- name: zammad-elasticsearch
persistentVolumeClaim:
claimName: zammad-elasticsearch
initContainers:
- name: sysctl
image: busybox
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
containers:
- name: zammad-elasticsearch
image: monotek/zammad-docker-compose:zammad-elasticsearch
image: zammad/zammad-docker-compose:zammad-elasticsearch
ports:
- name: elasticsearch
containerPort: 9200
- containerPort: 9200
name: elasticsearch
imagePullPolicy: Always
volumeMounts:
- name: zammad-elasticsearch
mountPath: /usr/share/elasticsearch/data
- mountPath: /usr/share/elasticsearch/data
name: zammad-elasticsearch
readinessProbe:
httpGet:
path: /_cluster/health?local=true
port: 9200
initialDelaySeconds: 60
periodSeconds: 10
livenessProbe:
httpGet:
path: /_cluster/health?local=true
port: 9200
initialDelaySeconds: 60
periodSeconds: 10

View File

@ -9,10 +9,8 @@ metadata:
component: services
spec:
ports:
- name: railsserver
port: 3000
- name: websocket
port: 6042
- name: nginx
port: 80
selector:
app: zammad
component: services
@ -54,44 +52,6 @@ spec:
component: cache
type: NodePort
---
kind: Service
apiVersion: v1
metadata:
name: zammad-nfs
namespace: zammad
labels:
app: nfs
component: networkstorage
spec:
ports:
- name: nfsserver
port: 2049
- name: rpc
port: 111
selector:
app: nfs
component: networkstorage
type: NodePort
---
kind: Service
apiVersion: v1
metadata:
name: zammad-nginx
namespace: zammad
labels:
app: nginx
component: webserver
spec:
ports:
- name: nginx
port: 80
selector:
app: nginx
component: webserver
type: NodePort
---
kind: Service
apiVersion: v1

View File

@ -4,8 +4,8 @@ metadata:
name: zammad.example.com
namespace: zammad
labels:
component: webserver
app: zammad
component: services
host: zammad.example.com
annotations:
ingress.kubernetes.io/ssl-redirect: 'true'
@ -22,5 +22,5 @@ spec:
paths:
- path: /
backend:
serviceName: zammad-nginx
serviceName: zammad
servicePort: 80

View File

@ -5,6 +5,7 @@ beta and not ready for production.
## Prerequisites
- Kubernetes 1.8.x Cluster with at least on node, 2 CPUs and 4 GB of RAM
- Change the ingress to your needs
@ -29,21 +30,13 @@ beta and not ready for production.
### Install on Google Kubernetes Engine
* connect to cluster via gcloud command
* kubectl proxy
* kubectl apply -f .
* open dashboard in browser
* http://127.0.0.1:8001/ui
## If you want to help to improve the Kuberntes deployments here are some todos:
* fix the nfs mount from entrypoint, which is currently used because of:
* https://github.com/kubernetes/kubernetes/issues/8735
* otherwise you have to manually:
* kubectl apply -f 00_namespace.yaml -f 80_svc.yaml
* kubectl --namespace=zammad describe services zammad-nfs
* use the NFS server IP to update
* 40_deployment_zammad.yaml
* 41_deployment_nginx.yaml
* kubectl apply -f 10_pvc.yaml -f 40_deployment_zammad.yaml -f 41_deployment_nginx.yaml -f 42_deployment_memcached.yaml -f 43_deployment_postgesql.yaml -f 44_deployment_elasticsearch.yaml -f 45_deployment_nfs.yaml -f 90_ingress.yaml
* create config map for nginx
* create a zammad helm chart
* document steps to use existing helm charts for elasticsearch, postgresql and so on
* add cpu & mem limits