From 174dd3146fbb03b0bee22fdb62853600a977b6b2 Mon Sep 17 00:00:00 2001
From: lemoer <git@irrelefant.net>
Date: Thu, 26 Sep 2019 16:12:09 +0200
Subject: [PATCH] contrib: Set up continuous integration through Jenkins

---
 contrib/ci/Jenkinsfile                        |  27 +++
 contrib/ci/jenkins-community-slave/Dockerfile |  33 ++++
 contrib/ci/jenkins-community-slave/README.md  |  32 ++++
 contrib/ci/jenkins-community-slave/slave.py   | 103 ++++++++++++
 contrib/ci/minimal-site/i18n                  |   1 +
 contrib/ci/minimal-site/modules               |   1 +
 contrib/ci/minimal-site/site.conf             | 154 ++++++++++++++++++
 contrib/ci/minimal-site/site.mk               |   1 +
 8 files changed, 352 insertions(+)
 create mode 100644 contrib/ci/Jenkinsfile
 create mode 100644 contrib/ci/jenkins-community-slave/Dockerfile
 create mode 100644 contrib/ci/jenkins-community-slave/README.md
 create mode 100644 contrib/ci/jenkins-community-slave/slave.py
 create mode 120000 contrib/ci/minimal-site/i18n
 create mode 120000 contrib/ci/minimal-site/modules
 create mode 100644 contrib/ci/minimal-site/site.conf
 create mode 120000 contrib/ci/minimal-site/site.mk

diff --git a/contrib/ci/Jenkinsfile b/contrib/ci/Jenkinsfile
new file mode 100644
index 00000000..5ecbe203
--- /dev/null
+++ b/contrib/ci/Jenkinsfile
@@ -0,0 +1,27 @@
+pipeline {
+    agent { label 'gluon-docker' }
+    environment {
+        GLUON_SITEDIR = "contrib/ci/minimal-site"
+        GLUON_TARGET = "x86-64"
+        BUILD_LOG = "1"
+    }
+    stages {
+        stage('lint') {
+            steps {
+                sh 'luacheck package scripts targets'
+            }
+        }
+        stage('docs') {
+            steps {
+                sh 'make -C docs html'
+            }
+        }
+        stage('build') {
+            steps {
+                sh 'make update'
+                sh 'test -d /dl_cache && ln -s /dl_cache openwrt/dl || true'
+                sh 'make -j$(nproc) V=s'
+            }
+        }
+    }
+}
diff --git a/contrib/ci/jenkins-community-slave/Dockerfile b/contrib/ci/jenkins-community-slave/Dockerfile
new file mode 100644
index 00000000..1ada00f9
--- /dev/null
+++ b/contrib/ci/jenkins-community-slave/Dockerfile
@@ -0,0 +1,33 @@
+FROM gluon
+
+USER root
+
+# this is needed to install default-jre-headless in debian slim images
+RUN mkdir -p /usr/share/man/man1
+ 
+RUN apt-get update && apt-get install -y default-jre-headless curl python3 python3-pip python3-sphinx git
+RUN pip3 install jenkins-webapi sphinx_rtd_theme
+ 
+# Get docker-compose in the agent container
+RUN mkdir -p /home/jenkins
+RUN mkdir -p /var/lib/jenkins
+RUN mkdir -p /remoting
+RUN chown gluon /home/jenkins
+RUN chown gluon /var/lib/jenkins
+RUN chown gluon /remoting
+ 
+# Start-up script to attach the slave to the master
+ADD slave.py /var/lib/jenkins/slave.py
+
+USER gluon
+ 
+WORKDIR /home/jenkins
+
+ENV JENKINS_URL "https://build.ffh.zone/"
+ENV JENKINS_SLAVE_ADDRESS ""
+ENV SLAVE_EXECUTORS "1"
+ENV SLAVE_LABELS "docker"
+ENV SLAVE_WORING_DIR ""
+ENV CLEAN_WORKING_DIR "true"
+ 
+CMD [ "python3", "-u", "/var/lib/jenkins/slave.py" ]
diff --git a/contrib/ci/jenkins-community-slave/README.md b/contrib/ci/jenkins-community-slave/README.md
new file mode 100644
index 00000000..aebc78e8
--- /dev/null
+++ b/contrib/ci/jenkins-community-slave/README.md
@@ -0,0 +1,32 @@
+# Gluon CI using Jenkins
+
+## Requirements
+- Only a host with docker.
+
+## Architecture
+
+![Screenshot from 2019-09-24 00-20-32](https://user-images.githubusercontent.com/601153/65468827-9edf2c80-de65-11e9-9fe0-56c3487719c3.png)
+
+## Installation
+You can support the gluon CI with your infrastructure:
+1. You need to query @lemoer (freifunk@irrelefant.net) for credentials.
+2. He will give you a `SLAVE_NAME` and a `SLAVE_SECRET` for your host.
+3. Then go to your docker host and substitute the values for  `SLAVE_NAME` and a `SLAVE_SECRET` in the following statements:
+``` shell
+git clone https://github.com/freifunk-gluon/gluon/
+cd gluon/contrib/ci/jenkins-community-slave/
+docker build -t gluon-jenkins .
+mkdir /var/cache/openwrt_dl_cache/
+docker run --detach --restart always \
+    -e "SLAVE_NAME=whoareyou" \
+    -e "SLAVE_SECRET=changeme" \
+    -v /var/cache/openwrt_dl_cache/:/dl_cache
+```
+4. Check whether the instance is running correctly:
+   - Your node should appear [here](https://build.ffh.zone/label/gluon-docker/).
+   - When clicking on it, Jenkins should state "Agent is connected." like here: 
+![Screenshot from 2019-09-24 01-00-52](https://user-images.githubusercontent.com/601153/65469209-dac6c180-de66-11e9-9d62-0d1c3b6b940b.png)
+5. **Your docker container needs to be rebuilt, when the build dependencies of gluon change. So please be aware of that and update your docker container in that case.** 
+
+## Backoff
+- If @lemoer is not reachable, please be patient at first if possible. Otherwise contact info@hannover.freifunk.net or join the channel `#freifunkh` on hackint.
diff --git a/contrib/ci/jenkins-community-slave/slave.py b/contrib/ci/jenkins-community-slave/slave.py
new file mode 100644
index 00000000..30455a87
--- /dev/null
+++ b/contrib/ci/jenkins-community-slave/slave.py
@@ -0,0 +1,103 @@
+from jenkins import Jenkins, JenkinsError, NodeLaunchMethod
+import os
+import signal
+import sys
+import urllib.request
+import subprocess
+import shutil
+import requests
+import time
+
+slave_jar = '/var/lib/jenkins/slave.jar'
+slave_name = os.environ['SLAVE_NAME'] if os.environ['SLAVE_NAME'] != '' else 'docker-slave-' + os.environ['HOSTNAME']
+jnlp_url = os.environ['JENKINS_URL'] + '/computer/' + slave_name + '/slave-agent.jnlp'
+slave_jar_url = os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar'
+print(slave_jar_url)
+process = None
+
+def clean_dir(dir):
+    for root, dirs, files in os.walk(dir):
+        for f in files:
+            os.unlink(os.path.join(root, f))
+        for d in dirs:
+            shutil.rmtree(os.path.join(root, d))
+
+def slave_create(node_name, working_dir, executors, labels):
+    j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS'])
+    j.node_create(node_name, working_dir, num_executors = int(executors), labels = labels, launcher = NodeLaunchMethod.JNLP)
+
+def slave_delete(node_name):
+    j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS'])
+    j.node_delete(node_name)
+
+def slave_download(target):
+    if os.path.isfile(slave_jar):
+        os.remove(slave_jar)
+
+    loader = urllib.request.URLopener()
+    loader.retrieve(os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar', '/var/lib/jenkins/slave.jar')
+
+def slave_run(slave_jar, jnlp_url):
+    params = [ 'java', '-jar', slave_jar, '-jnlpUrl', jnlp_url ]
+    if os.environ['JENKINS_SLAVE_ADDRESS'] != '':
+        params.extend([ '-connectTo', os.environ['JENKINS_SLAVE_ADDRESS' ] ])
+
+    if os.environ['SLAVE_SECRET'] == '':
+        params.extend([ '-jnlpCredentials', os.environ['JENKINS_USER'] + ':' + os.environ['JENKINS_PASS'] ])
+    else:
+        params.extend([ '-secret', os.environ['SLAVE_SECRET'] ])
+    return subprocess.Popen(params, stdout=subprocess.PIPE)
+
+def signal_handler(sig, frame):
+    if process != None:
+        process.send_signal(signal.SIGINT)
+
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+def h():
+    print("ERROR!: please specify environment variables")
+    print("")
+    print('docker run -e "SLAVE_NAME=test" -e "SLAVE_SECRET=..." jenkins')
+
+if os.environ.get('SLAVE_NAME') is None:
+    h()
+    sys.exit(1)
+
+if os.environ.get('SLAVE_SECRET') is None:
+    h()
+    sys.exit(1)
+
+def master_ready(url):
+    try:
+        r = requests.head(url, verify=False, timeout=None)
+        return r.status_code == requests.codes.ok
+    except:
+        return False
+
+while not master_ready(slave_jar_url):
+    print("Master not ready yet, sleeping for 10sec!")
+    time.sleep(10)
+
+slave_download(slave_jar)
+print('Downloaded Jenkins slave jar.')
+
+if os.environ['SLAVE_WORING_DIR']:
+    os.setcwd(os.environ['SLAVE_WORING_DIR'])
+
+if os.environ['CLEAN_WORKING_DIR'] == 'true':
+    clean_dir(os.getcwd())
+    print("Cleaned up working directory.")
+
+if os.environ['SLAVE_NAME'] == '':
+    slave_create(slave_name, os.getcwd(), os.environ['SLAVE_EXECUTORS'], os.environ['SLAVE_LABELS'])
+    print('Created temporary Jenkins slave.')
+
+process = slave_run(slave_jar, jnlp_url)
+print('Started Jenkins slave with name "' + slave_name + '" and labels [' + os.environ['SLAVE_LABELS'] + '].')
+process.wait()
+
+print('Jenkins slave stopped.')
+if os.environ['SLAVE_NAME'] == '':
+    slave_delete(slave_name)
+    print('Removed temporary Jenkins slave.')
diff --git a/contrib/ci/minimal-site/i18n b/contrib/ci/minimal-site/i18n
new file mode 120000
index 00000000..870b9aa3
--- /dev/null
+++ b/contrib/ci/minimal-site/i18n
@@ -0,0 +1 @@
+../../../docs/site-example/i18n/
\ No newline at end of file
diff --git a/contrib/ci/minimal-site/modules b/contrib/ci/minimal-site/modules
new file mode 120000
index 00000000..be20c51e
--- /dev/null
+++ b/contrib/ci/minimal-site/modules
@@ -0,0 +1 @@
+../../../docs/site-example/modules
\ No newline at end of file
diff --git a/contrib/ci/minimal-site/site.conf b/contrib/ci/minimal-site/site.conf
new file mode 100644
index 00000000..b7cb1ac4
--- /dev/null
+++ b/contrib/ci/minimal-site/site.conf
@@ -0,0 +1,154 @@
+-- This is an example site configuration for Gluon v2018.2+
+--
+-- Take a look at the documentation located at
+-- https://gluon.readthedocs.io/ for details.
+--
+-- This configuration will not work as is. You're required to make
+-- community specific changes to it!
+{
+  -- Used for generated hostnames, e.g. freifunk-abcdef123456. (optional)
+  -- hostname_prefix = 'freifunk-',
+
+  -- Name of the community.
+  site_name = 'Continious Integration',
+
+  -- Shorthand of the community.
+  site_code = 'ci',
+
+  -- 32 bytes of random data, encoded in hexadecimal
+  -- This data must be unique among all sites and domains!
+  -- Can be generated using: echo $(hexdump -v -n 32 -e '1/1 "%02x"' </dev/urandom)
+  domain_seed = 'e9608c4ff338b920992d629190e9ff11049de1dfc3f299eac07792dfbcda341c',
+
+  -- Prefixes used within the mesh.
+  -- prefix6 is required, prefix4 can be omitted if next_node.ip4
+  -- is not set.
+  prefix4 = '10.0.0.0/20',
+  prefix6 = 'fd::/64',
+
+  -- Timezone of your community.
+  -- See https://openwrt.org/docs/guide-user/base-system/system_configuration#time_zones
+  timezone = 'CET-1CEST,M3.5.0,M10.5.0/3',
+
+  -- List of NTP servers in your community.
+  -- Must be reachable using IPv6!
+  --  ntp_servers = {'1.ntp.services.ffxx'},
+
+  -- Wireless regulatory domain of your community.
+  regdom = 'DE',
+
+  -- Wireless configuration for 2.4 GHz interfaces.
+  wifi24 = {
+    -- Wireless channel.
+    channel = 1,
+
+    -- ESSID used for client network.
+    ap = {
+      ssid = 'gluon-ci-ssid',
+      -- disabled = true, -- (optional)
+    },
+
+    mesh = {
+      -- Adjust these values!
+      id = 'ueH3uXjdp', -- usually you don't want users to connect to this mesh-SSID, so use a cryptic id that no one will accidentally mistake for the client WiFi
+      mcast_rate = 12000,
+      -- disabled = true, -- (optional)
+    },
+  },
+
+  -- Wireless configuration for 5 GHz interfaces.
+  -- This should be equal to the 2.4 GHz variant, except
+  -- for channel.
+  wifi5 = {
+    channel = 44,
+    outdoor_chanlist = '100-140',
+    ap = {
+      ssid = 'gluon-ci-ssid',
+    },
+    mesh = {
+      -- Adjust these values!
+      id = 'ueH3uXjdp',
+      mcast_rate = 12000,
+    },
+  },
+
+
+  -- The next node feature allows clients to always reach the node it is
+  -- connected to using a known IP address.
+  next_node = {
+    -- anycast IPs of all nodes
+    -- name = { 'nextnode.location.community.example.org', 'nextnode', 'nn' },
+    ip4 = '10.0.0.1',
+    ip6 = 'fd::1',
+  },
+
+  mesh = {
+    vxlan = true,
+    batman_adv = {
+      routing_algo = 'BATMAN_IV'
+    }
+  },
+
+  mesh_vpn = {
+    -- enabled = true,
+    mtu = 1312,
+
+    fastd = {
+      -- Refer to https://fastd.readthedocs.io/en/latest/ to better understand
+      -- what these options do.
+
+      -- List of crypto-methods to use.
+      methods = {'salsa2012+umac'},
+      -- configurable = true,
+      -- syslog_level = 'warn',
+
+      groups = {
+        backbone = {
+          -- Limit number of connected peers to reduce bandwidth.
+          limit = 1,
+
+          -- List of peers.
+          peers = {
+          },
+
+        },
+      },
+    },
+
+    bandwidth_limit = {
+      -- The bandwidth limit can be enabled by default here.
+      enabled = false,
+
+      -- Default upload limit (kbit/s).
+      egress = 200,
+
+      -- Default download limit (kbit/s).
+      ingress = 3000,
+    },
+  },
+
+  autoupdater = {
+    -- Default branch. Don't forget to set GLUON_BRANCH when building!
+    branch = 'stable',
+
+    -- List of branches. You may define multiple branches.
+    branches = {
+      stable = {
+        name = 'stable',
+
+        -- List of mirrors to fetch images from. IPv6 required!
+        mirrors = {'http://1.updates.services.ffhl/stable/sysupgrade'},
+
+        -- Number of good signatures required.
+        -- Have multiple maintainers sign your build and only
+        -- accept it when a sufficient number of them have
+        -- signed it.
+        good_signatures = 2,
+
+        -- List of public keys of maintainers.
+        pubkeys = {
+        },
+      },
+    },
+  },
+}
diff --git a/contrib/ci/minimal-site/site.mk b/contrib/ci/minimal-site/site.mk
new file mode 120000
index 00000000..873f9cb7
--- /dev/null
+++ b/contrib/ci/minimal-site/site.mk
@@ -0,0 +1 @@
+../../../docs/site-example/site.mk
\ No newline at end of file