Merge pull request #2 from FreifunkBremen/master

Updaten auf upstream

.circleci/check-coverage

@@ -2,14 +2,16 @@
 # Issue:  https://github.com/mattn/goveralls/issues/20
 # Source: https://github.com/uber/go-torch/blob/63da5d33a225c195fea84610e2456d5f722f3963/.test-cover.sh
 
+cd ${GOPATH}/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}
+
 echo "mode: count" > profile.cov
 FAIL=0
 
 # Standard go tooling behavior is to ignore dirs with leading underscors
-for dir in $(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d);
+for dir in $(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d | grep -v '^./vendor/');
 do
   if ls $dir/*.go &> /dev/null; then
-    go test -p 1 -v -covermode=count -coverprofile=profile.tmp $dir || FAIL=$?
+    go test -v -covermode=count -coverprofile=profile.tmp $dir || FAIL=$?
     if [ -f profile.tmp ]
     then
       tail -n +2 < profile.tmp >> profile.cov
@@ -19,8 +21,7 @@ do
 done
 
 # Failures have incomplete results, so don't send
-if [ "$FAIL" -eq 0 ]; then
-  goveralls -service=travis-ci -v -coverprofile=profile.cov
-fi
+[ "$FAIL" -ne 0 ] && exit 1
 
-exit $FAIL
+goveralls -service=circle-ci -v -coverprofile=profile.cov
+bash <(curl -s https://codecov.io/bash) -t $CODECOV_TOKEN -f profile.cov

.circleci/check-gofmt

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+result="$(gofmt -s -l . | grep -v '^vendor/' )"
+if [ -n "$result" ]; then
+  echo "Go code is not formatted, run 'gofmt -s -w .'" >&2
+  echo "$result"
+  exit 1
+fi

.circleci/check-testfiles

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# checks if every desired package has test files
+
+import os
+import re
+import sys
+
+source_re = re.compile(".*\.go")
+test_re   = re.compile(".*_test\.go")
+missing   = False
+
+for root, dirs, files in os.walk("."):
+  # ignore some paths
+  if root == "." or root == "./database/graphite" or root.startswith("./vendor") or root.startswith("./."):
+    continue
+
+  # source files but not test files?
+  if len(filter(source_re.match, files)) > 0 and len(filter(test_re.match, files)) == 0:
+    print("no test files for {}".format(root))
+    missing = True
+
+if missing:
+  sys.exit(1)
+else:
+  print("every package has test files")

.circleci/config.yml

@@ -0,0 +1,46 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/golang:latest
+    working_directory: /go/src/github.com/FreifunkBremen/yanic
+    steps:
+      - checkout
+      - run: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+      - run: dep ensure
+      - run: go install -ldflags "-X github.com/FreifunkBremen/yanic/cmd.VERSION=`git -C $GOPATH/src/github.com/FreifunkBremen/yanic rev-parse HEAD`" github.com/FreifunkBremen/yanic
+      - store_artifacts:
+          path: /go/bin/
+          destination: yanic
+  test:
+    docker:
+      - image: circleci/golang:latest
+    working_directory: /go/src/github.com/FreifunkBremen/yanic
+    steps:
+      - checkout
+      - run: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+      - run: dep ensure
+      - run: go get github.com/mattn/goveralls
+      - run: go get golang.org/x/tools/cmd/cover
+      - run: ./.circleci/check-coverage
+      - store_test_results:
+          path: ./
+          destination: profile.cov
+      - run: ./.circleci/check-gofmt
+      - run: ./.circleci/check-testfiles
+  test_race:
+    docker:
+      - image: circleci/golang:latest
+    working_directory: /go/src/github.com/FreifunkBremen/yanic
+    steps:
+      - checkout
+      - run: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+      - run: dep ensure
+      - run: go test -race ./...
+workflows:
+  version: 2
+  build_and_tests:
+    jobs:
+      - build
+      - test
+      - test_race

.drone.yml

@@ -0,0 +1,27 @@
+workspace:
+  base: /go
+  path: src/github.com/FreifunkBremen/yanic
+
+pipeline:
+  build:
+    image: golang:latest
+    commands:
+      - go get ./...
+      - go build
+  codestyle:
+    image: golang:latest
+    commands:
+      - go get github.com/client9/misspell/cmd/misspell
+      - misspell -error .
+      - if [ -n "$(gofmt -s -l .)" ]; then echo "Go code is not formatted, run 'gofmt -s -w .'" >&2; exit 1; fi
+  test:
+    image: golang:latest
+    commands:
+      - go get github.com/stretchr/testify/assert
+      - go test ./... -v -cover
+  test-race:
+    image: golang:latest
+    commands:
+      - go get github.com/stretchr/testify/assert
+      - go test ./... -v -race
+

.editorconfig

@@ -0,0 +1,17 @@
+; http://editorconfig.org/
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+trim_trailing_whitespace = true
+
+[*.go]
+indent_style = tab
+indent_size = 4
+
+[*.{json,yml}]
+indent_style = space
+indent_size = 2

.github/CONTRIBUTING.md

@@ -0,0 +1,5 @@
+## Contributing is welcome
+
+Pull requests are welcome without the need of opening an issue. If you're unsure
+about your feature or your implementation open an issue and discuss your
+suggested changes.

.github/ISSUE_TEMPLATE/community.md

@@ -0,0 +1,7 @@
+
+## Checklist:
+<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
+<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
+- [ ] I mentioned the community
+- [ ] I mentioned the database type
+- [ ] I added a link to the visualization of the collected data

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+<!--- Use a prefix like [TASK], [BUGFIX], [DOC] or [TEST] and provide a general summary of your changes in the Title above -->
+
+## Description
+<!--- Describe your changes -->
+
+## Motivation and Context
+<!--- Why is this change required? What problem does it solve? -->
+<!--- If it fixes an open issue, please link to the issue here. -->
+
+## Checklist:
+<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
+<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
+- [ ] My code follows the code style of this project.
+- [ ] I have added also tests for my new code.
+- [ ] My change requires a change to the documentation.
+- [ ] I have updated the documentation accordingly.

.gitignore

@@ -24,3 +24,5 @@ _testmain.go
 *.prof
 webroot
 /config.toml
+/vendor
+/bin

.gitlab-ci.yml

@@ -0,0 +1,36 @@
+image: golang:latest
+stages:
+  - build
+  - test
+
+before_script:
+  - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+  - mkdir -p /go/src/github.com/FreifunkBremen/
+  - cp -R /builds/freifunkbremen/yanic /go/src/github.com/FreifunkBremen/yanic
+  - cd /go/src/github.com/FreifunkBremen/yanic
+  - dep ensure
+
+build-my-project:
+  stage: build
+  script:
+    - go install -ldflags "-X github.com/FreifunkBremen/yanic/cmd.VERSION=`git -C $GOPATH/src/github.com/FreifunkBremen/yanic rev-parse HEAD`" github.com/FreifunkBremen/yanic
+    - mkdir /builds/freifunkbremen/yanic/bin/
+    - cp /go/bin/yanic /builds/freifunkbremen/yanic/bin/yanic
+  artifacts:
+    paths:
+      - bin/yanic
+      - config_example.toml
+      - config-respondd_example.toml
+
+test-my-project:
+  stage: test
+  script:
+    - ./.circleci/check-gofmt
+    - ./.circleci/check-testfiles
+    - go test $(go list ./... | grep -v /vendor/) -v -coverprofile .testCoverage.txt
+    - go tool cover -func=.testCoverage.txt
+
+test-race-my-project:
+  stage: test
+  script:
+    - go test -race ./...

.gitlab/issue_templates/default.md

@@ -0,0 +1,7 @@
+
+## Checklist:
+<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
+<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
+- [ ] I mentioned the community
+- [ ] I mentioned the database type
+- [ ] I added a link to the visualization of the collected data

.travis.yml

@@ -1,12 +0,0 @@
-language: go
-go:
-  - 1.8.x
-  - master
-install:
-  - go get -t github.com/FreifunkBremen/yanic/...
-  - go get github.com/mattn/goveralls
-  - go get golang.org/x/tools/cmd/cover
-script:
-  - ./.test-coverage
-  - go install github.com/FreifunkBremen/yanic/cmd/yanic
-  - go install github.com/FreifunkBremen/yanic/cmd/yanic-query

Gopkg.lock

@@ -0,0 +1,167 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:af6e785bedb62fc2abb81977c58a7a44e5cf9f7e41b8d3c8dd4d872edea0ce08"
+  name = "github.com/NYTimes/gziphandler"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "dd0439581c7657cb652dfe5c71d7d48baf39541d"
+  version = "v1.1.1"
+
+[[projects]]
+  digest = "1:616fe3fc0d5eb7b89324dc5e47aef8cf05f3b0aed0bfe908e62516da4c638647"
+  name = "github.com/bdlm/log"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "b8b76dd35f1d7fdf6a49497c8ac22e6554efffed"
+  version = "v0.1.18"
+
+[[projects]]
+  branch = "master"
+  digest = "1:9da00d14e7f20473d129be028aa99a2f47859aed17e58cd24c5108df76ad3959"
+  name = "github.com/bdlm/std"
+  packages = ["logger"]
+  pruneopts = "UT"
+  revision = "fd3b596111c78b7d14f1e1308ebdb1153013f1a8"
+
+[[projects]]
+  digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec"
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  pruneopts = "UT"
+  revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
+  version = "v1.1.1"
+
+[[projects]]
+  digest = "1:6bea4bd16434bc6699590ed025152b690d08f552eeabfdc3258bd3aff738958c"
+  name = "github.com/fgrosse/graphigo"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "55a0a92a703041a55ad5ee2c2647f9577a87fdc1"
+  version = "v2"
+
+[[projects]]
+  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:50708c8fc92aec981df5c446581cf9f90ba9e2a5692118e0ce75d4534aaa14a2"
+  name = "github.com/influxdata/influxdb1-client"
+  packages = [
+    "models",
+    "pkg/escape",
+    "v2",
+  ]
+  pruneopts = "UT"
+  revision = "8ff2fc3824fcb533795f9a2f233275f0bb18d6c5"
+
+[[projects]]
+  digest = "1:b56c589214f01a5601e0821387db484617392d0042f26234bf2da853a2f498a1"
+  name = "github.com/naoina/go-stringutil"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "6b638e95a32d0c1131db0e7fe83775cbea4a0d0b"
+  version = "v0.1.0"
+
+[[projects]]
+  digest = "1:f58c3d0e46b64878d00652fedba24ee879725191ab919dca7b62586859281c04"
+  name = "github.com/naoina/toml"
+  packages = [
+    ".",
+    "ast",
+  ]
+  pruneopts = "UT"
+  revision = "e6f5723bf2a66af014955e0888881314cf294129"
+  version = "v0.1.1"
+
+[[projects]]
+  digest = "1:36cfc0b6facbc68c92a4d55f32eff22b872a899ab75c384ac24a08dd33bc1380"
+  name = "github.com/paulmach/go.geojson"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "a5c451da2d9c67c590ae6be92565b67698286a1a"
+  version = "v1.4"
+
+[[projects]]
+  digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b"
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
+
+[[projects]]
+  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  pruneopts = "UT"
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:22799aea8fe96dd5693abdd1eaa14b1b29e3eafbdc7733fa155b3cb556c8a7ae"
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "67fc4837d267bc9bfd6e47f77783fcc3dffc68de"
+  version = "v0.0.4"
+
+[[projects]]
+  digest = "1:c1b1102241e7f645bc8e0c22ae352e8f0dc6484b6cb4d132fa9f24174e0119e2"
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "298182f68c66c05229eb03ac171abe6e309ee79a"
+  version = "v1.0.3"
+
+[[projects]]
+  digest = "1:972c2427413d41a1e06ca4897e8528e5a1622894050e2f527b38ddf0f343f759"
+  name = "github.com/stretchr/testify"
+  packages = ["assert"]
+  pruneopts = "UT"
+  revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053"
+  version = "v1.3.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:bbe51412d9915d64ffaa96b51d409e070665efc5194fcf145c4a27d4133107a4"
+  name = "golang.org/x/crypto"
+  packages = ["ssh/terminal"]
+  pruneopts = "UT"
+  revision = "22d7a77e9e5f409e934ed268692e56707cd169e5"
+
+[[projects]]
+  branch = "master"
+  digest = "1:160e165ef13917405d1ebfde865cb9c292b8635f83655e3932662d5586d300ee"
+  name = "golang.org/x/sys"
+  packages = [
+    "unix",
+    "windows",
+  ]
+  pruneopts = "UT"
+  revision = "c432e742b0af385916e013f6a34e9e73d139cf82"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/NYTimes/gziphandler",
+    "github.com/bdlm/log",
+    "github.com/bdlm/std/logger",
+    "github.com/fgrosse/graphigo",
+    "github.com/influxdata/influxdb1-client/models",
+    "github.com/influxdata/influxdb1-client/v2",
+    "github.com/naoina/toml",
+    "github.com/paulmach/go.geojson",
+    "github.com/pkg/errors",
+    "github.com/spf13/cobra",
+    "github.com/stretchr/testify/assert",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,54 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/NYTimes/gziphandler"
+  version = "1.0.1"
+
+[[constraint]]
+  name = "github.com/fgrosse/graphigo"
+  version = "2.0.0"
+
+[[constraint]]
+  name = "github.com/naoina/toml"
+  version = "0.1.1"
+
+[[constraint]]
+  name = "github.com/pkg/errors"
+  version = "0.8.0"
+
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "0.0.2"
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "1.2.1"
+
+[prune]
+  go-tests = true
+  unused-packages = true

INSTALL.md

@@ -1,61 +1,86 @@
 # Howto install Yanic
 
 ## go
+
 ### Install
 ```sh
 cd /usr/local/
-wget https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz
-tar xvf go1.8.linux-amd64.tar.gz
-rm go1.8.linux-amd64.tar.gz
+wget https://dl.google.com/go/go1.13.1.linux-amd64.tar.gz -O go-release-linux-amd64.tar.gz
+tar xvf go-release-linux-amd64.tar.gz
+rm go-release-linux-amd64.tar.gz
 ```
 
-### Configurate
-put this lines into a shell place at root:
+### Configure go
+Add these lines in your root shell startup file (e.g. `/root/.bashrc`):
 ```sh
 export GOPATH=/opt/go
 export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin
 ```
 
-put this lines also into a shell place to use go by normal user:
-```sh
-export GOPATH=~/go
-export PATH=$PATH:$GOPATH/bin
-```
-
 ## Yanic
 
 ### Compile
+As root:
 ```sh
-go get -v -u github.com/FreifunkBremen/yanic/cmd/...
+go get -v -u github.com/FreifunkBremen/yanic
 ```
 
 #### Work with other databases
-If you did like a other database solution like influxdb,
-you are welcome to create another subpackage from database in your fork like the logging.
+If you like to use another database solution than influxdb, Pull Requests are
+welcome. Just fork this project and create another subpackage within the folder
+`database/`. Take this folder as example: `database/logging/`.
 
-### Configurate
+### Configure Yanic
 ```sh
 cp /opt/go/src/github.com/FreifunkBremen/yanic/config_example.toml /etc/yanic.conf
 ```
-You only need to edit `/etc/yanic.conf` under section `[respondd]` the `interface` for a easy startup.
-And create the following folders:
+For an easy startup you only need to edit the `interfaces` in section
+`[respondd]` in file `/etc/yanic.conf`.  
+
+Then create the following files and folders:
 ```sh
-mkdir -p /var/lib/collector
+mkdir -p /var/lib/yanic
 mkdir -p /var/www/html/meshviewer/data
+touch /var/log/yanic.log
+chown yanic /var/log/yanic.log /var/lib/yanic /var/www/html/meshviewer/data
 ```
 
 #### Standalone
-If you like to run a meshviewer standalone, just set `enable` under section `[webserver]` to `true`.
-Configurate the [meshviewer](https://github.com/ffrgb/meshviewer) set `dataPath` in `config.json` to `/data/` and put the `build` directory under `/var/www/html/meshviewer`.
+If you like to run a standalone meshviewer, just set `enable` in section
+`[webserver]` to `true`.
+
+##### Configure the [meshviewer](https://github.com/ffrgb/meshviewer):
+set `dataPath` in `config.json` to `/data/` and make the `build` directory
+accessible under `/var/www/html/meshviewer`.
 
 #### With webserver (Apache, nginx)
-Change following path under section `[nodes]` to what you need.
-For `nodes_path` and `graph_path` should be under the same folder for a meshviewer.
+The meshviewer needs the output files like `nodes_path` and `graph_path` inside
+the same directory as the `dataPath`. Change the path in the section
+`[[nodes.output.meshviewer]]` accordingly.
 
 ### Service
-```bash
-cp /opt/go/src/github.com/FreifunkBremen/yanic/contrib/init/linux-systemd/yanic.service /lib/systemd/system/
+```sh
+cp /opt/go/src/github.com/FreifunkBremen/yanic/contrib/init/linux-systemd/yanic.service /lib/systemd/system/yanic.service
 systemctl daemon-reload
+```
+
+Before start, you should configure yanic by the file `/etc/yanic.conf`:
+```sh
 systemctl start yanic
+```
+
+Enable to start on boot:
+```sh
 systemctl enable yanic
 ```
+
+### Update
+For an update just stop yanic and then call the same `go` command again (again as root):
+```sh
+systemctl stop yanic
+go get -v -u github.com/FreifunkBremen/yanic
+```
+Then update the config file, for example look at the diff with the new example:
+```sh
+diff /opt/go/src/github.com/FreifunkBremen/yanic/config_example.toml /etc/yanic.conf
+```

LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.

README.md

@@ -8,8 +8,9 @@ __   __          _
 Yet another node info collector
 ```
 
-[![Build Status](https://travis-ci.org/FreifunkBremen/yanic.svg?branch=master)](https://travis-ci.org/FreifunkBremen/yanic)
+[![CircleCI](https://circleci.com/gh/FreifunkBremen/yanic/tree/master.svg?style=shield)](https://circleci.com/gh/FreifunkBremen/yanic/tree/master)
 [![Coverage Status](https://coveralls.io/repos/github/FreifunkBremen/yanic/badge.svg?branch=master)](https://coveralls.io/github/FreifunkBremen/yanic?branch=master)
+[![codecov](https://codecov.io/gh/FreifunkBremen/yanic/branch/master/graph/badge.svg)](https://codecov.io/gh/FreifunkBremen/yanic)
 [![Go Report Card](https://goreportcard.com/badge/github.com/FreifunkBremen/yanic)](https://goreportcard.com/report/github.com/FreifunkBremen/yanic)
 
 `yanic` is a respondd client that fetches, stores and publishes information about a Freifunk network. The goals:
@@ -18,36 +19,116 @@ Yet another node info collector
 * Provide a little webserver for a standalone installation with a meshviewer
 
 ## How it works
-
-In the first step Yanic sends a multicast message to the group `ff02:0:0:0:0:0:2:1001` and port `1001`.
+In the first step Yanic sends a multicast message to the group `ff05::2:1001` and port `1001`.
 Recently seen nodes that does not reply are requested via a unicast message.
 
-## [Documentation](https://www.gitbook.com/book/freifunkbremen/yanic/details)
-* [Webview](https://freifunkbremen.gitbooks.io/yanic/content/)
-* [PDF](https://www.gitbook.com/download/pdf/book/freifunkbremen/yanic)
-* [Mobi](https://www.gitbook.com/download/mobi/book/freifunkbremen/yanic)
-* [ePUB](https://www.gitbook.com/download/epub/book/freifunkbremen/yanic)
+## Documentation
+Take a look at the [git](https://github.com/FreifunkBremen/yanic/blob/master/SUMMARY.md) or [Gitbook](https://freifunkbremen.gitbooks.io/yanic/content/)
 
+# Installation
+Take a look into the Documentation (see above) or for Quick Overview in [INSTALL.md](INSTALL.md).
 
-## Quick startup
+If you like Docker you may want to take a look [here](https://github.com/christf/docker-yanic).
 
-### Usage
-```
-Usage of ./yanic:
-  -config path/to/config.toml
-```
-### Configuration
+## Configuration
 Read comments in [config_example.toml](config_example.toml) for more information.
 
-### Live
-* [meshviewer](https://map.bremen.freifunk.net) **Freifunk Bremen** with a patch to show state-version of `nodes.json`
-* [grafana](https://grafana.bremen.freifunk.net)  **Freifunk Bremen** show data of InfluxDB
+## Running
+Yanic provides several commands:
+
+### Usage
+Run Yanic without any arguments to get the usage information:
+```
+Usage:
+  yanic [command]
+
+Available Commands:
+  help        Help about any command
+  import      Imports global statistics from the given RRD files, requires InfluxDB
+  query       Sends a query on the interface to the destination and waits for a response
+  serve       Runs the yanic server
+
+Flags:
+  -h, --help              help for yanic
+      --loglevel uint32   Show log message starting at level (default 40)
+      --timestamps        Enables timestamps for log output
+
+Use "yanic [command] --help" for more information about a command.
+```
+
+#### Serve
+Runs the yanic server
+```
+Usage:
+  yanic serve [flags]
+
+Examples:
+yanic serve --config /etc/yanic.toml
+
+Flags:
+  -c, --config string   Path to configuration file (default "config.toml")
+  -h, --help            help for serve
+
+Global Flags:
+      --loglevel uint32   Show log message starting at level (default 40)
+      --timestamps        Enables timestamps for log output
+```
+
+#### Query
+Sends a query on the interface to the destination and waits for a response
+```
+Usage:
+  yanic query <interfaces> <destination> [flags]
+
+Examples:
+yanic query "eth0,wlan0" "fe80::eade:27ff:dead:beef"
+
+Flags:
+  -h, --help        help for query
+      --ip string   ip address which is used for sending (optional - without definition used the link-local address)
+      --port int    define a port to listen (if not set or set to 0 the kernel will use a random free port at its own)
+      --wait int    Seconds to wait for a response (default 1)
+
+Global Flags:
+      --loglevel uint32   Show log message starting at level (default 40)
+      --timestamps        Enables timestamps for log output
+```
+
+#### Import
+Imports global statistics from the given RRD files (ffmap-backend).
+```
+Usage:
+  yanic import <file.rrd> <site> <domain> [flags]
+
+Examples:
+yanic import --config /etc/yanic.toml olddata.rrd global global
+
+Flags:
+  -c, --config string   Path to configuration file (default "config.toml")
+  -h, --help            help for import
+
+Global Flags:
+      --loglevel uint32   Show log message starting at level (default 40)
+      --timestamps        Enables timestamps for log output
+```
+
+
+
+## Communities using Yanic
+* **Freifunk Bremen** uses InfluxDB, [Grafana](https://grafana.bremen.freifunk.net), and [Meshviewer](https://map.bremen.freifunk.net) with a patch to show state-version of `nodes.json`. 
+* **Freifunk Nord** uses [hopglass](https://github.com/hopglass/hopglass) (commit 587740a) as frontend:  https://mesh.freifunknord.de/
+* **Freifunk Kiel** uses [Meshviewer](https://github.com/ffrgb/meshviewer/) as frontend: https://map.freifunk.in-kiel.de/
+
+Do you know someone else using Yanic? Create a [pull request](https://github.com/FreifunkBremen/yanic/issues/new?template=community.md&title=Mention+community+$name)!
 
 ## Related projects
-
 Collecting data from respondd:
 * [HopGlass Server](https://github.com/plumpudding/hopglass-server) written in Node.js
 
 Respondd for servers:
-* [ffnord-alfred-announce](https://github.com/ffnord/ffnord-alfred-announce) from FreiFunkNord
+* [mesh-announce](https://github.com/ffnord/mesh-announce) from ffnord
 * [respondd](https://github.com/Sunz3r/ext-respondd) from Sunz3r
+
+
+## License
+This software is licensed under the terms of the [AGPL v3 License](LICENSE).

SUMMARY.md

@@ -0,0 +1,15 @@
+# Summary
+
+* [Home](/docs/index.md)
+* [About](/docs/home_about.md)
+* [Running](/docs/home_running.md)
+
+* Documentation
+  * [Build and Install](/docs/docs_install.md)
+  * [Usage](/docs/docs_usage.md)
+  * [Quick Configuration](/docs/docs_quick_conf.md)
+  * [Configuration](/docs/docs_configuration.md)
+
+* Developing
+  * [Add new database type](/docs/dev_database.md)
+  * [Add new output type](/docs/dev_output.md)

book.json

@@ -0,0 +1,22 @@
+{
+    "plugins": [
+        "theme-api",
+        "anchors",
+        "sitemap"
+    ],
+    "pluginsConfig": {
+        "theme-api": {
+          "split": true,
+            "languages": [
+                {
+                    "lang": "toml",
+                    "name": "TOML",
+                    "default": true
+                }
+            ]
+        },
+        "sitemap": {
+            "hostname": "https://freifunkbremen.gitbooks.io/yanic/"
+        }
+    }
+}

cmd/config.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/naoina/toml"
+
+	"github.com/FreifunkBremen/yanic/database"
+	"github.com/FreifunkBremen/yanic/respond"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/FreifunkBremen/yanic/webserver"
+)
+
+// Config represents the whole configuration
+type Config struct {
+	Respondd  respond.Config
+	Webserver webserver.Config
+	Nodes     runtime.NodesConfig
+	Database  database.Config
+}
+
+var (
+	configPath string
+	collector  *respond.Collector
+	nodes      *runtime.Nodes
+)
+
+func loadConfig() *Config {
+	config, err := ReadConfigFile(configPath)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "unable to load config file:", err)
+		os.Exit(2)
+	}
+	return config
+}
+
+// ReadConfigFile reads a config model from path of a yml file
+func ReadConfigFile(path string) (config *Config, err error) {
+	config = &Config{}
+
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = toml.Unmarshal(file, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}

cmd/config_test.go

@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	config, err := ReadConfigFile("../config_example.toml")
+	assert.NoError(err)
+	assert.NotNil(config)
+
+	assert.True(config.Respondd.Enable)
+	assert.Equal("br-ffhb", config.Respondd.Interfaces[0].InterfaceName)
+	assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
+	assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
+	assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration)
+
+	assert.Len(config.Respondd.Sites, 1)
+	assert.Contains(config.Respondd.Sites, "ffhb")
+	assert.Contains(config.Respondd.Sites["ffhb"].Domains, "city")
+
+	// Test output plugins
+	assert.Len(config.Nodes.Output, 4)
+	outputs := config.Nodes.Output["meshviewer"].([]interface{})
+	assert.Len(outputs, 1)
+	meshviewer := outputs[0]
+
+	assert.EqualValues(map[string]interface{}{
+		"version":    int64(2),
+		"enable":     false,
+		"nodes_path": "/var/www/html/meshviewer/data/nodes.json",
+		"graph_path": "/var/www/html/meshviewer/data/graph.json",
+		"filter": map[string]interface{}{
+			"no_owner": true,
+		},
+	}, meshviewer)
+
+	_, err = ReadConfigFile("testdata/config_invalid.toml")
+	assert.Error(err, "not unmarshalable")
+	assert.Contains(err.Error(), "invalid TOML syntax")
+
+	_, err = ReadConfigFile("testdata/adsa.toml")
+	assert.Error(err, "not found able")
+	assert.Contains(err.Error(), "no such file or directory")
+}

cmd/import.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"github.com/bdlm/log"
+	"github.com/spf13/cobra"
+
+	allDatabase "github.com/FreifunkBremen/yanic/database/all"
+	"github.com/FreifunkBremen/yanic/rrd"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+// importCmd represents the import command
+var importCmd = &cobra.Command{
+	Use:     "import <file.rrd> <site> <domain>",
+	Short:   "Imports global statistics from the given RRD files, requires InfluxDB",
+	Example: "yanic import --config /etc/yanic.toml olddata.rrd global global",
+	Args:    cobra.ExactArgs(3),
+	Run: func(cmd *cobra.Command, args []string) {
+		path := args[0]
+		site := args[1]
+		domain := args[2]
+		config := loadConfig()
+
+		err := allDatabase.Start(config.Database)
+		if err != nil {
+			log.Panicf("could not connect to database: %s", err)
+		}
+		defer allDatabase.Close()
+
+		log.Infof("importing RRD from %s", path)
+
+		for ds := range rrd.Read(path) {
+			allDatabase.Conn.InsertGlobals(
+				&runtime.GlobalStats{
+					Nodes:   uint32(ds.Nodes),
+					Clients: uint32(ds.Clients),
+				},
+				ds.Time,
+				site,
+				domain,
+			)
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(importCmd)
+	importCmd.Flags().StringVarP(&configPath, "config", "c", "config.toml", "Path to configuration file")
+}

cmd/query.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/bdlm/log"
+	"github.com/spf13/cobra"
+
+	"github.com/FreifunkBremen/yanic/respond"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+var (
+	wait      int
+	port      int
+	ipAddress string
+)
+
+// queryCmd represents the query command
+var queryCmd = &cobra.Command{
+	Use:     "query <interfaces> <destination>",
+	Short:   "Sends a query on the interface to the destination and waits for a response",
+	Example: `yanic query "eth0,wlan0" "fe80::eade:27ff:dead:beef"`,
+	Args:    cobra.ExactArgs(2),
+	Run: func(cmd *cobra.Command, args []string) {
+		ifaces := strings.Split(args[0], ",")
+		dstAddress := net.ParseIP(args[1])
+
+		log.WithFields(map[string]interface{}{
+			"address": dstAddress,
+			"ifaces":  ifaces,
+		}).Info("sending request")
+
+		var ifacesConfigs []respond.InterfaceConfig
+		for _, iface := range ifaces {
+			ifaceConfig := respond.InterfaceConfig{
+				InterfaceName: iface,
+				Port:          port,
+				IPAddress:     ipAddress,
+			}
+			ifacesConfigs = append(ifacesConfigs, ifaceConfig)
+		}
+
+		nodes := runtime.NewNodes(&runtime.NodesConfig{})
+
+		sitesDomains := make(map[string][]string)
+		collector := respond.NewCollector(nil, nodes, sitesDomains, ifacesConfigs)
+		defer collector.Close()
+		collector.SendPacket(dstAddress)
+
+		time.Sleep(time.Second * time.Duration(wait))
+
+		for id, data := range nodes.List {
+			jq, err := json.Marshal(data)
+			if err != nil {
+				fmt.Printf("%s: %+v", id, data)
+			} else {
+				jqNeighbours, err := json.Marshal(data.Neighbours)
+				if err != nil {
+					fmt.Printf("%s: %s neighbours: %+v", id, string(jq), data.Neighbours)
+				} else {
+					fmt.Printf("%s: %s neighbours: %s", id, string(jq), string(jqNeighbours))
+				}
+			}
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(queryCmd)
+	queryCmd.Flags().IntVar(&wait, "wait", 1, "Seconds to wait for a response")
+	queryCmd.Flags().IntVar(&port, "port", 0, "define a port to listen (if not set or set to 0 the kernel will use a random free port at its own)")
+	queryCmd.Flags().StringVar(&ipAddress, "ip", "", "ip address which is used for sending (optional - without definition used the link-local address)")
+}

cmd/respondd-bridge/main.go

@@ -1,216 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"compress/flate"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net"
-	"os"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/data"
-	"github.com/FreifunkBremen/yanic/respond"
-)
-
-const maxDataGramSize = 8192
-
-// Collector for a specificle respond messages
-type Collector struct {
-	connection *net.UDPConn           // UDP socket
-	queue      chan *respond.Response // received responses
-	interval   time.Duration          // Interval for multicast packets
-	stop       chan interface{}
-	nodes      map[string]*data.ResponseData
-	interMac   map[string]string
-	addrFrom   net.UDPAddr
-	addrTo     net.UDPAddr
-}
-
-func main() {
-	iface := os.Args[1]
-	addrFrom := os.Args[2]
-	addrTo := os.Args[3]
-	linkLocalAddr, err := getLinkLocalAddr(iface)
-	if err != nil {
-		log.Panic(err)
-	}
-	conn, err := net.ListenUDP("udp", &net.UDPAddr{
-		IP:   linkLocalAddr,
-		Zone: iface,
-	})
-	if err != nil {
-		log.Panic(err)
-	}
-	conn.SetReadBuffer(maxDataGramSize)
-	collector := &Collector{
-		connection: conn,
-		queue:      make(chan *respond.Response, 400),
-		stop:       make(chan interface{}),
-		addrFrom:   net.UDPAddr{IP: net.ParseIP(addrFrom)},
-		addrTo:     net.UDPAddr{IP: net.ParseIP(addrTo)},
-		interval:   time.Second * 10,
-		nodes:      make(map[string]*data.ResponseData),
-		interMac:   make(map[string]string),
-	}
-	go collector.receiver(conn)
-	go collector.parser()
-	collector.sendOnce()
-	collector.sender()
-	collector.Close()
-}
-
-// Returns the first link local unicast address for the given interface name
-func getLinkLocalAddr(ifname string) (net.IP, error) {
-	iface, err := net.InterfaceByName(ifname)
-	if err != nil {
-		return nil, err
-	}
-
-	addresses, err := iface.Addrs()
-	if err != nil {
-		return nil, err
-	}
-
-	for _, addr := range addresses {
-		if ipnet := addr.(*net.IPNet); ipnet.IP.IsLinkLocalUnicast() {
-			return ipnet.IP, nil
-		}
-	}
-	return nil, fmt.Errorf("unable to find link local unicast address for %s", ifname)
-}
-
-// SendPacket sends a UDP request to the given unicast or multicast address
-func (coll *Collector) SendRequestPacket(addr net.UDPAddr) {
-	addr.Port = 1001
-	if _, err := coll.connection.WriteToUDP([]byte("GET nodeinfo statistics neighbours"), &addr); err != nil {
-		log.Println("WriteToUDP failed:", err)
-	}
-}
-
-func (coll *Collector) saveResponse(addr net.UDPAddr, node *data.ResponseData) {
-	if val := node.NodeInfo; val == nil {
-		log.Printf("no nodeinfo from %s", addr.String())
-		return
-	}
-	// save current node
-	coll.nodes[addr.IP.String()] = node
-
-	// Process the data and update IP address
-	var otherIP string
-	if addr.IP.Equal(coll.addrFrom.IP) {
-		otherIP = coll.addrTo.IP.String()
-	} else {
-		otherIP = coll.addrFrom.IP.String()
-	}
-
-	otherNode := coll.nodes[otherIP]
-	if otherIP == "" || otherNode == nil {
-		log.Print("othernode not found")
-		return
-	}
-
-	if node.Neighbours == nil {
-		node.Neighbours = &data.Neighbours{
-			Batadv: make(map[string]data.BatadvNeighbours),
-			NodeID: node.NodeInfo.NodeID,
-		}
-	}
-	interMac := node.NodeInfo.Network.Mesh["bat0"].Interfaces.Other[0]
-	if newMac, ok := coll.interMac[addr.IP.String()]; ok {
-		interMac = newMac
-	} else {
-		coll.interMac[addr.IP.String()] = interMac
-	}
-	if _, ok := node.Neighbours.Batadv[interMac]; !ok {
-		node.Neighbours.Batadv[interMac] = data.BatadvNeighbours{
-			Neighbours: make(map[string]data.BatmanLink),
-		}
-	}
-	interOtherMac := otherNode.NodeInfo.Network.Mesh["bat0"].Interfaces.Other[0]
-	if newMac, ok := coll.interMac[coll.addrTo.IP.String()]; ok {
-		interOtherMac = newMac
-	} else {
-		coll.interMac[otherIP] = interMac
-	}
-	node.Neighbours.Batadv[interMac].Neighbours[interOtherMac] = data.BatmanLink{
-		Tq:       253,
-		Lastseen: 0.2,
-	}
-	buf := bytes.Buffer{}
-	writer := bufio.NewWriter(&buf)
-	deflater, err := flate.NewWriter(writer, flate.DefaultCompression)
-
-	err = json.NewEncoder(deflater).Encode(node)
-	if err != nil {
-		panic(err)
-	}
-	deflater.Close()
-	writer.Flush()
-
-	coll.connection.WriteToUDP(buf.Bytes(), &net.UDPAddr{
-		IP:   net.ParseIP("fe80::de:faff:fe9f:2414"),
-		Port: 12345,
-	})
-	log.Print("send response from: ", addr.IP.String())
-}
-
-func (coll *Collector) receiver(conn *net.UDPConn) {
-	buf := make([]byte, maxDataGramSize)
-	for {
-		n, src, err := conn.ReadFromUDP(buf)
-
-		if err != nil {
-			log.Println("ReadFromUDP failed:", err)
-			return
-		}
-
-		raw := make([]byte, n)
-		copy(raw, buf)
-
-		coll.queue <- &respond.Response{
-			Address: *src,
-			Raw:     raw,
-		}
-	}
-}
-
-func (coll *Collector) parser() {
-	for obj := range coll.queue {
-		if data, err := obj.Parse(); err != nil {
-			log.Println("unable to decode response from", obj.Address.String(), err, "\n", string(obj.Raw))
-		} else {
-			coll.saveResponse(obj.Address, data)
-		}
-	}
-}
-
-func (coll *Collector) sendOnce() {
-	coll.SendRequestPacket(coll.addrFrom)
-	coll.SendRequestPacket(coll.addrTo)
-	log.Print("send request")
-}
-
-// send packets continously
-func (coll *Collector) sender() {
-	ticker := time.NewTicker(coll.interval)
-	for {
-		select {
-		case <-coll.stop:
-			ticker.Stop()
-			return
-		case <-ticker.C:
-			// send the multicast packet to request per-node statistics
-			coll.sendOnce()
-		}
-	}
-}
-
-// Close Collector
-func (coll *Collector) Close() {
-	close(coll.stop)
-	coll.connection.Close()
-	close(coll.queue)
-}

cmd/root.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/bdlm/log"
+	"github.com/bdlm/std/logger"
+	"github.com/spf13/cobra"
+)
+
+var (
+	timestamps bool
+	loglevel   uint32
+)
+
+// RootCmd represents the base command when called without any subcommands
+var RootCmd = &cobra.Command{
+	Use:   "yanic",
+	Short: "Yet another node info collector",
+	Long:  `A respondd client that fetches, stores and publishes information about a Freifunk network.`,
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := RootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+	RootCmd.PersistentFlags().BoolVar(&timestamps, "timestamps", false, "Enables timestamps for log output")
+	RootCmd.PersistentFlags().Uint32Var(&loglevel, "loglevel", 40, "Show log message starting at level")
+}
+
+func initConfig() {
+	log.SetLevel(logger.Level(loglevel))
+	log.SetFormatter(&log.TextFormatter{
+		DisableTimestamp: timestamps,
+	})
+}

cmd/serve.go

@@ -0,0 +1,75 @@
+package cmd
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/bdlm/log"
+	"github.com/spf13/cobra"
+
+	allDatabase "github.com/FreifunkBremen/yanic/database/all"
+	allOutput "github.com/FreifunkBremen/yanic/output/all"
+	"github.com/FreifunkBremen/yanic/respond"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/FreifunkBremen/yanic/webserver"
+)
+
+// serveCmd represents the serve command
+var serveCmd = &cobra.Command{
+	Use:     "serve",
+	Short:   "Runs the yanic server",
+	Example: "yanic serve --config /etc/yanic.toml",
+	Run: func(cmd *cobra.Command, args []string) {
+		config := loadConfig()
+
+		err := allDatabase.Start(config.Database)
+		if err != nil {
+			log.Panicf("could not connect to database: %s", err)
+		}
+		defer allDatabase.Close()
+
+		nodes = runtime.NewNodes(&config.Nodes)
+		nodes.Start()
+
+		err = allOutput.Start(nodes, config.Nodes)
+		if err != nil {
+			log.Panicf("error on init outputs: %s", err)
+		}
+		defer allOutput.Close()
+
+		if config.Webserver.Enable {
+			log.Infof("starting webserver on %s", config.Webserver.Bind)
+			srv := webserver.New(config.Webserver.Bind, config.Webserver.Webroot)
+			go webserver.Start(srv)
+			defer srv.Close()
+		}
+
+		if config.Respondd.Enable {
+			// Delaying startup to start at a multiple of `duration` since the zero time.
+			if duration := config.Respondd.Synchronize.Duration; duration > 0 {
+				now := time.Now()
+				delay := duration - now.Sub(now.Truncate(duration))
+				log.Infof("delaying %0.1f seconds", delay.Seconds())
+				time.Sleep(delay)
+			}
+
+			collector = respond.NewCollector(allDatabase.Conn, nodes, config.Respondd.SitesDomains(), config.Respondd.Interfaces)
+			collector.Start(config.Respondd.CollectInterval.Duration)
+			defer collector.Close()
+		}
+
+		// Wait for INT/TERM
+		sigs := make(chan os.Signal, 1)
+		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+		sig := <-sigs
+		log.Infof("received %s", sig)
+
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(serveCmd)
+	serveCmd.Flags().StringVarP(&configPath, "config", "c", "config.toml", "Path to configuration file")
+}

cmd/testdata/config_invalid.toml

@@ -0,0 +1 @@
+foobar

cmd/version.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// VERSION is set at build time
+var VERSION string
+
+// versionCMD to print version
+var versionCMD = &cobra.Command{
+	Use:   "version",
+	Short: "print version of yanic",
+	Run: func(cmd *cobra.Command, args []string) {
+
+		fmt.Printf("yanic version: %s\n", VERSION)
+	},
+}
+
+func init() {
+	if VERSION != "" {
+		RootCmd.AddCommand(versionCMD)
+	}
+}

cmd/yanic-query/main.go

@@ -1,35 +0,0 @@
-package main
-
-import (
-	"log"
-	"net"
-	"os"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/respond"
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-// Usage: yanic-query wlp4s0 "[fe80::eade:27ff:dead:beef%wlp4s0]:1001"
-func main() {
-	iface := os.Args[1]
-	dstAddress := os.Args[2]
-
-	log.Printf("Sending request address=%s iface=%s", dstAddress, iface)
-
-	nodes := runtime.NewNodes(&runtime.Config{})
-
-	collector := respond.NewCollector(nil, nodes, iface, iface, iface, 0)
-	collector.SendPacket(net.UDPAddr{
-		IP:   net.ParseIP(dstAddress),
-		Zone: iface,
-	})
-
-	time.Sleep(time.Second)
-
-	for id, data := range nodes.List {
-		log.Printf("%s: %+v", id, data)
-	}
-
-	collector.Close()
-}

cmd/yanic/main.go

@@ -1,117 +0,0 @@
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/database"
-	allDB "github.com/FreifunkBremen/yanic/database/all"
-	"github.com/FreifunkBremen/yanic/output"
-	allOutput "github.com/FreifunkBremen/yanic/output/all"
-	"github.com/FreifunkBremen/yanic/respond"
-	"github.com/FreifunkBremen/yanic/rrd"
-	"github.com/FreifunkBremen/yanic/runtime"
-	"github.com/FreifunkBremen/yanic/webserver"
-)
-
-var (
-	configFile  string
-	config      *runtime.Config
-	collector   *respond.Collector
-	connections database.Connection
-	nodes       *runtime.Nodes
-)
-
-func main() {
-	var importPath string
-	var timestamps bool
-	flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb")
-	flag.StringVar(&configFile, "config", "config.toml", "path of configuration file (default:config.yaml)")
-	flag.BoolVar(&timestamps, "timestamps", true, "print timestamps in output")
-	flag.Parse()
-
-	if !timestamps {
-		log.SetFlags(0)
-	}
-	log.Println("Yanic say hello")
-
-	config, err := runtime.ReadConfigFile(configFile)
-	if err != nil {
-		panic(err)
-	}
-	if config.Respondd.InterfaceSendUnicast == "" {
-		config.Respondd.InterfaceSendUnicast = config.Respondd.InterfaceListen
-	}
-	if config.Respondd.InterfaceSendMulticast == "" {
-		config.Respondd.InterfaceSendMulticast = config.Respondd.InterfaceListen
-	}
-	if config.Respondd.MulticastDestination != "" {
-		respond.MulticastGroup = config.Respondd.MulticastDestination
-	}
-
-	connections, err = allDB.Connect(config.Database.Connection)
-	if err != nil {
-		panic(err)
-	}
-	database.Start(connections, config)
-	defer database.Close(connections)
-
-	if connections != nil && importPath != "" {
-		importRRD(importPath)
-		return
-	}
-
-	nodes = runtime.NewNodes(config)
-	nodes.Start()
-
-	outputs, err := allOutput.Register(nodes, config.Nodes.Output)
-	if err != nil {
-		panic(err)
-	}
-
-	output.Start(outputs, config)
-	defer output.Close()
-
-	if config.Webserver.Enable {
-		log.Println("starting webserver on", config.Webserver.Bind)
-		srv := webserver.New(config.Webserver.Bind, config.Webserver.Webroot)
-		go srv.Close()
-	}
-
-	if config.Respondd.Enable {
-		// Delaying startup to start at a multiple of `duration` since the zero time.
-		if duration := config.Respondd.Synchronize.Duration; duration > 0 {
-			now := time.Now()
-			delay := duration - now.Sub(now.Truncate(duration))
-			log.Printf("delaying %0.1f seconds", delay.Seconds())
-			time.Sleep(delay)
-		}
-
-		collector = respond.NewCollector(connections, nodes, config.Respondd.InterfaceListen, config.Respondd.InterfaceSendMulticast, config.Respondd.InterfaceSendUnicast, config.Respondd.ListenPort)
-		collector.Start(config.Respondd.CollectInterval.Duration)
-		defer collector.Close()
-	}
-
-	// Wait for INT/TERM
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	sig := <-sigs
-	log.Println("received", sig)
-}
-
-func importRRD(path string) {
-	log.Println("importing RRD from", path)
-	for ds := range rrd.Read(path) {
-		connections.InsertGlobals(
-			&runtime.GlobalStats{
-				Nodes:   uint32(ds.Nodes),
-				Clients: uint32(ds.Clients),
-			},
-			ds.Time,
-		)
-	}
-}

config_example.toml

@@ -1,22 +1,40 @@
+# This is the config file for Yanic written in "Tom's Obvious, Minimal Language."
+# syntax: https://github.com/toml-lang/toml
+# (if you need something multiple times, checkout out the [[array of table]] section)
+
 # Send respondd request to update information
 [respondd]
 enable           = true
 # Delay startup until a multiple of the period since zero time
 synchronize      = "1m"
-# how oftern request per multicast
+# how often request per multicast
 collect_interval = "1m"
-# on which interface to listen
-interface        = "eth0"
-# send unicast request (default: see interface)
-## interface_send_unicast = "eth0"
-# send multicast request (default: see interface)
-# interface_send_multicast = "eth1"
 
+# table of a site to save stats for (not exists for global only)
+#[respondd.sites.example]
+## list of domains on this site to save stats for (empty for global only)
+#domains = []
+## example
+[respondd.sites.ffhb]
+domains            = ["city"]
+
+# interface that has an IP in your mesh network
+[[respondd.interfaces]]
+# name of interface on which this collector is running
+ifname = "br-ffhb"
+# ip address which is used for sending
+# (optional - without definition used a address of ifname - preferred link local)
+ip_address = "fd2f:5119:f2d::5"
+# disable sending multicast respondd request
+# (for receiving only respondd packages e.g. database respondd)
+#send_no_request = false
+# multicast address to destination of respondd
+# (optional - without definition used default ff05::2:1001)
+#multicast_address = "ff02::2:1001"
 # define a port to listen
-# (no or 0 would choose at port at his own)
+# if not set or set to 0 the kernel will use a random free port at its own
 #port = 10001
 
-
 # A little build-in webserver, which statically serves a directory.
 # This is useful for testing purposes or for a little standalone installation.
 [webserver]
@@ -25,99 +43,167 @@ bind    = "127.0.0.1:8080"
 webroot = "/var/www/html/meshviewer"
 
 
-
 [nodes]
-enable         = true
-# state-version of nodes.json to store cached data,
-#  these is the directly collected respondd data
-state_path     = "/var/lib/collector/state.json"
-
+# Cache file
+# a json file to cache all data collected directly from respondd
+state_path    = "/var/lib/yanic/state.json"
+# prune data in RAM, cache-file and output json files (i.e. nodes.json)
+# that were inactive for longer than
+prune_after   = "7d"
 # Export nodes and graph periodically
 save_interval = "5s"
-
 # Set node to offline if not seen within this period
 offline_after = "10m"
 
-# Prune offline nodes after a time of inactivity
-prune_after = "7d"
+
+## [[nodes.output.example]]
+# Each output format has its own config block and needs to be enabled by adding:
+#enable = true
+#
+# For each output format there can be set different filters
+#[nodes.output.example.filter]
+#
+# WARNING: if it is not set, it will publish contact information of other persons
+# Set to true, if you did not want the json files to contain the owner information
+#no_owner = true
+#
+# List of nodeids of nodes that should be filtered out, so they won't appear in output
+#blacklist = ["00112233445566", "1337f0badead"]
+#
+# List of site_codes of nodes that should be included in the output
+#sites = ["ffhb"]
+#
+# replace the site_code with the domain_code in this output
+# e.g. site_code='ffhb',domain_code='city' => site_code='city', domain_code=''
+#domain_as_site = true
+#
+# append on the site_code the domain_code with a '.' in this output
+# e.g. site_code='ffhb',domain_code='city' => site_code='ffhb.city', domain_code=''
+#domain_append_site = true
+#
+# set has_location to true if you want to include only nodes that have geo-coordinates set
+# (setting this to false has no sensible effect, unless you'd want to hide nodes that have coordinates)
+#has_location = true
+
+#[nodes.output.example.filter.in_area]
+# nodes outside this area are not shown on the map but are still listed as a node without coordinates
+#latitude_min = 34.30
+#latitude_max = 71.85
+#longitude_min = -24.96
+#longitude_max = 39.72
 
 
-[[nodes.output.meshviewer]]
+# outputs all nodes as points into nodes.geojson
+[[nodes.output.geojson]]
 enable   = true
-# structur of nodes.json, which to support
-# version 1 is to support legacy meshviewer (which are in master branch)
-#    i.e. https://github.com/ffnord/meshviewer/tree/master
-# version 2 is to support new version of meshviewer (which are in legacy develop branch or newer)
-#    i.e. https://github.com/ffnord/meshviewer/tree/dev
-#         https://github.com/ffrgb/meshviewer/tree/develop
-version  = 2
+path = "/var/www/html/meshviewer/data/nodes.geojson"
+
+# definition for the new more compressed meshviewer.json
+[[nodes.output.meshviewer-ffrgb]]
+enable   = true
+path = "/var/www/html/meshviewer/data/meshviewer.json"
+
+[nodes.output.meshviewer-ffrgb.filter]
+# WARNING: if it is not set, it will publish contact information of other persons
+no_owner = false
+#blacklist = ["00112233445566", "1337f0badead"]
+#sites = ["ffhb"]
+#has_location = true
+
+#[nodes.output.meshviewer-ffrgb.filter.in_area]
+#latitude_min = 34.30
+#latitude_max = 71.85
+#longitude_min = -24.96
+#longitude_max = 39.72
+
+
+# definition for nodes.json
+[[nodes.output.meshviewer]]
+enable = false
+# The structure version of the output which should be generated (i.e. nodes.json)
+# version 1 is accepted by the legacy meshviewer (which is the master branch)
+#   i.e. https://github.com/ffnord/meshviewer/tree/master
+# version 2 is accepted by the new versions of meshviewer (which are in the legacy develop branch or newer)
+#   i.e. https://github.com/ffnord/meshviewer/tree/dev
+#        https://github.com/ffrgb/meshviewer/tree/develop
+version    = 2
 # path where to store nodes.json
-nodes_path     = "/var/www/html/meshviewer/data/nodes.json"
+nodes_path = "/var/www/html/meshviewer/data/nodes.json"
 # path where to store graph.json
-graph_path     = "/var/www/html/meshviewer/data/graph.json"
+graph_path = "/var/www/html/meshviewer/data/graph.json"
 
 [nodes.output.meshviewer.filter]
-# no_owner = true
-has_location = true
-blacklist = ["vpnid"]
+# WARNING: if it is not set, it will publish contact information of other persons
+no_owner = true
 
-[nodes.output.meshviewer.filter.in_area]
-latitude_min = 34.30
-latitude_max = 71.85
-longitude_min = -24.96
-longitude_max = 39.72
-
-[[nodes.output.template]]
-enable = false
-template_path = "/var/lib/collector/html-template.tmp"
-output_path = "/var/www/html/index.html"
 
+# definition for nodelist.json
 [[nodes.output.nodelist]]
 enable   = true
 path = "/var/www/html/meshviewer/data/nodelist.json"
 
+[nodes.output.nodelist.filter]
+# WARNING: if it is not set, it will publish contact information of other persons
+no_owner = true
+
+
 
 [database]
-# cleaning data of measurement node,
-#   which are older than 7d
-delete_after = "7d"
-#   how often run the cleaning
+# this will send delete commands to the database to prune data
+# which is older than:
+delete_after    = "7d"
+# how often run the cleaning
 delete_interval = "1h"
 
-# Save collected data to InfluxDB
-# there would be the following measurments:
-#  node: store node spezific data i.e. clients memory, airtime
-#  global: store global data, i.e. count of clients and nodes
-#  firmware: store count of nodes tagged with firmware
-#  model: store count of nodes tagged with hardware model
+## [[database.connection.example]]
+# Each database-connection has its own config block and needs to be enabled by adding:
+#enable = true
+
+# Save collected data to InfluxDB.
+# There are the following measurments:
+#   node: store node specific data i.e. clients memory, airtime
+#   global: store global data, i.e. count of clients and nodes
+#   firmware: store the count of nodes tagged with firmware
+#   model: store the count of nodes tagged with hardware model
 [[database.connection.influxdb]]
 enable   = false
 address  = "http://localhost:8086"
 database = "ffhb"
 username = ""
 password = ""
-# tagging of the data are optional
-# be carefull tags by system would overright config
+
+# Tagging of the data (optional)
 [database.connection.influxdb.tags]
-site = "ffhb01"
-system = "testing"
-
-[[database.connection.logging]]
-enable   = false
-path     = "/var/log/yanic.log"
-
+# Tags used by Yanic would override the tags from this config
+# nodeid, hostname, owner, model, firmware_base, firmware_release,frequency11g and frequency11a are tags which are already used
+#tagname1 = "tagvalue 1"
+# some useful e.g.:
+#system   = "productive"
+#site     = "ffhb"
 
+# Graphite settings
 [[database.connection.graphite]]
 enable   = false
 address  = "localhost:2003"
+# Graphite is replacing every "." in the metric name with a slash "/" and uses
+# that for the file system hierarchy it generates. it is recommended to at least
+# move the metrics out of the root namespace (that would be the empty prefix).
+# If you only intend to run one community and only freifunk on your graphite node
+# then the prefix can be set to anything (including the empty string) since you
+# probably wont care much about "polluting" the namespace.
 prefix   = "freifunk"
 
-[[database.connection.socket]]
+# respondd (yanic)
+# forward collected respondd package to a address
+# (e.g. to another respondd collector like a central yanic instance or hopglass)
+[[database.connection.respondd]]
 enable   = false
-type     = "tcp"
-address  = ":8081"
+# type of network to create a connection
+type     = "udp6"
+# destination address to connect/send respondd package
+address  = "stats.bremen.freifunk.net:11001"
 
-[[database.connection.socket]]
+# Logging
+[[database.connection.logging]]
 enable   = false
-type     = "unix"
-address  = "/var/lib/collector/database.socket"
+path     = "/var/log/yanic.log"

contrib/example-template.tmpl

@@ -1,16 +0,0 @@
-function ffhbCurrentStats(data) {
-	$("#freifunk").html("
-	<h1><a href="https://bremen.freifunk.net/" style="color: #444;">bremen.freifunk.net</a></h1>
-	<p>
-		Nutzer: <span id="freifunk_clients">0</span><br>
-		<i style="font-style: italic;">(auf  <span id="freifunk_nodes">0</span> Geräte verteilt)</i>
-	</p>
-	<p style="text-align: right;">
-		<a href="https://events.ffhb.de/meshviewer">mehr</a>
-	</p>");
-
-	$("#freifunk_clients").html(data.Clients);
-	$("#freifunk_nodes").html(data.Nodes);
-};
-
-ffhbCurrentStats({{json .GlobalStatistic}});

contrib/init/linux-systemd/yanic.service

@@ -4,7 +4,7 @@ Description=yanic
 [Service]
 Type=simple
 User=yanic
-ExecStart=/opt/go/bin/yanic -config /etc/yanic.conf
+ExecStart=/opt/go/bin/yanic serve --config /etc/yanic.conf
 Restart=always
 RestartSec=5s
 Environment=PATH=/usr/bin:/usr/local/bin

contrib/yanic-remove-node

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import sys
+
+
+def main(states, nodeid):
+    with open(states) as handle:
+        data = json.load(handle)
+
+    if nodeid in data['nodes']:
+        del data['nodes'][nodeid]
+        print("node {} removed".format(nodeid))
+    else:
+        print('node not in state file', file=sys.stderr)
+        sys.exit(1)
+
+    with open(states, 'w') as handle:
+        json.dump(data, handle)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-s', '--states', action='store', required=True,
+                        help='path to the states json file')
+    parser.add_argument('-n', '--nodeid', action='store', required=True,
+                        help='nodeid of the node to remove')
+
+    args = parser.parse_args()
+
+    main(args.states, args.nodeid)

data/neighbours.go

@@ -2,11 +2,11 @@ package data
 
 // Neighbours struct
 type Neighbours struct {
-	Batadv         map[string]BatadvNeighbours `json:"batadv"`
-	LLDP           map[string]LLDPNeighbours   `json:"lldp"`
-	Babel          map[string]BabelNeighbours  `json:"babel"`
-	WifiNeighbours map[string]WifiNeighbours   `json:"wifi"`
-	NodeID         string                      `json:"node_id"`
+	Batadv map[string]BatadvNeighbours `json:"batadv"`
+	Babel  map[string]BabelNeighbours  `json:"babel"`
+	LLDP   map[string]LLDPNeighbours   `json:"lldp"`
+	//WifiNeighbours map[string]WifiNeighbours   `json:"wifi"`
+	NodeID string `json:"node_id"`
 }
 
 // WifiLink struct
@@ -22,24 +22,32 @@ type BatmanLink struct {
 	Tq       int     `json:"tq"`
 }
 
-// BabelLink struct
-type BabelLink struct {
-	Address string `json:"address"`
-}
-
 // LLDPLink struct
 type LLDPLink struct {
 	Name        string `json:"name"`
 	Description string `json:"descr"`
 }
 
+// BabelLink struct
+type BabelLink struct {
+	// How need this:
+	RXCost       int `json:"rxcost"`
+	TXCost       int `json:"txcost"`
+	Cost         int `json:"cost"`
+	Reachability int `json:"reachability"`
+}
+
 // BatadvNeighbours struct
 type BatadvNeighbours struct {
 	Neighbours map[string]BatmanLink `json:"neighbours"`
 }
 
 // BabelNeighbours struct
-type BabelNeighbours []BabelLink
+type BabelNeighbours struct {
+	Protocol         string               `json:"protocol"`
+	LinkLocalAddress string               `json:"ll-addr"`
+	Neighbours       map[string]BabelLink `json:"neighbours"`
+}
 
 // WifiNeighbours struct
 type WifiNeighbours struct {

data/nodeinfo.go

@@ -1,7 +1,7 @@
 package data
 
-// NodeInfo struct
-type NodeInfo struct {
+// Nodeinfo struct
+type Nodeinfo struct {
 	NodeID   string    `json:"node_id"`
 	Network  Network   `json:"network"`
 	Owner    *Owner    `json:"owner"`
@@ -14,8 +14,8 @@ type NodeInfo struct {
 	Wireless *Wireless `json:"wireless,omitempty"`
 }
 
-// BatInterface struct
-type BatInterface struct {
+// NetworkInterface struct
+type NetworkInterface struct {
 	Interfaces struct {
 		Wireless []string `json:"wireless,omitempty"`
 		Other    []string `json:"other,omitempty"`
@@ -23,12 +23,18 @@ type BatInterface struct {
 	} `json:"interfaces"`
 }
 
+// Addresses returns a flat list of all MAC addresses
+func (iface *NetworkInterface) Addresses() []string {
+	return append(append(iface.Interfaces.Other, iface.Interfaces.Tunnel...), iface.Interfaces.Wireless...)
+}
+
 // Network struct
 type Network struct {
-	Mac            string                   `json:"mac"`
-	Addresses      []string                 `json:"addresses"`
-	Mesh           map[string]*BatInterface `json:"mesh"`
-	MeshInterfaces []string                 `json:"mesh_interfaces"`
+	Mac       string                       `json:"mac"`
+	Addresses []string                     `json:"addresses"`
+	Mesh      map[string]*NetworkInterface `json:"mesh"`
+	// still used in gluon?
+	MeshInterfaces []string `json:"mesh_interfaces"`
 }
 
 // Owner struct
@@ -38,14 +44,15 @@ type Owner struct {
 
 // System struct
 type System struct {
-	SiteCode string `json:"site_code,omitempty"`
+	SiteCode   string `json:"site_code,omitempty"`
+	DomainCode string `json:"domain_code,omitempty"`
 }
 
 // Location struct
 type Location struct {
-	Longtitude float64 `json:"longitude,omitempty"`
-	Latitude   float64 `json:"latitude,omitempty"`
-	Altitude   float64 `json:"altitude,omitempty"`
+	Longitude float64 `json:"longitude,omitempty"`
+	Latitude  float64 `json:"latitude,omitempty"`
+	Altitude  float64 `json:"altitude,omitempty"`
 }
 
 // Software struct
@@ -58,9 +65,13 @@ type Software struct {
 		Version string `json:"version,omitempty"`
 		Compat  int    `json:"compat,omitempty"`
 	} `json:"batman-adv,omitempty"`
-	Fastd struct {
-		Enabled bool   `json:"enabled,omitempty"`
+	Babeld struct {
 		Version string `json:"version,omitempty"`
+	} `json:"babeld,omitempty"`
+	Fastd struct {
+		Enabled   bool   `json:"enabled,omitempty"`
+		Version   string `json:"version,omitempty"`
+		PublicKey string `json:"public_key,omitempty"`
 	} `json:"fastd,omitempty"`
 	Firmware struct {
 		Base    string `json:"base,omitempty"`

data/nodeinfo_test.go

@@ -0,0 +1,26 @@
+package data
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNodeinfoBatAddresses(t *testing.T) {
+	assert := assert.New(t)
+	iface := &NetworkInterface{
+		Interfaces: struct {
+			Wireless []string `json:"wireless,omitempty"`
+			Other    []string `json:"other,omitempty"`
+			Tunnel   []string `json:"tunnel,omitempty"`
+		}{
+			Wireless: nil,
+			Other:    []string{"aa:aa:aa:aa:aa", "aa:aa:aa:aa:ab"},
+			Tunnel:   []string{},
+		},
+	}
+
+	addr := iface.Addresses()
+	assert.NotNil(addr)
+	assert.Equal([]string{"aa:aa:aa:aa:aa", "aa:aa:aa:aa:ab"}, addr)
+}

data/response.go

@@ -3,6 +3,6 @@ package data
 // ResponseData struct
 type ResponseData struct {
 	Neighbours *Neighbours `json:"neighbours"`
-	NodeInfo   *NodeInfo   `json:"nodeinfo"`
+	Nodeinfo   *Nodeinfo   `json:"nodeinfo"`
 	Statistics *Statistics `json:"statistics"`
 }

data/statistics.go

@@ -9,6 +9,7 @@ package data
 type Statistics struct {
 	NodeID         string  `json:"node_id"`
 	Clients        Clients `json:"clients"`
+	DHCP           *DHCP   `json:"dhcp"`
 	RootFsUsage    float64 `json:"rootfs_usage,omitempty"`
 	LoadAverage    float64 `json:"loadavg,omitempty"`
 	Memory         Memory  `json:"memory,omitempty"`
@@ -29,8 +30,9 @@ type Statistics struct {
 		MgmtTx  *Traffic `json:"mgmt_tx"`
 		MgmtRx  *Traffic `json:"mgmt_rx"`
 	} `json:"traffic,omitempty"`
-	Switch   map[string]*SwitchPort `json:"switch,omitempty"`
-	Wireless WirelessStatistics     `json:"wireless,omitempty"`
+	Switch    map[string]*SwitchPort `json:"switch,omitempty"`
+	Wireless  WirelessStatistics     `json:"wireless,omitempty"`
+	ProcStats *ProcStats             `json:"stat,omitempty"`
 }
 
 // MeshVPNPeerLink struct
@@ -64,15 +66,52 @@ type Clients struct {
 	Total  uint32 `json:"total"`
 }
 
+// DHCP struct
+type DHCP struct {
+	// Packet counters
+	Decline  uint32 `json:"dhcp_decline"`
+	Offer    uint32 `json:"dhcp_offer"`
+	Ack      uint32 `json:"dhcp_ack"`
+	Nak      uint32 `json:"dhcp_nak"`
+	Request  uint32 `json:"dhcp_request"`
+	Discover uint32 `json:"dhcp_discover"`
+	Inform   uint32 `json:"dhcp_inform"`
+	Release  uint32 `json:"dhcp_release"`
+
+	LeasesAllocated uint32 `json:"leases_allocated_4"`
+	LeasesPruned    uint32 `json:"leases_pruned_4"`
+}
+
 // Memory struct
 type Memory struct {
-	Cached  uint32 `json:"cached"`
-	Total   uint32 `json:"total"`
-	Buffers uint32 `json:"buffers"`
-	Free    uint32 `json:"free"`
+	Cached    int64 `json:"cached"`
+	Total     int64 `json:"total"`
+	Buffers   int64 `json:"buffers"`
+	Free      int64 `json:"free,omitempty"`
+	Available int64 `json:"available,omitempty"`
 }
 
 // SwitchPort struct
 type SwitchPort struct {
 	Speed uint32 `json:"speed"`
 }
+
+// ProcStats struct
+type ProcStats struct {
+	CPU             ProcStatsCPU `json:"cpu"`
+	Intr            int64        `json:"intr"`
+	ContextSwitches int64        `json:"ctxt"`
+	SoftIRQ         int64        `json:"softirq"`
+	Processes       int64        `json:"processes"`
+}
+
+// ProcStatsCPU struct
+type ProcStatsCPU struct {
+	User    int64 `json:"user"`
+	Nice    int64 `json:"nice"`
+	System  int64 `json:"system"`
+	Idle    int64 `json:"idle"`
+	IOWait  int64 `json:"iowait"`
+	IRQ     int64 `json:"irq"`
+	SoftIRQ int64 `json:"softirq"`
+}

data/statistics_airtime.go

@@ -23,7 +23,7 @@ type WirelessAirtime struct {
 	BusyTime   uint64 `json:"busy"`
 	RxTime     uint64 `json:"rx"`
 	TxTime     uint64 `json:"tx"`
-	Noise      uint32 `json:"noise"`
+	Noise      int32  `json:"noise"`
 	Frequency  uint32 `json:"frequency"`
 }
 

database/all/connection.go

@@ -0,0 +1,80 @@
+package all
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/bdlm/log"
+
+	"github.com/FreifunkBremen/yanic/database"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Connection struct {
+	database.Connection
+	list []database.Connection
+}
+
+func Connect(allConnection map[string]interface{}) (database.Connection, error) {
+	var list []database.Connection
+	for dbType, conn := range database.Adapters {
+		configForType := allConnection[dbType]
+		if configForType == nil {
+			log.WithField("database", dbType).Infof("no configuration found")
+			continue
+		}
+		dbConfigs, ok := configForType.([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("the database type '%s' has the wrong format", dbType)
+		}
+
+		for _, dbConfig := range dbConfigs {
+			config, ok := dbConfig.(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("the database type '%s' has the wrong format", dbType)
+			}
+			if c, ok := config["enable"].(bool); ok && !c {
+				continue
+			}
+			connected, err := conn(config)
+			if err != nil {
+				return nil, err
+			}
+			if connected == nil {
+				continue
+			}
+			list = append(list, connected)
+		}
+	}
+	return &Connection{list: list}, nil
+}
+
+func (conn *Connection) InsertNode(node *runtime.Node) {
+	for _, item := range conn.list {
+		item.InsertNode(node)
+	}
+}
+
+func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
+	for _, item := range conn.list {
+		item.InsertLink(link, time)
+	}
+}
+
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
+	for _, item := range conn.list {
+		item.InsertGlobals(stats, time, site, domain)
+	}
+}
+
+func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
+	for _, item := range conn.list {
+		item.PruneNodes(deleteAfter)
+	}
+}
+
+func (conn *Connection) Close() {
+	for _, item := range conn.list {
+		item.Close()
+	}
+}

database/all/internal.go

@@ -1,56 +1,45 @@
 package all
 
 import (
+	"sync"
 	"time"
 
 	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/runtime"
 )
 
-type Connection struct {
-	database.Connection
-	list []database.Connection
+var Conn database.Connection
+var wg = sync.WaitGroup{}
+var quit chan struct{}
+
+func Start(config database.Config) (err error) {
+	Conn, err = Connect(config.Connection)
+	if err != nil {
+		return
+	}
+	quit = make(chan struct{})
+	wg.Add(1)
+	go deleteWorker(config.DeleteInterval.Duration, config.DeleteAfter.Duration)
+	return
 }
 
-func Connect(configuration interface{}) (database.Connection, error) {
-	var list []database.Connection
-	allConnection := configuration.(map[string][]interface{})
-	for dbType, conn := range database.Adapters {
-		dbConfigs := allConnection[dbType]
-		for _, config := range dbConfigs {
-			connected, err := conn(config)
-			if err != nil {
-				return nil, err
-			}
-			if connected == nil {
-				continue
-			}
-			list = append(list, connected)
+func Close() {
+	close(quit)
+	wg.Wait()
+	Conn.Close()
+	quit = nil
+}
+
+// prunes node-specific data periodically
+func deleteWorker(deleteInterval time.Duration, deleteAfter time.Duration) {
+	ticker := time.NewTicker(deleteInterval)
+	for {
+		select {
+		case <-ticker.C:
+			Conn.PruneNodes(deleteAfter)
+		case <-quit:
+			ticker.Stop()
+			wg.Done()
+			return
 		}
 	}
-	return &Connection{list: list}, nil
-}
-
-func (conn *Connection) InsertNode(node *runtime.Node) {
-	for _, item := range conn.list {
-		item.InsertNode(node)
-	}
-}
-
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
-	for _, item := range conn.list {
-		item.InsertGlobals(stats, time)
-	}
-}
-
-func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
-	for _, item := range conn.list {
-		item.PruneNodes(deleteAfter)
-	}
-}
-
-func (conn *Connection) Close() {
-	for _, item := range conn.list {
-		item.Close()
-	}
 }

database/all/internel_test.go

@@ -0,0 +1,75 @@
+package all
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/FreifunkBremen/yanic/database"
+	"github.com/FreifunkBremen/yanic/lib/duration"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStart(t *testing.T) {
+	assert := assert.New(t)
+
+	database.RegisterAdapter("d", func(config map[string]interface{}) (database.Connection, error) {
+		return nil, nil
+	})
+	database.RegisterAdapter("e", func(config map[string]interface{}) (database.Connection, error) {
+		return nil, errors.New("blub")
+	})
+	// Test for PruneNodes (by start)
+	assert.Nil(quit)
+	err := Start(database.Config{
+		DeleteInterval: duration.Duration{Duration: time.Millisecond},
+		Connection: map[string]interface{}{
+			"a": []map[string]interface{}{
+				{
+					"enable": false,
+					"path":   "a1",
+				},
+				{
+					"path": "a2",
+				},
+				{
+					"enable": true,
+					"path":   "a3",
+				},
+			},
+			"b": nil,
+			"c": []interface{}{
+				map[string]interface{}{
+					"path": "c1",
+				},
+			},
+			// fetch continue command in Connect
+			"d": []interface{}{
+				map[string]interface{}{
+					"path": "d0",
+				},
+			},
+		},
+	})
+	assert.NoError(err)
+	assert.NotNil(quit)
+
+	// connection type not found
+	_, err = Connect(map[string]interface{}{
+		"e": []map[string]interface{}{
+			{},
+		},
+	})
+	assert.Error(err)
+
+	// test close
+	Close()
+
+	// wrong format
+	err = Start(database.Config{
+		Connection: map[string]interface{}{
+			"e": true,
+		},
+	})
+	assert.Error(err)
+}

database/all/main.go

@@ -4,6 +4,5 @@ import (
 	_ "github.com/FreifunkBremen/yanic/database/graphite"
 	_ "github.com/FreifunkBremen/yanic/database/influxdb"
 	_ "github.com/FreifunkBremen/yanic/database/logging"
-	_ "github.com/FreifunkBremen/yanic/database/socket"
-	_ "github.com/FreifunkBremen/yanic/database/yanic"
+	_ "github.com/FreifunkBremen/yanic/database/respondd"
 )

database/config.go

@@ -0,0 +1,9 @@
+package database
+
+import "github.com/FreifunkBremen/yanic/lib/duration"
+
+type Config struct {
+	DeleteInterval duration.Duration `toml:"delete_interval"` // Delete stats of nodes every n minutes
+	DeleteAfter    duration.Duration `toml:"delete_after"`    // Delete stats of nodes till now-deletetill n minutes
+	Connection     map[string]interface{}
+}

database/database.go

@@ -11,8 +11,11 @@ type Connection interface {
 	// InsertNode stores statistics per node
 	InsertNode(node *runtime.Node)
 
+	// InsertLink stores statistics per link
+	InsertLink(*runtime.Link, time.Time)
+
 	// InsertGlobals stores global statistics
-	InsertGlobals(stats *runtime.GlobalStats, time time.Time)
+	InsertGlobals(*runtime.GlobalStats, time.Time, string, string)
 
 	// PruneNodes prunes historical per-node data
 	PruneNodes(deleteAfter time.Duration)
@@ -22,7 +25,7 @@ type Connection interface {
 }
 
 // Connect function with config to get DB connection interface
-type Connect func(config interface{}) (Connection, error)
+type Connect func(config map[string]interface{}) (Connection, error)
 
 // Adapters is the list of registered database adapters
 var Adapters = map[string]Connect{}

database/database_test.go

@@ -0,0 +1,18 @@
+package database
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegister(t *testing.T) {
+	assert := assert.New(t)
+	assert.Len(Adapters, 0)
+
+	RegisterAdapter("blub", func(config map[string]interface{}) (Connection, error) {
+		return nil, nil
+	})
+
+	assert.Len(Adapters, 1)
+}

database/graphite/database.go

@@ -1,17 +1,20 @@
 package graphite
 
 import (
-	"github.com/FreifunkBremen/yanic/database"
-	"github.com/fgrosse/graphigo"
-	"log"
 	"sync"
+
+	"github.com/bdlm/log"
+	"github.com/fgrosse/graphigo"
+
+	"github.com/FreifunkBremen/yanic/database"
 )
 
 const (
-	MeasurementNode            = "node"     // Measurement for per-node statistics
-	MeasurementGlobal          = "global"   // Measurement for summarized global statistics
-	CounterMeasurementFirmware = "firmware" // Measurement for firmware statistics
-	CounterMeasurementModel    = "model"    // Measurement for model statistics
+	MeasurementNode               = "node"        // Measurement for per-node statistics
+	MeasurementGlobal             = "global"      // Measurement for summarized global statistics
+	CounterMeasurementFirmware    = "firmware"    // Measurement for firmware statistics
+	CounterMeasurementModel       = "model"       // Measurement for model statistics
+	CounterMeasurementAutoupdater = "autoupdater" // Measurement for autoupdater
 )
 
 type Connection struct {
@@ -31,18 +34,10 @@ func (c Config) Prefix() string {
 	return c["prefix"].(string)
 }
 
-func (c Config) Enable() bool {
-	return c["enable"].(bool)
-}
-
-func Connect(configuration interface{}) (database.Connection, error) {
+func Connect(configuration map[string]interface{}) (database.Connection, error) {
 	var config Config
 
-	config = configuration.(map[string]interface{})
-
-	if !config.Enable() {
-		return nil, nil
-	}
+	config = configuration
 
 	con := &Connection{
 		client: graphigo.Client{
@@ -75,7 +70,7 @@ func (c *Connection) addWorker() {
 	for point := range c.points {
 		err := c.client.SendAll(point)
 		if err != nil {
-			log.Fatal(err)
+			log.WithField("database", "graphite").Fatal(err)
 			return
 		}
 	}

database/graphite/global.go

@@ -7,27 +7,47 @@ import (
 	"github.com/fgrosse/graphigo"
 )
 
-func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
-	c.addPoint(GlobalStatsFields(stats))
-	c.addCounterMap(CounterMeasurementModel, stats.Models, time)
-	c.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time)
+func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
+	measurementGlobal := MeasurementGlobal
+	counterMeasurementModel := CounterMeasurementModel
+	counterMeasurementFirmware := CounterMeasurementFirmware
+	counterMeasurementAutoupdater := CounterMeasurementAutoupdater
+
+	if site != runtime.GLOBAL_SITE {
+		measurementGlobal += "_" + site
+		counterMeasurementModel += "_" + site
+		counterMeasurementFirmware += "_" + site
+		counterMeasurementAutoupdater += "_" + site
+	}
+
+	if domain != runtime.GLOBAL_DOMAIN {
+		measurementGlobal += "_" + domain
+		counterMeasurementModel += "_" + domain
+		counterMeasurementFirmware += "_" + domain
+		counterMeasurementAutoupdater += "_" + domain
+	}
+
+	c.addPoint(GlobalStatsFields(measurementGlobal, stats))
+	c.addCounterMap(counterMeasurementModel, stats.Models, time)
+	c.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time)
+	c.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time)
 }
 
-func GlobalStatsFields(stats *runtime.GlobalStats) []graphigo.Metric {
+func GlobalStatsFields(name string, stats *runtime.GlobalStats) []graphigo.Metric {
 	return []graphigo.Metric{
-		{Name: MeasurementGlobal + ".nodes", Value: stats.Nodes},
-		{Name: MeasurementGlobal + ".gateways", Value: stats.Gateways},
-		{Name: MeasurementGlobal + ".clients.total", Value: stats.Clients},
-		{Name: MeasurementGlobal + ".clients.wifi", Value: stats.ClientsWifi},
-		{Name: MeasurementGlobal + ".clients.wifi24", Value: stats.ClientsWifi24},
-		{Name: MeasurementGlobal + ".clients.wifi5", Value: stats.ClientsWifi5},
+		{Name: name + ".nodes", Value: stats.Nodes},
+		{Name: name + ".gateways", Value: stats.Gateways},
+		{Name: name + ".clients.total", Value: stats.Clients},
+		{Name: name + ".clients.wifi", Value: stats.ClientsWifi},
+		{Name: name + ".clients.wifi24", Value: stats.ClientsWifi24},
+		{Name: name + ".clients.wifi5", Value: stats.ClientsWifi5},
 	}
 }
 
 func (c *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time) {
 	var fields []graphigo.Metric
 	for key, count := range m {
-		fields = append(fields, graphigo.Metric{Name: name + `.` + key + `.count`, Value: count, Timestamp: t})
+		fields = append(fields, graphigo.Metric{Name: name + `.` + replaceInvalidChars(key) + `.count`, Value: count, Timestamp: t})
 	}
 	c.addPoint(fields)
 }

database/graphite/link.go

@@ -0,0 +1,11 @@
+package graphite
+
+import (
+	"time"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+// InsertLink stores per link statistics
+func (c *Connection) InsertLink(link *runtime.Link, time time.Time) {
+}

database/graphite/node.go

@@ -1,7 +1,6 @@
 package graphite
 
 import (
-	"strings"
 	"time"
 
 	"github.com/FreifunkBremen/yanic/runtime"
@@ -25,35 +24,39 @@ func (c *Connection) InsertNode(node *runtime.Node) {
 		return
 	}
 
-	node_prefix := MeasurementNode + `.` + stats.NodeID + `.` + strings.Replace(nodeinfo.Hostname, ".", "__", -1)
+	node_prefix := MeasurementNode + `.` + stats.NodeID + `.` + replaceInvalidChars(nodeinfo.Hostname)
 
 	addField := func(name string, value interface{}) {
 		fields = append(fields, graphigo.Metric{Name: node_prefix + "." + name, Value: value})
 	}
 
+	vpnInterfaces := make(map[string]bool)
+	for _, mIface := range nodeinfo.Network.Mesh {
+		for _, tunnel := range mIface.Interfaces.Tunnel {
+			vpnInterfaces[tunnel] = true
+		}
+	}
+
 	if neighbours := node.Neighbours; neighbours != nil {
 		vpn := 0
-		if meshvpn := stats.MeshVPN; meshvpn != nil {
-			for _, group := range meshvpn.Groups {
-				for _, link := range group.Peers {
-					if link != nil && link.Established > 1 {
-						vpn++
-					}
-				}
-			}
-		}
-		addField("neighbours.vpn", vpn)
+
 		// protocol: Batman Advance
 		batadv := 0
-		for _, batadvNeighbours := range neighbours.Batadv {
+		for mac, batadvNeighbours := range neighbours.Batadv {
 			batadv += len(batadvNeighbours.Neighbours)
+			if _, ok := vpnInterfaces[mac]; ok {
+				vpn += len(batadvNeighbours.Neighbours)
+			}
 		}
 		addField("neighbours.batadv", batadv)
 
 		// protocol: Babel
 		babel := 0
 		for _, babelNeighbours := range neighbours.Babel {
-			babel += len(babelNeighbours)
+			babel += len(babelNeighbours.Neighbours)
+			if _, ok := vpnInterfaces[babelNeighbours.LinkLocalAddress]; ok {
+				vpn += len(babelNeighbours.Neighbours)
+			}
 		}
 		addField("neighbours.babel", babel)
 
@@ -64,8 +67,10 @@ func (c *Connection) InsertNode(node *runtime.Node) {
 		}
 		addField("neighbours.lldp", lldp)
 
+		addField("neighbours.vpn", vpn)
+
 		// total is the sum of all protocols
-		addField("neighbours.total", batadv+lldp)
+		addField("neighbours.total", batadv+babel+lldp)
 	}
 
 	if t := stats.Traffic.Rx; t != nil {
@@ -100,6 +105,7 @@ func (c *Connection) InsertNode(node *runtime.Node) {
 	}
 
 	addField("load", stats.LoadAverage)
+	addField("nproc", nodeinfo.Hardware.Nproc)
 	addField("time.up", int64(stats.Uptime))
 	addField("time.idle", int64(stats.Idletime))
 	addField("proc.running", stats.Processes.Running)
@@ -111,6 +117,7 @@ func (c *Connection) InsertNode(node *runtime.Node) {
 	addField("memory.cached", stats.Memory.Cached)
 	addField("memory.free", stats.Memory.Free)
 	addField("memory.total", stats.Memory.Total)
+	addField("memory.available", stats.Memory.Available)
 
 	c.addPoint(fields)
 }

database/graphite/utils.go

@@ -0,0 +1,11 @@
+package graphite
+
+import (
+	"regexp"
+)
+
+var reInvalidChars = regexp.MustCompile("(?i)[^a-z0-9\\-]")
+
+func replaceInvalidChars(name string) string {
+	return reInvalidChars.ReplaceAllString(name, "_")
+}

database/influxdb/database.go

@@ -1,23 +1,26 @@
 package influxdb
 
 import (
-	"log"
 	"sync"
 	"time"
 
-	"github.com/influxdata/influxdb/client/v2"
-	"github.com/influxdata/influxdb/models"
+	"github.com/bdlm/log"
+	"github.com/influxdata/influxdb1-client/models"
+	"github.com/influxdata/influxdb1-client/v2"
 
 	"github.com/FreifunkBremen/yanic/database"
 )
 
 const (
-	MeasurementNode            = "node"     // Measurement for per-node statistics
-	MeasurementGlobal          = "global"   // Measurement for summarized global statistics
-	CounterMeasurementFirmware = "firmware" // Measurement for firmware statistics
-	CounterMeasurementModel    = "model"    // Measurement for model statistics
-	batchMaxSize               = 500
-	batchTimeout               = 5 * time.Second
+	MeasurementLink               = "link"        // Measurement for per-link statistics
+	MeasurementNode               = "node"        // Measurement for per-node statistics
+	MeasurementDHCP               = "dhcp"        // Measurement for DHCP server statistics
+	MeasurementGlobal             = "global"      // Measurement for summarized global statistics
+	CounterMeasurementFirmware    = "firmware"    // Measurement for firmware statistics
+	CounterMeasurementModel       = "model"       // Measurement for model statistics
+	CounterMeasurementAutoupdater = "autoupdater" // Measurement for autoupdater
+	batchMaxSize                  = 1000
+	batchTimeout                  = 5 * time.Second
 )
 
 type Connection struct {
@@ -30,9 +33,6 @@ type Connection struct {
 
 type Config map[string]interface{}
 
-func (c Config) Enable() bool {
-	return c["enable"].(bool)
-}
 func (c Config) Address() string {
 	return c["address"].(string)
 }
@@ -45,6 +45,12 @@ func (c Config) Username() string {
 func (c Config) Password() string {
 	return c["password"].(string)
 }
+func (c Config) InsecureSkipVerify() bool {
+	if d, ok := c["insecure_skip_verify"]; ok {
+		return d.(bool)
+	}
+	return false
+}
 func (c Config) Tags() map[string]interface{} {
 	if c["tags"] != nil {
 		return c["tags"].(map[string]interface{})
@@ -55,27 +61,31 @@ func (c Config) Tags() map[string]interface{} {
 func init() {
 	database.RegisterAdapter("influxdb", Connect)
 }
-func Connect(configuration interface{}) (database.Connection, error) {
+func Connect(configuration map[string]interface{}) (database.Connection, error) {
 	var config Config
-	config = configuration.(map[string]interface{})
-	if !config.Enable() {
-		return nil, nil
-	}
+	config = configuration
+
 	// Make client
 	c, err := client.NewHTTPClient(client.HTTPConfig{
-		Addr:     config.Address(),
-		Username: config.Username(),
-		Password: config.Password(),
+		Addr:               config.Address(),
+		Username:           config.Username(),
+		Password:           config.Password(),
+		InsecureSkipVerify: config.InsecureSkipVerify(),
 	})
 
 	if err != nil {
 		return nil, err
 	}
 
+	_, _, err = c.Ping(time.Millisecond * 50)
+	if err != nil {
+		return nil, err
+	}
+
 	db := &Connection{
 		config: config,
 		client: c,
-		points: make(chan *client.Point, 1000),
+		points: make(chan *client.Point, batchMaxSize),
 	}
 
 	db.wg.Add(1)
@@ -90,13 +100,16 @@ func (conn *Connection) addPoint(name string, tags models.Tags, fields models.Fi
 			if value, ok := valueInterface.(string); ok && tags.Get([]byte(tag)) == nil {
 				tags.SetString(tag, value)
 			} else {
-				log.Println(name, "could not saved configured value of tag", tag)
+				log.WithFields(map[string]interface{}{
+					"name": name,
+					"tag":  tag,
+				}).Warnf("count not save tag configuration on point")
 			}
 		}
 	}
 	point, err := client.NewPoint(name, tags.Map(), fields, t...)
 	if err != nil {
-		panic(err)
+		log.Panicf("count not save points: %s", err)
 	}
 	conn.points <- point
 }
@@ -146,10 +159,10 @@ func (conn *Connection) addWorker() {
 
 		// write batch now?
 		if bp != nil && (writeNow || closed || len(bp.Points()) >= batchMaxSize) {
-			log.Println("saving", len(bp.Points()), "points")
+			log.WithField("count", len(bp.Points())).Info("saving points")
 
 			if err = conn.client.Write(bp); err != nil {
-				log.Print(err)
+				log.Error(err)
 			}
 			writeNow = false
 			bp = nil

database/influxdb/database_test.go

@@ -1,15 +1,54 @@
 package influxdb
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
-	"github.com/influxdata/influxdb/client/v2"
-	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/influxdb1-client/models"
+	"github.com/influxdata/influxdb1-client/v2"
 
 	"github.com/stretchr/testify/assert"
 )
 
+func TestConnect(t *testing.T) {
+	assert := assert.New(t)
+
+	conn, err := Connect(map[string]interface{}{
+		"address":              "",
+		"username":             "",
+		"password":             "",
+		"insecure_skip_verify": true,
+	})
+	assert.Nil(conn)
+	assert.Error(err)
+
+	conn, err = Connect(map[string]interface{}{
+		"address":  "http://localhost",
+		"database": "",
+		"username": "",
+		"password": "",
+	})
+	assert.Nil(conn)
+	assert.Error(err)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer srv.Close()
+
+	conn, err = Connect(map[string]interface{}{
+		"address":  srv.URL,
+		"database": "",
+		"username": "",
+		"password": "",
+	})
+
+	assert.NotNil(conn)
+	assert.NoError(err)
+}
+
 func TestAddPoint(t *testing.T) {
 	assert := assert.New(t)
 

database/influxdb/global.go

@@ -4,14 +4,39 @@ import (
 	"time"
 
 	"github.com/FreifunkBremen/yanic/runtime"
-	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/influxdb1-client/models"
 )
 
 // InsertGlobals implementation of database
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
-	conn.addPoint(MeasurementGlobal, nil, GlobalStatsFields(stats), time)
-	conn.addCounterMap(CounterMeasurementModel, stats.Models, time)
-	conn.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time)
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
+	tags := models.Tags{}
+
+	measurementGlobal := MeasurementGlobal
+	counterMeasurementModel := CounterMeasurementModel
+	counterMeasurementFirmware := CounterMeasurementFirmware
+	counterMeasurementAutoupdater := CounterMeasurementAutoupdater
+
+	if site != runtime.GLOBAL_SITE {
+		tags.Set([]byte("site"), []byte(site))
+
+		measurementGlobal += "_site"
+		counterMeasurementModel += "_site"
+		counterMeasurementFirmware += "_site"
+		counterMeasurementAutoupdater += "_site"
+	}
+	if domain != runtime.GLOBAL_DOMAIN {
+		tags.Set([]byte("domain"), []byte(domain))
+
+		measurementGlobal += "_domain"
+		counterMeasurementModel += "_domain"
+		counterMeasurementFirmware += "_domain"
+		counterMeasurementAutoupdater += "_domain"
+	}
+
+	conn.addPoint(measurementGlobal, tags, GlobalStatsFields(stats), time)
+	conn.addCounterMap(counterMeasurementModel, stats.Models, time, site, domain)
+	conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site, domain)
+	conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site, domain)
 }
 
 // GlobalStatsFields returns fields for InfluxDB
@@ -29,12 +54,14 @@ func GlobalStatsFields(stats *runtime.GlobalStats) map[string]interface{} {
 // Saves the values of a CounterMap in the database.
 // The key are used as 'value' tag.
 // The value is used as 'counter' field.
-func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time) {
+func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string, domain string) {
 	for key, count := range m {
 		conn.addPoint(
 			name,
 			models.Tags{
 				models.Tag{Key: []byte("value"), Value: []byte(key)},
+				models.Tag{Key: []byte("site"), Value: []byte(site)},
+				models.Tag{Key: []byte("domain"), Value: []byte(domain)},
 			},
 			models.Fields{"count": count},
 			t,

database/influxdb/global_test.go

@@ -1,61 +1,171 @@
 package influxdb
 
 import (
+	"sync"
 	"testing"
+	"time"
 
+	"github.com/influxdata/influxdb1-client/v2"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/FreifunkBremen/yanic/data"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 
+const (
+	TEST_SITE   = "ffhb"
+	TEST_DOMAIN = "city"
+)
+
 func TestGlobalStats(t *testing.T) {
-	stats := runtime.NewGlobalStats(createTestNodes())
+	stats := runtime.NewGlobalStats(createTestNodes(), map[string][]string{TEST_SITE: {TEST_DOMAIN}})
 
 	assert := assert.New(t)
-	fields := GlobalStatsFields(stats)
 
-	// check fields
+	// check SITE_GLOBAL fields
+	fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE][runtime.GLOBAL_DOMAIN])
 	assert.EqualValues(3, fields["nodes"])
+
+	fields = GlobalStatsFields(stats[TEST_SITE][runtime.GLOBAL_DOMAIN])
+	assert.EqualValues(2, fields["nodes"])
+	fields = GlobalStatsFields(stats[TEST_SITE][TEST_DOMAIN])
+
+	assert.EqualValues(1, fields["nodes"])
+
+	conn := &Connection{
+		points: make(chan *client.Point),
+	}
+
+	global := 0
+	globalSite := 0
+	globalDomain := 0
+
+	model := 0
+	modelSite := 0
+	modelDomain := 0
+
+	firmware := 0
+	firmwareSite := 0
+	firmwareDomain := 0
+
+	autoupdater := 0
+	autoupdaterSite := 0
+	autoupdaterDomain := 0
+
+	wg := sync.WaitGroup{}
+	wg.Add(15)
+	go func() {
+		for p := range conn.points {
+			switch p.Name() {
+			case MeasurementGlobal:
+				global++
+			case "global_site":
+				globalSite++
+			case "global_site_domain":
+				globalDomain++
+
+			case CounterMeasurementModel:
+				model++
+			case "model_site":
+				modelSite++
+			case "model_site_domain":
+				modelDomain++
+
+			case CounterMeasurementFirmware:
+				firmware++
+			case "firmware_site":
+				firmwareSite++
+			case "firmware_site_domain":
+				firmwareDomain++
+
+			case CounterMeasurementAutoupdater:
+				autoupdater++
+			case "autoupdater_site":
+				autoupdaterSite++
+			case "autoupdater_site_domain":
+				autoupdaterDomain++
+
+			default:
+				assert.Equal("invalid p.Name found", p.Name())
+			}
+			wg.Done()
+		}
+	}()
+	for site, domains := range stats {
+		for domain, stat := range domains {
+			conn.InsertGlobals(stat, time.Now(), site, domain)
+		}
+	}
+	wg.Wait()
+	assert.Equal(1, global)
+	assert.Equal(1, globalSite)
+	assert.Equal(1, globalDomain)
+
+	assert.Equal(2, model)
+	assert.Equal(2, modelSite)
+	assert.Equal(1, modelDomain)
+
+	assert.Equal(1, firmware)
+	assert.Equal(1, firmwareSite)
+	assert.Equal(0, firmwareDomain)
+
+	assert.Equal(2, autoupdater)
+	assert.Equal(2, autoupdaterSite)
+	assert.Equal(1, autoupdaterDomain)
 }
 
 func createTestNodes() *runtime.Nodes {
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 
-	nodeData := &data.ResponseData{
+	nodeData := &runtime.Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 23,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "abcdef012345",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
+			System: data.System{
+				SiteCode: TEST_SITE,
+			},
 		},
 	}
-	nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
-	nodes.Update("abcdef012345", nodeData)
+	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
+	nodeData.Nodeinfo.Software.Autoupdater.Enabled = true
+	nodeData.Nodeinfo.Software.Autoupdater.Branch = "stable"
+	nodes.AddNode(nodeData)
 
-	nodes.Update("112233445566", &data.ResponseData{
+	nodes.AddNode(&runtime.Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 2,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "112233445566",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	})
 
-	nodes.Update("0xdeadbeef0x", &data.ResponseData{
-		NodeInfo: &data.NodeInfo{
-			VPN: true,
+	nodes.AddNode(&runtime.Node{
+		Online: true,
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "0xdeadbeef0x",
+			VPN:    true,
 			Hardware: data.Hardware{
 				Model: "Xeon Multi-Core",
 			},
+			System: data.System{
+				SiteCode:   TEST_SITE,
+				DomainCode: TEST_DOMAIN,
+			},
 		},
 	})
 

database/influxdb/link.go

@@ -0,0 +1,19 @@
+package influxdb
+
+import (
+	"time"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+	models "github.com/influxdata/influxdb1-client/models"
+)
+
+// InsertLink adds a link data point
+func (conn *Connection) InsertLink(link *runtime.Link, t time.Time) {
+	tags := models.Tags{}
+	tags.SetString("source.id", link.SourceID)
+	tags.SetString("source.addr", link.SourceAddress)
+	tags.SetString("target.id", link.TargetID)
+	tags.SetString("target.addr", link.TargetAddress)
+
+	conn.addPoint(MeasurementLink, tags, models.Fields{"tq": link.TQ * 100}, t)
+}

database/influxdb/node.go

@@ -5,46 +5,65 @@ import (
 	"strconv"
 	"time"
 
-	client "github.com/influxdata/influxdb/client/v2"
-	models "github.com/influxdata/influxdb/models"
+	models "github.com/influxdata/influxdb1-client/models"
+	client "github.com/influxdata/influxdb1-client/v2"
 
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 
-// InsertNode implementation of database
-func (conn *Connection) InsertNode(node *runtime.Node) {
-	tags, fields := buildNodeStats(node)
-	conn.addPoint(MeasurementNode, tags, fields, time.Now())
-}
-
+// PruneNodes prunes historical per-node data
 func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
-	query := fmt.Sprintf("delete from %s where time < now() - %ds", MeasurementNode, deleteAfter/time.Second)
-	conn.client.Query(client.NewQuery(query, conn.config.Database(), "m"))
-}
-
-// returns tags and fields for InfluxDB
-func buildNodeStats(node *runtime.Node) (tags models.Tags, fields models.Fields) {
-	stats := node.Statistics
-
-	tags.SetString("nodeid", stats.NodeID)
-
-	fields = map[string]interface{}{
-		"load":           stats.LoadAverage,
-		"time.up":        int64(stats.Uptime),
-		"time.idle":      int64(stats.Idletime),
-		"proc.running":   stats.Processes.Running,
-		"clients.wifi":   stats.Clients.Wifi,
-		"clients.wifi24": stats.Clients.Wifi24,
-		"clients.wifi5":  stats.Clients.Wifi5,
-		"clients.total":  stats.Clients.Total,
-		"memory.buffers": stats.Memory.Buffers,
-		"memory.cached":  stats.Memory.Cached,
-		"memory.free":    stats.Memory.Free,
-		"memory.total":   stats.Memory.Total,
+	for _, measurement := range []string{MeasurementNode, MeasurementLink} {
+		query := fmt.Sprintf("delete from %s where time < now() - %ds", measurement, deleteAfter/time.Second)
+		conn.client.Query(client.NewQuery(query, conn.config.Database(), "m"))
 	}
 
+}
+
+// InsertNode stores statistics and neighbours in the database
+func (conn *Connection) InsertNode(node *runtime.Node) {
+	stats := node.Statistics
+	time := node.Lastseen.GetTime()
+
+	if stats == nil || stats.NodeID == "" {
+		return
+	}
+
+	tags := models.Tags{}
+	tags.SetString("nodeid", stats.NodeID)
+
+	fields := models.Fields{
+		"load":             stats.LoadAverage,
+		"time.up":          int64(stats.Uptime),
+		"time.idle":        int64(stats.Idletime),
+		"proc.running":     stats.Processes.Running,
+		"clients.wifi":     stats.Clients.Wifi,
+		"clients.wifi24":   stats.Clients.Wifi24,
+		"clients.wifi5":    stats.Clients.Wifi5,
+		"clients.total":    stats.Clients.Total,
+		"memory.buffers":   stats.Memory.Buffers,
+		"memory.cached":    stats.Memory.Cached,
+		"memory.free":      stats.Memory.Free,
+		"memory.total":     stats.Memory.Total,
+		"memory.available": stats.Memory.Available,
+	}
+
+	vpnInterfaces := make(map[string]bool)
+
 	if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+		for _, mIface := range nodeinfo.Network.Mesh {
+			for _, tunnel := range mIface.Interfaces.Tunnel {
+				vpnInterfaces[tunnel] = true
+			}
+		}
+
 		tags.SetString("hostname", nodeinfo.Hostname)
+		if nodeinfo.System.SiteCode != "" {
+			tags.SetString("site", nodeinfo.System.SiteCode)
+		}
+		if nodeinfo.System.DomainCode != "" {
+			tags.SetString("domain", nodeinfo.System.DomainCode)
+		}
 		if owner := nodeinfo.Owner; owner != nil {
 			tags.SetString("owner", owner.Contact)
 		}
@@ -54,36 +73,37 @@ func buildNodeStats(node *runtime.Node) (tags models.Tags, fields models.Fields)
 		}
 		// Hardware
 		tags.SetString("model", nodeinfo.Hardware.Model)
+		fields["nproc"] = nodeinfo.Hardware.Nproc
 		tags.SetString("firmware_base", nodeinfo.Software.Firmware.Base)
 		tags.SetString("firmware_release", nodeinfo.Software.Firmware.Release)
+		if nodeinfo.Software.Autoupdater.Enabled {
+			tags.SetString("autoupdater", nodeinfo.Software.Autoupdater.Branch)
+		} else {
+			tags.SetString("autoupdater", runtime.DISABLED_AUTOUPDATER)
+		}
 
 	}
-
 	if neighbours := node.Neighbours; neighbours != nil {
 		// VPN Neighbours are Neighbours but includet in one protocol
 		vpn := 0
-		if meshvpn := stats.MeshVPN; meshvpn != nil {
-			for _, group := range meshvpn.Groups {
-				for _, link := range group.Peers {
-					if link != nil && link.Established > 1 {
-						vpn++
-					}
-				}
-			}
-		}
-		fields["neighbours.vpn"] = vpn
 
 		// protocol: Batman Advance
 		batadv := 0
-		for _, batadvNeighbours := range neighbours.Batadv {
+		for mac, batadvNeighbours := range neighbours.Batadv {
 			batadv += len(batadvNeighbours.Neighbours)
+			if _, ok := vpnInterfaces[mac]; ok {
+				vpn += len(batadvNeighbours.Neighbours)
+			}
 		}
 		fields["neighbours.batadv"] = batadv
 
 		// protocol: Babel
 		babel := 0
 		for _, babelNeighbours := range neighbours.Babel {
-			babel += len(babelNeighbours)
+			babel += len(babelNeighbours.Neighbours)
+			if _, ok := vpnInterfaces[babelNeighbours.LinkLocalAddress]; ok {
+				vpn += len(babelNeighbours.Neighbours)
+			}
 		}
 		fields["neighbours.babel"] = babel
 
@@ -94,8 +114,24 @@ func buildNodeStats(node *runtime.Node) (tags models.Tags, fields models.Fields)
 		}
 		fields["neighbours.lldp"] = lldp
 
+		// vpn  wait for babel
+		fields["neighbours.vpn"] = vpn
+
 		// total is the sum of all protocols
-		fields["neighbours.total"] = batadv + lldp
+		fields["neighbours.total"] = batadv + babel + lldp
+	}
+	if procstat := stats.ProcStats; procstat != nil {
+		fields["stat.cpu.user"] = procstat.CPU.User
+		fields["stat.cpu.nice"] = procstat.CPU.Nice
+		fields["stat.cpu.system"] = procstat.CPU.System
+		fields["stat.cpu.idle"] = procstat.CPU.Idle
+		fields["stat.cpu.iowait"] = procstat.CPU.IOWait
+		fields["stat.cpu.irq"] = procstat.CPU.IRQ
+		fields["stat.cpu.softirq"] = procstat.CPU.SoftIRQ
+		fields["stat.intr"] = procstat.Intr
+		fields["stat.ctxt"] = procstat.ContextSwitches
+		fields["stat.softirq"] = procstat.SoftIRQ
+		fields["stat.processes"] = procstat.Processes
 	}
 
 	if t := stats.Traffic.Rx; t != nil {
@@ -130,5 +166,32 @@ func buildNodeStats(node *runtime.Node) (tags models.Tags, fields models.Fields)
 		tags.SetString("frequency"+suffix, strconv.Itoa(int(airtime.Frequency)))
 	}
 
+	conn.addPoint(MeasurementNode, tags, fields, time)
+
+	// Add DHCP statistics
+	if dhcp := stats.DHCP; dhcp != nil {
+		fields := models.Fields{
+			"decline":  dhcp.Decline,
+			"offer":    dhcp.Offer,
+			"ack":      dhcp.Ack,
+			"nak":      dhcp.Nak,
+			"request":  dhcp.Request,
+			"discover": dhcp.Discover,
+			"inform":   dhcp.Inform,
+			"release":  dhcp.Release,
+
+			"leases.allocated": dhcp.LeasesAllocated,
+			"leases.pruned":    dhcp.LeasesPruned,
+		}
+
+		// Tags
+		tags.SetString("nodeid", stats.NodeID)
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			tags.SetString("hostname", nodeinfo.Hostname)
+		}
+
+		conn.addPoint(MeasurementDHCP, tags, fields, time)
+	}
+
 	return
 }

database/influxdb/node_test.go

@@ -3,6 +3,7 @@ package influxdb
 import (
 	"testing"
 
+	"github.com/influxdata/influxdb1-client/v2"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/FreifunkBremen/yanic/data"
@@ -14,8 +15,14 @@ func TestToInflux(t *testing.T) {
 
 	node := &runtime.Node{
 		Statistics: &data.Statistics{
-			NodeID:      "foobar",
+			NodeID:      "deadbeef",
 			LoadAverage: 0.5,
+			ProcStats: &data.ProcStats{
+				CPU: data.ProcStatsCPU{
+					User: 1,
+				},
+				ContextSwitches: 3,
+			},
 			Wireless: data.WirelessStatistics{
 				&data.WirelessAirtime{Frequency: 5500},
 			},
@@ -32,57 +39,191 @@ func TestToInflux(t *testing.T) {
 				MgmtTx:  &data.Traffic{Packets: 2327},
 				MgmtRx:  &data.Traffic{Bytes: 2331},
 			},
-			MeshVPN: &data.MeshVPN{
-				Groups: map[string]*data.MeshVPNPeerGroup{
-					"ffhb": &data.MeshVPNPeerGroup{
-						Peers: map[string]*data.MeshVPNPeerLink{
-							"vpn01": &data.MeshVPNPeerLink{Established: 3},
-							"vpn02": &data.MeshVPNPeerLink{},
-							"trash": nil,
-							"vpn03": &data.MeshVPNPeerLink{Established: 0},
-						},
-					},
-				},
-			},
 		},
-		Nodeinfo: &data.NodeInfo{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "deadbeef",
 			Owner: &data.Owner{
 				Contact: "nobody",
 			},
+			System: data.System{
+				SiteCode:   "ffhb",
+				DomainCode: "city",
+			},
 			Wireless: &data.Wireless{
 				TxPower24: 3,
 				Channel24: 4,
 			},
-		},
-		Neighbours: &data.Neighbours{
-			Batadv: map[string]data.BatadvNeighbours{
-				"a-interface": data.BatadvNeighbours{
-					Neighbours: map[string]data.BatmanLink{
-						"b-neigbourinterface": data.BatmanLink{},
+			Network: data.Network{
+				Mac: "DEADMAC",
+				Mesh: map[string]*data.NetworkInterface{
+					"bat0": {
+						Interfaces: struct {
+							Wireless []string `json:"wireless,omitempty"`
+							Other    []string `json:"other,omitempty"`
+							Tunnel   []string `json:"tunnel,omitempty"`
+						}{
+							Tunnel: []string{"a-interface-mac", "fe80::1"},
+						},
 					},
 				},
 			},
-			LLDP: map[string]data.LLDPNeighbours{},
+			Software: data.Software{
+				Autoupdater: struct {
+					Enabled bool   `json:"enabled,omitempty"`
+					Branch  string `json:"branch,omitempty"`
+				}{
+					Enabled: true,
+					Branch:  "testing",
+				},
+			},
+		},
+		Neighbours: &data.Neighbours{
+			NodeID: "deadbeef",
+			Batadv: map[string]data.BatadvNeighbours{
+				"a-interface-mac": {
+					Neighbours: map[string]data.BatmanLink{
+						"BAFF1E5": {
+							Tq: 204,
+						},
+					},
+				},
+			},
+			Babel: map[string]data.BabelNeighbours{
+				"wg-01": {
+					LinkLocalAddress: "fe80::1",
+					Neighbours: map[string]data.BabelLink{
+						"fe80::2": {
+							Cost: 0,
+						},
+					},
+				},
+			},
+			LLDP: map[string]data.LLDPNeighbours{
+				"b-interface-mac": {},
+			},
 		},
 	}
 
-	tags, fields := buildNodeStats(node)
+	neighbour := &runtime.Node{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "foobar",
+			Network: data.Network{
+				Mac: "BAFF1E5",
+			},
+			Software: data.Software{
+				Autoupdater: struct {
+					Enabled bool   `json:"enabled,omitempty"`
+					Branch  string `json:"branch,omitempty"`
+				}{
+					Enabled: false,
+				},
+			},
+		},
+		Statistics: &data.Statistics{
+			NodeID: "foobar",
+		},
+	}
 
-	assert.Equal("foobar", tags.GetString("nodeid"))
-	assert.Equal("nobody", tags.GetString("owner"))
-	assert.Equal(0.5, fields["load"])
-	assert.Equal(0, fields["neighbours.lldp"])
-	assert.Equal(1, fields["neighbours.batadv"])
-	assert.Equal(1, fields["neighbours.vpn"])
-	assert.Equal(1, fields["neighbours.total"])
+	// do not add a empty statistics of a node
+	droppednode := &runtime.Node{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "notfound",
+			Network: data.Network{
+				Mac: "instats",
+			},
+		},
+		Statistics: &data.Statistics{},
+	}
 
-	assert.Equal(uint32(3), fields["wireless.txpower24"])
-	assert.Equal(uint32(5500), fields["airtime11a.frequency"])
-	assert.Equal("", tags.GetString("frequency5500"))
+	points := testPoints(node, neighbour, droppednode)
+	var fields map[string]interface{}
+	var tags map[string]string
 
-	assert.Equal(int64(1213), fields["traffic.rx.bytes"])
-	assert.Equal(float64(1321), fields["traffic.tx.dropped"])
-	assert.Equal(int64(1322), fields["traffic.forward.bytes"])
-	assert.Equal(int64(2331), fields["traffic.mgmt_rx.bytes"])
-	assert.Equal(float64(2327), fields["traffic.mgmt_tx.packets"])
+	assert.Len(points, 3)
+
+	// first point contains the neighbour
+	sPoint := points[0]
+	tags = sPoint.Tags()
+	fields, _ = sPoint.Fields()
+
+	assert.EqualValues("deadbeef", tags["nodeid"])
+	assert.EqualValues("nobody", tags["owner"])
+	assert.EqualValues("testing", tags["autoupdater"])
+	assert.EqualValues("ffhb", tags["site"])
+	assert.EqualValues("city", tags["domain"])
+	assert.EqualValues(0.5, fields["load"])
+	assert.EqualValues(0, fields["neighbours.lldp"])
+	assert.EqualValues(1, fields["neighbours.babel"])
+	assert.EqualValues(1, fields["neighbours.batadv"])
+	assert.EqualValues(2, fields["neighbours.vpn"])
+	assert.EqualValues(2, fields["neighbours.total"])
+
+	assert.EqualValues(uint32(3), fields["wireless.txpower24"])
+	assert.EqualValues(uint32(5500), fields["airtime11a.frequency"])
+	assert.EqualValues("", tags["frequency5500"])
+
+	assert.EqualValues(int64(1213), fields["traffic.rx.bytes"])
+	assert.EqualValues(float64(1321), fields["traffic.tx.dropped"])
+	assert.EqualValues(int64(1322), fields["traffic.forward.bytes"])
+	assert.EqualValues(int64(2331), fields["traffic.mgmt_rx.bytes"])
+	assert.EqualValues(float64(2327), fields["traffic.mgmt_tx.packets"])
+
+	// second point contains the link
+	nPoint := points[1]
+	tags = nPoint.Tags()
+	fields, _ = nPoint.Fields()
+	assert.EqualValues("link", nPoint.Name())
+	assert.EqualValues(map[string]string{
+		"source.id":   "deadbeef",
+		"source.addr": "a-interface-mac",
+		"target.id":   "foobar",
+		"target.addr": "BAFF1E5",
+	}, tags)
+	assert.EqualValues(80, fields["tq"])
+
+	// third point contains the neighbour
+	nPoint = points[2]
+	tags = nPoint.Tags()
+	assert.EqualValues("disabled", tags["autoupdater"])
+}
+
+// Processes data and returns the InfluxDB points
+func testPoints(nodes ...*runtime.Node) (points []*client.Point) {
+	// Create dummy client
+	influxClient, err := client.NewHTTPClient(client.HTTPConfig{Addr: "http://127.0.0.1"})
+	if err != nil {
+		panic(err)
+	}
+
+	nodesList := runtime.NewNodes(&runtime.NodesConfig{})
+
+	// Create dummy connection
+	conn := &Connection{
+		points: make(chan *client.Point),
+		client: influxClient,
+	}
+
+	for _, node := range nodes {
+		nodesList.AddNode(node)
+	}
+
+	// Process data
+	go func() {
+		for _, node := range nodes {
+			conn.InsertNode(node)
+			if node.Neighbours != nil {
+				for _, link := range nodesList.NodeLinks(node) {
+					conn.InsertLink(&link, node.Lastseen.GetTime())
+				}
+			}
+		}
+		conn.Close()
+	}()
+
+	// Read points
+	for point := range conn.points {
+		points = append(points, point)
+	}
+
+	return
 }

database/internal.go

@@ -1,40 +0,0 @@
-package database
-
-import (
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-var quit chan struct{}
-
-// Start workers of database
-// WARNING: Do not override this function
-//  you should use New()
-func Start(conn Connection, config *runtime.Config) {
-	quit = make(chan struct{})
-	go deleteWorker(conn, config.Database.DeleteInterval.Duration, config.Database.DeleteAfter.Duration)
-}
-
-func Close(conn Connection) {
-	if quit != nil {
-		close(quit)
-	}
-	if conn != nil {
-		conn.Close()
-	}
-}
-
-// prunes node-specific data periodically
-func deleteWorker(conn Connection, deleteInterval time.Duration, deleteAfter time.Duration) {
-	ticker := time.NewTicker(deleteInterval)
-	for {
-		select {
-		case <-ticker.C:
-			conn.PruneNodes(deleteAfter)
-		case <-quit:
-			ticker.Stop()
-			return
-		}
-	}
-}

database/logging/file.go

@@ -7,7 +7,6 @@ package logging
  */
 import (
 	"fmt"
-	"log"
 	"os"
 	"time"
 
@@ -23,9 +22,6 @@ type Connection struct {
 
 type Config map[string]interface{}
 
-func (c Config) Enable() bool {
-	return c["enable"].(bool)
-}
 func (c Config) Path() string {
 	return c["path"].(string)
 }
@@ -34,12 +30,9 @@ func init() {
 	database.RegisterAdapter("logging", Connect)
 }
 
-func Connect(configuration interface{}) (database.Connection, error) {
+func Connect(configuration map[string]interface{}) (database.Connection, error) {
 	var config Config
-	config = configuration.(map[string]interface{})
-	if !config.Enable() {
-		return nil, nil
-	}
+	config = configuration
 
 	file, err := os.OpenFile(config.Path(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
 	if err != nil {
@@ -52,8 +45,12 @@ func (conn *Connection) InsertNode(node *runtime.Node) {
 	conn.log("InsertNode: [", node.Statistics.NodeID, "] clients: ", node.Statistics.Clients.Total)
 }
 
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
-	conn.log("InsertGlobals: [", time.String(), "] nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
+func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
+	conn.log("InsertLink: ", link)
+}
+
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
+	conn.log("InsertGlobals: [", time.String(), "] site: ", site, " domain: ", domain, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
 }
 
 func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
@@ -66,6 +63,6 @@ func (conn *Connection) Close() {
 }
 
 func (conn *Connection) log(v ...interface{}) {
-	log.Println(v)
+	fmt.Println(v...)
 	conn.file.WriteString(fmt.Sprintln("[", time.Now().String(), "]", v))
 }

database/logging/file_test.go

@@ -0,0 +1,61 @@
+package logging
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStart(t *testing.T) {
+	assert := assert.New(t)
+
+	conn, err := Connect(map[string]interface{}{
+		"path": "/dev/notexists/file",
+	})
+	assert.Nil(conn)
+	assert.Error(err)
+
+	path := "/tmp/testlogfile"
+
+	conn, err = Connect(map[string]interface{}{
+		"path": path,
+	})
+	assert.NoError(err)
+
+	dat, _ := ioutil.ReadFile(path)
+	assert.NotContains(string(dat), "InsertNode")
+
+	conn.InsertNode(&runtime.Node{
+		Statistics: &data.Statistics{},
+	})
+
+	dat, _ = ioutil.ReadFile(path)
+	assert.Contains(string(dat), "InsertNode")
+
+	assert.NotContains(string(dat), "InsertLink")
+	conn.InsertLink(&runtime.Link{}, time.Now())
+	dat, _ = ioutil.ReadFile(path)
+	assert.Contains(string(dat), "InsertLink")
+
+	assert.NotContains(string(dat), "InsertGlobals")
+	conn.InsertGlobals(&runtime.GlobalStats{}, time.Now(), runtime.GLOBAL_SITE, runtime.GLOBAL_DOMAIN)
+	dat, _ = ioutil.ReadFile(path)
+	assert.Contains(string(dat), "InsertGlobals")
+
+	assert.NotContains(string(dat), "PruneNodes")
+	conn.PruneNodes(time.Second)
+	dat, _ = ioutil.ReadFile(path)
+	assert.Contains(string(dat), "PruneNodes")
+
+	assert.NotContains(string(dat), "Close")
+	conn.Close()
+	dat, _ = ioutil.ReadFile(path)
+	assert.Contains(string(dat), "Close")
+
+	os.Remove(path)
+}

database/respondd/main.go

@@ -0,0 +1,97 @@
+package respondd
+
+/**
+ * This database type is for injecting into another yanic instance.
+ */
+import (
+	"bufio"
+	"compress/flate"
+	"encoding/json"
+	"net"
+	"time"
+
+	"github.com/bdlm/log"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/database"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Connection struct {
+	database.Connection
+	config Config
+	conn   net.Conn
+}
+
+type Config map[string]interface{}
+
+func (c Config) Type() string {
+	return c["type"].(string)
+}
+
+func (c Config) Address() string {
+	return c["address"].(string)
+}
+
+func init() {
+	database.RegisterAdapter("respondd", Connect)
+}
+
+func Connect(configuration map[string]interface{}) (database.Connection, error) {
+	var config Config
+	config = configuration
+
+	conn, err := net.Dial(config.Type(), config.Address())
+	if err != nil {
+		return nil, err
+	}
+
+	return &Connection{conn: conn, config: config}, nil
+}
+
+func (conn *Connection) InsertNode(node *runtime.Node) {
+	res := &data.ResponseData{
+		Nodeinfo:   node.Nodeinfo,
+		Statistics: node.Statistics,
+		Neighbours: node.Neighbours,
+	}
+
+	writer := bufio.NewWriterSize(conn.conn, 8192)
+
+	flater, err := flate.NewWriter(writer, flate.BestCompression)
+	if err != nil {
+		log.Errorf("[database-yanic] could not create flater: %s", err)
+		return
+	}
+	defer flater.Close()
+	err = json.NewEncoder(flater).Encode(res)
+	if err != nil {
+		nodeid := "unknown"
+		if node.Nodeinfo != nil && node.Nodeinfo.NodeID != "" {
+			nodeid = node.Nodeinfo.NodeID
+		}
+		log.WithField("node_id", nodeid).Errorf("[database-yanic] could not encode node: %s", err)
+		return
+	}
+	err = flater.Flush()
+	if err != nil {
+		log.Errorf("[database-yanic] could not compress: %s", err)
+	}
+	err = writer.Flush()
+	if err != nil {
+		log.Errorf("[database-yanic] could not send: %s", err)
+	}
+}
+
+func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
+}
+
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
+}
+
+func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
+}
+
+func (conn *Connection) Close() {
+	conn.conn.Close()
+}

database/respondd/main_test.go

@@ -0,0 +1,49 @@
+package respondd
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStart(t *testing.T) {
+	assert := assert.New(t)
+
+	conn, err := Connect(map[string]interface{}{
+		"type":    "udp6",
+		"address": "fasfs",
+	})
+	assert.Nil(conn)
+	assert.Error(err)
+
+	conn, err = Connect(map[string]interface{}{
+		"type":    "udp",
+		"address": "localhost:11001",
+	})
+	assert.NoError(err)
+
+	conn.InsertNode(&runtime.Node{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID:   "73deadbeaf13",
+			Hostname: "inject-test",
+			Network: data.Network{
+				Mac:       "73:de:ad:be:af:13",
+				Addresses: []string{"a", "b"},
+			},
+		},
+		Statistics: &data.Statistics{
+			NodeID: "73deadbeaf13",
+			Clients: data.Clients{
+				Total:  1000,
+				Wifi:   500,
+				Wifi24: 100,
+				Wifi5:  300,
+			},
+		},
+	})
+
+	conn.Close()
+
+}

database/socket/client/dial.go

@@ -1,86 +0,0 @@
-package yanic
-
-import (
-	"encoding/json"
-	"log"
-	"net"
-
-	"github.com/FreifunkBremen/yanic/database/socket"
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-type Dialer struct {
-	conn              net.Conn
-	queue             chan socket.Message
-	quit              chan struct{}
-	NodeHandler       func(*runtime.Node)
-	GlobalsHandler    func(*runtime.GlobalStats)
-	PruneNodesHandler func()
-}
-
-func Dial(ctype, addr string) *Dialer {
-	conn, err := net.Dial(ctype, addr)
-	if err != nil {
-		log.Panicf("yanic dial to %s:%s failed", ctype, addr)
-	}
-	dialer := &Dialer{
-		conn:  conn,
-		queue: make(chan socket.Message),
-		quit:  make(chan struct{}),
-	}
-
-	return dialer
-}
-
-func (d *Dialer) Start() {
-	go d.reciever()
-	d.parser()
-}
-func (d *Dialer) Close() {
-	d.conn.Close()
-	close(d.quit)
-}
-
-func (d *Dialer) reciever() {
-	decoder := json.NewDecoder(d.conn)
-	var msg socket.Message
-
-	for {
-		select {
-		case <-d.quit:
-			close(d.queue)
-			return
-		default:
-			decoder.Decode(&msg)
-			d.queue <- msg
-		}
-	}
-}
-
-func (d *Dialer) parser() {
-	for msg := range d.queue {
-		switch msg.Event {
-		case socket.MessageEventInsertNode:
-			if d.NodeHandler != nil {
-				var node runtime.Node
-
-				obj, _ := json.Marshal(msg.Body)
-				json.Unmarshal(obj, &node)
-				d.NodeHandler(&node)
-			}
-		case socket.MessageEventInsertGlobals:
-			if d.GlobalsHandler != nil {
-				var globals runtime.GlobalStats
-
-				obj, _ := json.Marshal(msg.Body)
-				json.Unmarshal(obj, &globals)
-
-				d.GlobalsHandler(&globals)
-			}
-		case socket.MessageEventPruneNodes:
-			if d.PruneNodesHandler != nil {
-				d.PruneNodesHandler()
-			}
-		}
-	}
-}

database/socket/client/dial_test.go

@@ -1,69 +0,0 @@
-package yanic
-
-import (
-	"testing"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/database/socket"
-	"github.com/FreifunkBremen/yanic/runtime"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestConnectError(t *testing.T) {
-	assert := assert.New(t)
-	assert.Panics(func() {
-		Dial("tcp6", "[::]:30303")
-	}, "could connect")
-}
-
-func TestRecieveMessages(t *testing.T) {
-	assert := assert.New(t)
-	server, err := socket.Connect(map[string]interface{}{
-		"enable":  true,
-		"type":    "tcp6",
-		"address": "[::]:1337",
-	})
-	assert.NoError(err)
-
-	d := Dial("tcp6", "[::]:1337")
-	assert.NotNil(d)
-	go d.Start()
-	time.Sleep(5 * time.Millisecond)
-
-	runned := false
-	d.NodeHandler = func(node *runtime.Node) {
-		runned = true
-	}
-	server.InsertNode(nil)
-	time.Sleep(5 * time.Millisecond)
-	assert.True(runned, "node not inserted")
-
-	runned = false
-	d.GlobalsHandler = func(stats *runtime.GlobalStats) {
-		runned = true
-	}
-	server.InsertGlobals(nil, time.Now())
-	time.Sleep(5 * time.Millisecond)
-	assert.True(runned, "global stats not inserted")
-
-	runned = false
-	d.PruneNodesHandler = func() {
-		runned = true
-	}
-	server.PruneNodes(time.Hour * 24 * 7)
-	time.Sleep(5 * time.Millisecond)
-	assert.True(runned, "node not pruned")
-
-	d.Close()
-
-	time.Sleep(5 * time.Millisecond)
-	runned = false
-	d.PruneNodesHandler = func() {
-		runned = true
-	}
-	server.PruneNodes(time.Hour * 24 * 7)
-	time.Sleep(5 * time.Millisecond)
-	assert.False(runned, "message recieve")
-
-	server.Close()
-}

database/socket/database.go

@@ -1,68 +0,0 @@
-package socket
-
-/*
- * This socket database is to run another service
- * (without flooding the network with respondd packages)
- * e.g. https://github.com/FreifunkBremen/freifunkmanager
- */
-
-import (
-	"log"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-type Connection struct {
-	database.Connection
-	listener  net.Listener
-	clients   map[net.Addr]net.Conn
-	clientMux sync.Mutex
-}
-
-func init() {
-	database.RegisterAdapter("socket", Connect)
-}
-
-func Connect(configuration interface{}) (database.Connection, error) {
-	config := configuration.(map[string]interface{})
-
-	if !config["enable"].(bool) {
-		return nil, nil
-	}
-
-	ln, err := net.Listen(config["type"].(string), config["address"].(string))
-	if err != nil {
-		return nil, err
-	}
-	conn := &Connection{listener: ln, clients: make(map[net.Addr]net.Conn)}
-	go conn.handleSocketConnection(ln)
-
-	log.Println("[socket-database] listen on: ", ln.Addr())
-
-	return conn, nil
-}
-
-func (conn *Connection) InsertNode(node *runtime.Node) {
-	conn.sendJSON(Message{Event: MessageEventInsertNode, Body: node})
-}
-
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
-	conn.sendJSON(Message{Event: MessageEventInsertGlobals, Body: stats})
-}
-
-func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
-	conn.sendJSON(Message{Event: MessageEventPruneNodes})
-}
-
-func (conn *Connection) Close() {
-	conn.clientMux.Lock()
-	for _, c := range conn.clients {
-		c.Close()
-	}
-	conn.clientMux.Unlock()
-	conn.listener.Close()
-}

database/socket/database_test.go

@@ -1,80 +0,0 @@
-package socket
-
-import (
-	"encoding/json"
-	"net"
-	"testing"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestStartup(t *testing.T) {
-	assert := assert.New(t)
-
-	config := make(map[string]interface{})
-
-	config["enable"] = false
-	conn, err := Connect(config)
-	assert.Nil(conn)
-
-	config["enable"] = true
-	config["type"] = ""
-	config["address"] = ""
-	conn, err = Connect(config)
-	assert.Error(err, "connection should not work")
-	assert.Nil(conn)
-
-	config["type"] = "tcp6"
-	config["address"] = "[::]:1337"
-
-	conn, err = Connect(config)
-	assert.NoError(err, "connection should work")
-	assert.NotNil(conn)
-
-	conn.Close()
-}
-
-func TestClient(t *testing.T) {
-	assert := assert.New(t)
-
-	config := make(map[string]interface{})
-
-	config["enable"] = true
-	config["type"] = "tcp6"
-	config["address"] = "[::]:1337"
-
-	conn, err := Connect(config)
-	assert.NoError(err, "connection should work")
-	assert.NotNil(conn)
-
-	client, err := net.Dial("tcp6", "[::]:1337")
-	assert.NoError(err, "connection should work")
-	assert.NotNil(client)
-	time.Sleep(time.Duration(3) * time.Microsecond)
-
-	decoder := json.NewDecoder(client)
-	var msg Message
-
-	conn.InsertNode(&runtime.Node{})
-	decoder.Decode(&msg)
-	assert.Equal("insert_node", msg.Event)
-
-	conn.InsertGlobals(&runtime.GlobalStats{}, time.Now())
-	decoder.Decode(&msg)
-	assert.Equal("insert_globals", msg.Event)
-
-	conn.PruneNodes(time.Hour * 24 * 7)
-	decoder.Decode(&msg)
-	assert.Equal("prune_nodes", msg.Event)
-	time.Sleep(time.Duration(3) * time.Microsecond)
-
-	// to reach in sendJSON removing of disconnection
-	conn.Close()
-
-	conn.InsertNode(&runtime.Node{})
-	err = decoder.Decode(&msg)
-	assert.Error(err)
-
-}

database/socket/internal.go

@@ -1,35 +0,0 @@
-package socket
-
-import (
-	"encoding/json"
-	"log"
-	"net"
-)
-
-func (conn *Connection) handleSocketConnection(ln net.Listener) {
-	for {
-		c, err := ln.Accept()
-		if err != nil {
-			log.Println("[socket-database] error during connection of a client", err)
-			continue
-		}
-		conn.clientMux.Lock()
-		conn.clients[c.RemoteAddr()] = c
-		conn.clientMux.Unlock()
-	}
-}
-
-func (conn *Connection) sendJSON(msg Message) {
-	conn.clientMux.Lock()
-	for addr, c := range conn.clients {
-		d := json.NewEncoder(c)
-
-		err := d.Encode(&msg)
-		if err != nil {
-			log.Println("[socket-database] client has not recieve event:", err)
-			c.Close()
-			delete(conn.clients, addr)
-		}
-	}
-	conn.clientMux.Unlock()
-}

database/socket/message.go

@@ -1,12 +0,0 @@
-package socket
-
-type Message struct {
-	Event string      `json:"event"`
-	Body  interface{} `json:"body,omitempty"`
-}
-
-const (
-	MessageEventInsertNode    = "insert_node"
-	MessageEventInsertGlobals = "insert_globals"
-	MessageEventPruneNodes    = "prune_nodes"
-)

database/yanic/main.go

@@ -1,81 +0,0 @@
-package yanic
-
-import (
-	"bufio"
-	"bytes"
-	"compress/flate"
-	"encoding/json"
-	"log"
-	"net"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/data"
-	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-type Connection struct {
-	database.Connection
-	config Config
-	conn   *net.UDPConn
-}
-
-type Config map[string]interface{}
-
-func (c Config) Enable() bool {
-	return c["enable"].(bool)
-}
-func (c Config) Address() string {
-	return c["address"].(string)
-}
-
-func init() {
-	database.RegisterAdapter("yanic", Connect)
-}
-
-func Connect(configuration interface{}) (database.Connection, error) {
-	var config Config
-	config = configuration.(map[string]interface{})
-	if !config.Enable() {
-		return nil, nil
-	}
-	udpAddr, err := net.ResolveUDPAddr("udp", config.Address())
-	if err != nil {
-		log.Panicf("Invalid yanic address: %s", err)
-	}
-
-	conn, err := net.DialUDP("udp", nil, udpAddr)
-	if err != nil {
-		log.Panicf("Unable to dial yanic: %s", err)
-	}
-	return &Connection{config: config, conn: conn}, nil
-}
-
-func (conn *Connection) InsertNode(node *runtime.Node) {
-	buf := bytes.Buffer{}
-	writer := bufio.NewWriter(&buf)
-	deflater, err := flate.NewWriter(writer, flate.DefaultCompression)
-
-	err = json.NewEncoder(deflater).Encode(&data.ResponseData{
-		Statistics: node.Statistics,
-		NodeInfo:   node.Nodeinfo,
-	})
-	if err != nil {
-		panic(err)
-	}
-	deflater.Close()
-	writer.Flush()
-
-	conn.conn.Write(buf.Bytes())
-}
-
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
-
-}
-
-func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
-}
-
-func (conn *Connection) Close() {
-	conn.conn.Close()
-}

docs/dev_database.md

@@ -0,0 +1,49 @@
+# Add new database type
+
+Write a new package to implement the interface [database.Connection:](https://github.com/FreifunkBremen/yanic/blob/master/database/database.go)
+
+```go
+type Connection interface {
+	InsertNode(node *runtime.Node)
+
+	InsertLink(*runtime.Link, time.Time)
+
+	InsertGlobals(*runtime.GlobalStats, time.Time, string)
+
+	PruneNodes(deleteAfter time.Duration)
+
+	Close()
+}
+```
+
+**InsertNode** is stores statistics per node
+
+**InsertLink** is stores statistics per link
+
+**InsertGlobals** is stores global statistics (by `site_code`, and "global" like in `runtime.GLOBAL_SITE` overall sites).
+
+**PruneNodes** is prunes historical per-node data
+
+**Close** is called during shutdown of Yanic.
+
+
+
+For startup, you need to bind your database type by calling `database.RegisterAdapter("typeofdatabase",ConnectFunction)`
+
+it should be in the `func init() {}` of your package.
+
+
+
+The _typeofdatabase_ is used as mapping in the configuration `[[database.connection.typeofdatabase]]` the `map[string]interface{}` of the content are parsed to the _ConnectFunction_ and on of your implemented `Connection` or a `error` is needed as result.
+
+
+
+Short: the function signature of _ConnectFunction_ should be `func Connect(configuration interface{}) (Connection, error)`
+
+
+
+At last add you import string to compile the your database as well in this [all](https://github.com/FreifunkBremen/yanic/blob/master/database/all/main.go) package.
+
+
+
+TIP: take a look in the easy database type [logging](https://github.com/FreifunkBremen/yanic/blob/master/database/logging/file.go).

docs/dev_output.md

@@ -0,0 +1,30 @@
+# Add new output type
+
+Write a new package to implement the interface [output.Output:](https://github.com/FreifunkBremen/yanic/blob/master/output/output.go)
+
+```go
+type Output interface {
+	Save(nodes *runtime.Nodes)
+}
+```
+
+**Save** a pre-filtered state of the Nodes
+
+
+
+For startup, you need to bind your output type by calling
+ `output.RegisterAdapter("typeofoutput",Register)`
+
+it should be in the `func init() {}` of your package.
+
+
+
+The _typeofoutput_ is used as mapping in the configuration `[[nodes.output.typeofoutput]]` the `map[string]interface{}` of the content are parsed to the _Register_ and on of your implemented `Output` or a `error` is needed as result.
+
+
+
+Short: the function signature of _Register_ should be `func Register(configuration map[string]interface{}) (Output, error)`
+
+
+
+At last add you import string to compile the your database as well in this [all](https://github.com/FreifunkBremen/yanic/blob/master/output/all/main.go) package.

docs/docs_configuration.md

@@ -0,0 +1,782 @@
+# Configuration
+
+Here you would find a long description, maybe the description in [example file](https://github.com/FreifunkBremen/yanic/blob/master/config_example.toml) are enough for you.
+
+The config file for Yanic written in "Tom's Obvious, Minimal Language." [syntax](https://github.com/toml-lang/toml).
+(if you need somethink multiple times, checkout out the [[array of table]] section)
+
+## [respondd]
+{% method %}
+Group for configuration of respondd request.
+{% sample lang="toml" %}
+```toml
+[respondd]
+enable           = true
+# synchronize    = "1m"
+collect_interval = "1m"
+
+#[respondd.sites.example]
+#domains            = ["city"]
+
+[[respondd.interfaces]]
+ifname             = "br-ffhb"
+#ip_address        = "fe80::..."
+#send_no_request   = false
+#multicast_address = "ff02::2:1001"
+#port              = 10001
+```
+{% endmethod %}
+
+
+### enable
+{% method %}
+Enable request and collection of data per respondd requests
+{% sample lang="toml" %}
+```toml
+enable = true
+```
+{% endmethod %}
+
+
+### synchronize
+{% method %}
+Delay startup until a multiple of the period since zero time
+{% sample lang="toml" %}
+```toml
+synchronize      = "1m"
+```
+{% endmethod %}
+
+
+### collect_interval
+{% method %}
+How often send request per respondd.
+
+It will send UDP packets with multicast address `ff05::2:1001` and port `1001`.
+If a node does not answer after the half time, it will request with the last know address under the port `1001`.
+{% sample lang="toml" %}
+```toml
+collect_interval = "1m"
+```
+{% endmethod %}
+
+
+### [respondd.sites.example]
+{% method %}
+Tables of sites to save stats for (not exists for global only).
+Here is the site _ffhb_.
+{% sample lang="toml" %}
+```toml
+[respondd.sites.ffhb]
+domains            = ["city"]
+```
+{% endmethod %}
+
+#### domains
+{% method %}
+list of domains on this site to save stats for (empty for global only)
+{% sample lang="toml" %}
+```toml
+domains            = ["city"]
+```
+{% endmethod %}
+
+
+### [[respondd.interfaces]]
+{% method %}
+Interface that has an ip address in your mesh network.
+It is possible to have multiple interfaces, just add this group again with new parameters (see toml [[array of table]]).
+{% sample lang="toml" %}
+```toml
+[[respondd.interfaces]]
+ifname             = "br-ffhb"
+#ip_address        = "fe80::..."
+#send_no_request   = false
+#multicast_address = "ff02::2:1001"
+#port              = 10001
+```
+{% endmethod %}
+
+### ifname
+{% method %}
+name of interface on which this collector is running.
+{% sample lang="toml" %}
+```toml
+ifname              = "br-ffhb"
+```
+{% endmethod %}
+
+### ip_address
+{% method %}
+ip address is the own address which is used for sending.
+If not set or set with empty string it will take an address of ifname.
+(It prefers the link local address, so at babel mesh-network it should be configurated)
+{% sample lang="toml" %}
+```toml
+ip_address          = "fe80::..."
+```
+{% endmethod %}
+
+### send_no_request
+{% method %}
+Disable sending multicast respondd request.
+For receiving only respondd packages e.g. database respondd.
+{% sample lang="toml" %}
+```toml
+send_no_request     = true
+```
+{% endmethod %}
+
+### multicast_address
+{% method %}
+Multicast address to destination of respondd.
+If not set or set with empty string it will take the batman default multicast address `ff05::2:1001`
+(Needed to set for legacy `ff02::2:1001`)
+{% sample lang="toml" %}
+```toml
+multicast_address    = "ff02::2:1001"
+```
+{% endmethod %}
+
+### port
+{% method %}
+Define a port to listen and send the respondd packages.
+If not set or set to 0 the kernel will use a random free port at its own.
+{% sample lang="toml" %}
+```toml
+port              = 10001
+```
+{% endmethod %}
+
+
+## [webserver]
+{% method %}
+Yanic has a little build-in webserver, which statically serves a directory.
+This is useful for testing purposes or for a little standalone installation.
+{% sample lang="toml" %}
+```toml
+[webserver]
+enable  = false
+bind    = "127.0.0.1:8080"
+webroot = "/var/www/html/meshviewer"
+```
+{% endmethod %}
+
+
+### enable
+{% method %}
+Enable to start the built-in webserver of Yanic
+{% sample lang="toml" %}
+```toml
+enable  = false
+```
+{% endmethod %}
+
+
+### bind
+{% method %}
+On which ip address and port listen the webserver
+{% sample lang="toml" %}
+```toml
+bind    = "127.0.0.1:8080"
+```
+{% endmethod %}
+
+
+### webroot
+{% method %}
+The path to a folder, which files are published on this webserver.
+{% sample lang="toml" %}
+```toml
+webroot = "/var/www/html/meshviewer"
+```
+{% endmethod %}
+
+
+
+## [nodes]
+{% method %}
+{% sample lang="toml" %}
+```toml
+[nodes]
+state_path     = "/var/lib/yanic/state.json"
+prune_after    = "7d"
+save_interval  = "5s"
+offline_after  = "10m"
+```
+{% endmethod %}
+
+
+### state_path
+{% method %}
+A json file to cache all data collected directly from respondd.
+{% sample lang="toml" %}
+```toml
+state_path     = "/var/lib/yanic/state.json"
+```
+{% endmethod %}
+
+
+### prune_after
+{% method %}
+Prune data in RAM, cache-file and output json files (i.e. nodes.json) that were inactive for longer than.
+{% sample lang="toml" %}
+```toml
+prune_after = "7d"
+```
+{% endmethod %}
+
+
+### save_interval
+{% method %}
+Export nodes and graph periodically.
+{% sample lang="toml" %}
+```toml
+save_interval = "5s"
+```
+{% endmethod %}
+
+
+### offline_after
+{% method %}
+Set node to offline if not seen within this period.
+{% sample lang="toml" %}
+```toml
+offline_after = "10m"
+```
+{% endmethod %}
+
+
+## [[nodes.output.example]]
+{% method %}
+This example block shows all option which is useable for every following output type.
+Every output type has his own configuration under `nodes.output`.
+It is possible to have multiple output for one type of output, just add this group again with new parameters (see toml [[array of table]]).
+{% sample lang="toml" %}
+```toml
+[[nodes.output.example]]
+enable = true
+[nodes.output.example.filter]
+no_owner  = true
+blacklist = ["00112233445566", "1337f0badead"]
+sites = ["ffhb"]
+domain_as_site = true
+domain_append_site = true
+has_location = true
+[nodes.output.example.filter.in_area]
+latitude_min  = 34.30
+latitude_max  = 71.85
+longitude_min = -24.96
+longitude_max = 39.72
+```
+{% endmethod %}
+
+### enable
+{% method %}
+Each output format has its own config block and needs to be enabled by adding:
+{% sample lang="toml" %}
+```toml
+enable = true
+```
+{% endmethod %}
+
+### [nodes.output.example.filter]
+{% method %}
+For each output format there can be set different filters
+{% sample lang="toml" %}
+```toml
+[nodes.output.example.filter]
+no_owner  = true
+blacklist = ["00112233445566", "1337f0badead"]
+sites = ["ffhb"]
+has_location = true
+[nodes.output.example.filter.in_area]
+latitude_min  = 34.30
+latitude_max  = 71.85
+longitude_min = -24.96
+longitude_max = 39.72
+```
+{% endmethod %}
+
+
+### no_owner
+{% method %}
+Set to false, if you want the json files to contain the owner information
+
+
+**WARNING: if it is not set, it will publish contact information of other persons.**
+
+{% sample lang="toml" %}
+```toml
+no_owner = true
+```
+{% endmethod %}
+
+
+### blacklist
+{% method %}
+List of nodeids of nodes that should be filtered out, so they won't appear in output
+{% sample lang="toml" %}
+```toml
+blacklist = ["00112233445566", "1337f0badead"]
+```
+{% endmethod %}
+
+
+### sites
+{% method %}
+List of site_codes of nodes that should be included in output
+{% sample lang="toml" %}
+```toml
+sites = ["ffhb"]
+```
+{% endmethod %}
+
+### domain_as_site
+{% method %}
+Replace the `site_code` with the `domain_code` in this output.
+e.g. `site_code='ffhb',domain_code='city'` becomes `site_code='city', domain_code=''`
+{% sample lang="toml" %}
+```toml
+domain_as_site = true
+```
+{% endmethod %}
+
+### domain_append_site
+{% method %}
+Append on the `site_code` the `domain_code` with a `.` in this output.
+e.g. `site_code='ffhb',domain_code='city'` becomes `site_code='ffhb.city', domain_code=''`
+{% sample lang="toml" %}
+```toml
+domain_append_site = true
+```
+{% endmethod %}
+
+### sites
+{% method %}
+List of site_codes of nodes that should be included in output
+{% sample lang="toml" %}
+```toml
+sites = ["ffhb"]
+```
+{% endmethod %}
+
+
+### has_location
+{% method %}
+set has_location to true if you want to include only nodes that have geo-coordinates set
+(setting this to false has no sensible effect, unless you'd want to hide nodes that have coordinates)
+{% sample lang="toml" %}
+```toml
+has_location = true
+```
+{% endmethod %}
+
+
+### [nodes.output.example.filter.in_area]
+{% method %}
+nodes outside this area are not shown on the map but are still listed as a node without coordinates
+{% sample lang="toml" %}
+```toml
+latitude_min = 34.30
+latitude_max = 71.85
+longitude_min = -24.96
+longitude_max = 39.72
+```
+{% endmethod %}
+
+
+
+## [[nodes.output.geojson]]
+{% method %}
+The geojson output produces a geojson file which contains the location data of all monitored nodes to be used to visualize the location of the nodes.
+It is optimized to be used with [UMap](https://github.com/umap-project/umap) but should work with other tools as well.
+
+Here is a public demo provided by Freifunk Muenchen: http://u.osmfr.org/m/328494/
+{% sample lang="toml" %}
+```toml
+[[nodes.output.geojson]]
+enable   = true
+path = "/var/www/html/meshviewer/data/nodes.geojson"
+```
+{% endmethod %}
+
+
+### path
+{% method %}
+The path, where to store nodes.geojson
+{% sample lang="toml" %}
+```toml
+path     = "/var/www/html/meshviewer/data/nodes.geojson"
+```
+{% endmethod %}
+
+
+
+## [[nodes.output.meshviewer-ffrgb]]
+{% method %}
+The new json file format for the [meshviewer](https://github.com/ffrgb/meshviewer) developed in Regensburg.
+
+{% sample lang="toml" %}
+```toml
+[[nodes.output.meshviewer-ffrgb]]
+enable   = true
+path     = "/var/www/html/meshviewer/data/meshviewer.json"
+#[nodes.output.meshviewer-ffrgb.filter]
+#no_owner = false
+#blacklist = ["00112233445566", "1337f0badead"]
+#has_location = true
+
+#[nodes.output.meshviewer-ffrgb.filter.in_area]
+#latitude_min = 34.30
+#latitude_max = 71.85
+#longitude_min = -24.96
+#longitude_max = 39.72
+```
+{% endmethod %}
+
+
+### path
+{% method %}
+The path, where to store meshviewer.json
+{% sample lang="toml" %}
+```toml
+path     = "/var/www/html/meshviewer/data/meshviewer.json"
+```
+{% endmethod %}
+
+
+
+## [[nodes.output.meshviewer]]
+{% method %}
+{% sample lang="toml" %}
+```toml
+[[nodes.output.meshviewer]]
+enable         = false
+version        = 2
+nodes_path     = "/var/www/html/meshviewer/data/nodes.json"
+graph_path     = "/var/www/html/meshviewer/data/graph.json"
+```
+{% endmethod %}
+
+
+### version
+{% method %}
+The structure version of the output which should be generated (i.e. nodes.json)
+* version 1 is accepted by the legacy meshviewer (which is the master branch)
+* https://github.com/ffnord/meshviewer/tree/master
+* version 2 is accepted by the new version of meshviewer (which are in legacy develop branch or newer)
+* https://github.com/ffnord/meshviewer/tree/dev
+* https://github.com/ffrgb/meshviewer/tree/develop
+
+{% sample lang="toml" %}
+```toml
+version = 2
+```
+{% endmethod %}
+
+
+### nodes_path
+{% method %}
+The path, where to store nodes.json (supports version 1 and two, see `nodes_version`)
+{% sample lang="toml" %}
+```toml
+nodes_path = "/var/www/html/meshviewer/data/nodes.json"
+```
+{% endmethod %}
+
+
+### graph_path
+{% method %}
+The path, where to store graph.json (only version 1)
+{% sample lang="toml" %}
+```toml
+graph_path = "/var/www/html/meshviewer/data/graph.json"
+```
+{% endmethod %}
+
+
+
+## [[nodes.output.nodelist]]
+{% method %}
+The nodelist output is a minimal output with current state of collected data.
+Should be preferred to use it on the [ffapi](https://freifunk.net/api-generator/) for the [freifunk-karte.de](https://freifunk-karte.de)
+{% sample lang="toml" %}
+```toml
+[[nodes.output.nodelist]]
+enable   = false
+path     = "/var/www/html/meshviewer/data/nodelist.json"
+#[nodes.output.nodelist.filter]
+#no_owner = false
+```
+{% endmethod %}
+
+
+### path
+{% method %}
+The path, where to store nodelist.json
+{% sample lang="toml" %}
+```toml
+path     = "/var/www/html/meshviewer/data/nodelist.json"
+```
+{% endmethod %}
+
+## [[nodes.output.raw]]
+{% method %}
+This output takes the respondd response as sent by the node and includes it in a JSON document.
+{% endmethod %}
+
+
+### path
+{% method %}
+The path, where to store raw.json
+{% sample lang="toml" %}
+```toml
+path     = "/var/www/html/meshviewer/data/raw.json"
+```
+{% endmethod %}
+
+
+
+## [database]
+{% method %}
+The database organize all database types.
+For all database types the is a internal job, which reset data for nodes (global statistics are still stored).
+_(We have for privacy policy to store node data for maximum seven days.)_
+{% sample lang="toml" %}
+```toml
+delete_after = "7d"
+delete_interval = "1h"
+```
+{% endmethod %}
+
+
+### delete_after
+{% method %}
+This will send delete commands to the database to prune data which is older than:
+{% sample lang="toml" %}
+```toml
+delete_after = "7d"
+```
+{% endmethod %}
+
+
+### delete_interval
+{% method %}
+How often run the delete commands.
+{% sample lang="toml" %}
+```toml
+delete_interval = "1h"
+```
+{% endmethod %}
+
+
+## [[database.connection.example]]
+{% method %}
+This example block shows all option which is useable for every following database type.
+Every database type has his own configuration under `database.connection`.
+It is possible to have multiple connections for one type of database, just add this group again with new parameters (see toml [[array of table]]).
+{% sample lang="toml" %}
+```toml
+[[database.connection.example]]
+enable = true
+```
+{% endmethod %}
+
+
+### enable
+{% method %}
+Each database-connection has its own config block and needs to be enabled by adding:
+{% sample lang="toml" %}
+```toml
+enable = true
+```
+{% endmethod %}
+
+
+
+## [[database.connection.influxdb]]
+{% method %}
+Save collected data to InfluxDB.
+There are would be the following measurements:
+- node: store node specific data i.e. clients memory, airtime
+- global: store global data, i.e. count of clients and nodes
+- firmware: store the count of nodes tagged with firmware
+- model: store the count of nodes tagged with hardware model
+{% sample lang="toml" %}
+```toml
+enable   = false
+address  = "http://localhost:8086"
+database = "ffhb"
+username = ""
+password = ""
+insecure_skip_verify = false
+[database.connection.influxdb.tags]
+tagname1 = "tagvalue 1"
+system   = "productive"
+site     = "ffhb"
+```
+{% endmethod %}
+
+
+### address
+{% method %}
+Address to connect on InfluxDB server.
+{% sample lang="toml" %}
+```toml
+address  = "http://localhost:8086"
+```
+{% endmethod %}
+
+
+### database
+{% method %}
+Database on which the measurement should be stored.
+{% sample lang="toml" %}
+```toml
+database = "ffhb"
+```
+{% endmethod %}
+
+
+### username
+{% method %}
+Username to authenticate on InfluxDB
+{% sample lang="toml" %}
+```toml
+username = ""
+```
+{% endmethod %}
+
+
+### password
+{% method %}
+Password to authenticate on InfluxDB.
+{% sample lang="toml" %}
+```toml
+password = ""
+```
+{% endmethod %}
+
+### insecure_skip_verify
+{% method %}
+Skip insecure verify for self-signed certificates.
+{% sample lang="toml" %}
+```toml
+insecure_skip_verify = true
+```
+{% endmethod %}
+
+
+### [database.connection.influxdb.tags]
+{% method %}
+You could set manuelle tags with inserting into a influxdb.
+Useful if you want to identify the yanic instance when you use multiple own on the same influxdb (e.g. multisites).
+
+Warning:
+Tags used by Yanic would override the tags from this config (e.g. `nodeid`, `hostname`, `owner`, `model`, `firmware_base`, `firmware_release`, `frequency11g`, `frequency11a`).
+{% sample lang="toml" %}
+```toml
+tagname1 = "tagvalue 1s"
+# some useful e.g.:
+system   = "productive"
+site     = "ffhb"
+```
+{% endmethod %}
+
+
+
+## [[database.connection.graphite]]
+{% method %}
+Save collected data to a graphite database.
+{% sample lang="toml" %}
+```toml
+enable   = false
+address  = "localhost:2003"
+prefix   = "freifunk"
+```
+{% endmethod %}
+
+
+### address
+{% method %}
+Address to connect on graphite server.
+{% sample lang="toml" %}
+```toml
+address = "localhost:2003"
+```
+{% endmethod %}
+
+
+### prefix
+{% method %}
+Graphite is replacing every "." in the metric name with a slash "/" and uses
+that for the file system hierarchy it generates. it is recommended to at least
+move the metrics out of the root namespace (that would be the empty prefix).
+If you only intend to run one community and only freifunk on your graphite node
+then the prefix can be set to anything (including the empty string) since you
+probably wont care much about "polluting" the namespace.
+{% sample lang="toml" %}
+```toml
+prefix = "freifunk"
+```
+{% endmethod %}
+
+
+
+## [[database.connection.respondd]]
+{% method %}
+Forward collected respondd package to a address
+(e.g. to another respondd collector like a central yanic instance or hopglass)
+{% sample lang="toml" %}
+```toml
+enable   = false
+type     = "udp6"
+address  = "stats.bremen.freifunk.net:11001"
+```
+{% endmethod %}
+
+
+### type
+{% method %}
+Type of network to create a connection.
+
+Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only), "udp", "udp4" (IPv4-only), "udp6" (IPv6-only), "ip", "ip4" (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and "unixpacket".
+{% sample lang="toml" %}
+```toml
+type     = "udp6"
+```
+{% endmethod %}
+
+
+### address
+{% method %}
+Destination address to connect/send respondd package.
+{% sample lang="toml" %}
+```toml
+address  = "stats.bremen.freifunk.net:11001"
+```
+{% endmethod %}
+
+
+
+## [[database.connection.logging]]
+{% method %}
+This database type is just for, debugging without a real database connection.
+A example for other developers for new database types.
+{% sample lang="toml" %}
+```toml
+enable   = false
+path     = "/var/log/yanic.log"
+```
+{% endmethod %}
+
+
+### path
+{% method %}
+Path to file where to store some examples with every line.
+{% sample lang="toml" %}
+```toml
+path     = "/var/log/yanic.log"
+```
+{% endmethod %}

docs/docs_install.md

@@ -0,0 +1,54 @@
+# Build and Installation
+
+## go
+
+### Install
+```sh
+cd /usr/local/
+wget https://dl.google.com/go/go1.13.1.linux-amd64.tar.gz -O go-release-linux-amd64.tar.gz
+tar xvf go-release-linux-amd64.tar.gz
+rm go-release-linux-amd64.tar.gz
+```
+
+### Configure go
+Add these lines in your root shell startup file (e.g. `/root/.bashrc`):
+```sh
+export GOPATH=/opt/go
+export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin
+```
+
+## Yanic
+
+### Compile
+As root:
+```sh
+go get -v -u github.com/FreifunkBremen/yanic
+```
+
+### Install
+
+```sh
+cp /opt/go/src/github.com/FreifunkBremen/yanic/contrib/init/linux-systemd/yanic.service /lib/systemd/system/yanic.service
+systemctl daemon-reload
+```
+
+Before start, you should configure yanic by the file `/etc/yanic.conf`:
+```sh
+systemctl start yanic
+```
+
+Enable to start on boot:
+```sh
+systemctl enable yanic
+```
+
+### Update
+For an update just stop yanic and then call the same `go` command again (again as root):
+```sh
+systemctl stop yanic
+go get -v -u github.com/FreifunkBremen/yanic
+```
+Then update the config file, for example look at the diff with the new example:
+```sh
+diff /opt/go/src/github.com/FreifunkBremen/yanic/config_example.toml /etc/yanic.conf
+```

docs/docs_quick_conf.md

@@ -0,0 +1,31 @@
+# Quick Configuration
+
+```sh
+cp /opt/go/src/github.com/FreifunkBremen/yanic/config_example.toml /etc/yanic.conf
+```
+
+# Quick configuration
+For an easy startup you only need to edit the `[[respondd.interfaces]]` in section
+`[respondd]` in file `/etc/yanic.conf`.  
+
+Then create the following files and folders:
+```sh
+adduser --system yanic --home /var/lib/yanic
+mkdir -p /var/lib/yanic
+mkdir -p /var/www/html/meshviewer/data
+touch /var/log/yanic.log
+chown yanic /var/log/yanic.log /var/lib/yanic /var/www/html/meshviewer/data
+```
+
+#### Standalone
+If you like to run a standalone meshviewer, just set `enable` in section
+`[webserver]` to `true`.
+
+##### Configure the [meshviewer](https://github.com/ffrgb/meshviewer):
+set `dataPath` in `config.json` to `/data/` and make the `build` directory
+accessible under `/var/www/html/meshviewer`.
+
+#### With webserver (Apache, nginx)
+The meshviewer needs the output files like `nodes_path` and `graph_path` inside
+the same directory as the `dataPath`. Change the path in the section
+`[meshviewer]` accordingly.

docs/docs_usage.md

@@ -0,0 +1,75 @@
+# Usage
+
+Yanic provides several commands:
+
+* `import`
+* `query`
+* `serve`
+
+## Import
+
+### RRD-File
+Warning, just tested with olddata.rrd from Freifunk Bremen generated by detailed-rrds branch of [ffmap-backend](https://github.com/ffnord/ffmap-backend/tree/detailed-rrds)
+
+```
+Usage:
+  yanic import <file.rrd> [flags]
+
+Examples:
+  yanic import --config /etc/yanic.toml olddata.rrd
+
+Flags:
+  -c, --config string   Path to configuration file (default "config.toml")
+  -h, --help            help for import
+```
+
+### Firstseen
+To import firstseen values there is a little script in contrib:
+
+```
+/opt/go/src/github.com/FreifunkBremen/yanic/contrib/yanic-import-timestamp -n path/to/nodes_old.json -s state.json /var/lib/yanic/state.json
+```
+
+On a productive system @ once:
+
+```
+systemctl stop yanic; cp /var/lib/yanic/state.json /var/lib/yanic/state.bak; /opt/go/src/github.com/FreifunkBremen/yanic/contrib/yanic-import-timestamp -n path/to/nodes_old.json -s /var/lib/yanic/state.json; systemctl start yanic;
+```
+
+## Serve
+runs yanic in collector-modus to genereate files (e.g. for meshviewer) and save values in databases
+
+from shell
+
+```
+Usage:
+  yanic serve [flags]
+
+Examples:
+  yanic serve --config /etc/yanic.toml
+
+Flags:
+  -c, --config string   Path to configuration file (default "config.toml")
+  -h, --help            help for serve
+```
+
+or run as [daemon]({{site.baseurl}}/docs/install.html)
+
+
+## Query
+
+Send a single request and show response like `gluon-neighbour-info` on gluon.
+
+e.g.  to check the right interface
+
+```
+Usage:
+  yanic query <interface> <destination> [flags]
+
+Examples:
+  yanic query wlan0 "fe80::eade:27ff:dead:beef"
+
+Flags:
+  -h, --help       help for query
+      --wait int   Seconds to wait for a response (default 1)
+```

docs/home_about.md

@@ -0,0 +1,45 @@
+# About
+
+A little overview of yanic in connection with other software:
+![Overview](overview.svg)
+
+## How respondd works
+
+It sends the `gluon-neighbour-info` request and collects the answers.
+
+It will send UDP packets with multicast address `ff05:0:0:0:0:0:2:1001` and port `1001`.
+
+If a node does not answer, it will request with the last know address under the port `1001`.
+
+## Related projects
+
+#### yanic collecting data
+VPNs (respondd for servers):
+
+* [mesh-announce](https://github.com/ffnord/mesh-announce) from FreiFunkNord
+* [respondd](https://github.com/Sunz3r/ext-respondd) from Sunz3r
+
+Nodes (respondd for nodes): [gluon](https://github.com/freifunk-gluon/gluon/)
+
+#### Alternative collectors of respondd data:
+
+* [Node informant](https://github.com/ffdo/node-informant) written in Go
+* [HopGlass Server](https://github.com/plumpudding/hopglass-server) written in Node.js
+
+#### yanic published data
+
+**Databases:**
+
+* [InfluxDB](https://influxdata.com/) SQL-like timeserial database
+* [Graphite](https://graphiteapp.org/) RRD file Based
+
+	Visualization from Databases: [Grafana](https://grafana.com/)
+
+**Output:**
+* meshviewer-ffrgb:
+  * [meshviewer](https://github.com/ffrgb/meshviewer)
+* nodelist:
+  * [ffapi](https://freifunk.net/api-generator/)
+    * [freifunk-karte.de](https://freifunk-karte.de)
+* meshviewer (others):
+  *  unmaintained [origin meshviewer](https://github.com/ffnord/meshviewer) branch: master (v1) and dev (v2)

docs/home_running.md

@@ -0,0 +1,24 @@
+# Running Yanic
+
+| Community | Meshviewer | Database Visualisation \(Grafana\) |
+| :--- | :--- | :--- |
+| Freifunk Bremen | [meshviewer](https://map.bremen.freifunk.net) from Freifunk Regensburgwith a patch to show state.json | [grafana](https://grafana.bremen.freifunk.net) - with influxdb |
+| Freifunk Regensburg | [meshviewer ](https://regensburg.freifunk.net/meshviewer/)from Freifunk Regensburg | [grafana](https://grafana.regensburg.freifunk.net/) - without yanic |
+| Freifunk Frankfurt am Main | [meshviewer ](https://map.ffm.freifunk.net)from Freifunk Regensburg | - |
+| Freifunk Hannover | [meshviewer ](https://hannover.freifunk.net/karte/)from Freifunk Regensburg | [grafana ](https://stats.ffh.zone)with influxdb |
+| Freifunk Hochstift | [meshviewer ](https://map.hochstift.freifunk.net)from Freifunk Regensburg | - |
+| Freifunk in Mainz , Wiesbaden & Umgebung | [meshviewer ](https://mapng.freifunk-mwu.de/)from Freifunk Regensburg | [grafana](https://stats.freifunk-mwu.de) with influxdb |
+| Freifunk Ulzen | [meshviewer ](http://map.ffue.eu)from Freifunk Regensburg | - |
+| Freifunk Gera Greiz | [meshviewer](https://www.freifunk-gera-greiz.de/meshviewer/) from Freifunk Regensburg | [grafana](https://www.freifunk-gera-greiz.de/grafana/dashboard/db/meshviewer-graphen) - with influxdb |
+| Freifunk Westpfalz | [meshviewer ](https://map.freifunk-westpfalz.de/)from Freifunk Regensburg | [grafana](https://stats.freifunk-westpfalz.de/)- with influxdb |
+
+
+### meshviewer-collector
+A little project starts, to collect meshviewer-ffrgb data from multiple communities.
+
+
+List of communities:
+[https://github.com/genofire/meshviewer-collector](https://github.com/genofire/meshviewer-collector/blob/master/config.toml)
+
+Meshviewer with colleced data:
+[https://web.fireorbit.de/meshviewer](https://web.fireorbit.de/meshviewer).

docs/index.md

@@ -0,0 +1,24 @@
+# Home
+
+
+    __   __          _
+    \ \ / /_ _ _ __ (_) ___
+     \ V / _` | '_ \| |/ __|
+      | | (_| | | | | | (__
+      |_|\__,_|_| |_|_|\___|
+    Yet another node info collector
+
+(previously [respond-collector](https://github.com/FreifunkBremen/respond-collector))
+
+[![Build Status](https://travis-ci.org/FreifunkBremen/yanic.svg?branch=master)](https://travis-ci.org/FreifunkBremen/yanic)
+[![Coverage Status](https://coveralls.io/repos/github/FreifunkBremen/yanic/badge.svg?branch=master)](https://coveralls.io/github/FreifunkBremen/yanic?branch=master)
+[![codecov](https://codecov.io/gh/FreifunkBremen/yanic/branch/master/graph/badge.svg)](https://codecov.io/gh/FreifunkBremen/yanic)
+[![Go Report Card](https://goreportcard.com/badge/github.com/FreifunkBremen/yanic)](https://goreportcard.com/report/github.com/FreifunkBremen/yanic)
+
+`yanic` is a respondd client that fetches, stores and publishes information about a Freifunk network.
+
+## The goals:
+
+* Generating JSON for [Meshviewer](https://github.com/ffrgb/meshviewer)
+* Storing statistics in [InfluxDB](https://influxdata.com/) or [Graphite](https://graphiteapp.org/) to be analyzed by [Grafana](http://grafana.org/)
+* Provide a little webserver for a standalone installation with a meshviewer

docs/overview.xml

@@ -0,0 +1 @@
+<mxfile userAgent="Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" version="7.6.7" editor="www.draw.io" type="device"><diagram id="c22c9a54-a830-b601-ab41-3f056e49af3e" name="Page-1">5Vptc6M2EP41nsl9sAchwM7HOG/tTO96M2nvmo+yEaAEI1eI2L5fXwkk8yLsOrHB7jUfYlgtQuw+z+5qYQBvF+tHhpbRZ+rjeGBb/noA7wa2DSxnIn6kZFNIPGgVgpARXymVgifyA+srlTQjPk5ripzSmJNlXTinSYLnvCZDjNFVXS2gcf2uSxRiQ/A0R7Ep/U58HhXSiWuV8l8wCSN9Z2CpkRmav4aMZom638CGQf5XDC+QnkvppxHy6aoigvcDeMso5cXRYn2LY2lbbbbiuocdo9t1M5zwQy6AUK2Db/SzY1+YQp1SxiMa0gTF96V0mj8fljNY4izii1gcAnGI14T/JcUjV509q5EXzPlG+RllnApROfdvlC6VXrEauYSdD6REKc3YXGnZW8sJRGK6wJxthArDMeLkrT4VUtAIt3rbS79SIm5iWwrGUKNRgdj2rPoUHLEQc3VVaWRxUFlGKcpN3+4Gu5j4DcWZWu0XKglge7F46umMiaNQHl1pERLqEcNBru39nUnACFdwSZAbuRT7ISQ8ymajOV2Ik4BhEmTJ6zCMM5rIUfVbXq3voUaK+wgNpAc+GVhZRYTjpyXKHbESsaCOB5QuC3YGZC3xotz7hhnH6/0ONr2pvTKue0VzclXyFGhHRVWONr1XBUDNdfv8BAwTdEIXYQu2qQzJ02c1QbdUUvGggLay8JHsOtS40CDBt69fTsqBIKHMl8vAaTRESSIcI567jQMNjQvlwjbtnIMLOjpW/HWHOJqhFHdDErvfpOKaTPB6YoLdUVYeX06cabEuGPdlXud/ad5JT+Z1rJ7Me6gJ97nhdOa97qYIdesh3gGT+hTFsowi1Jho3KhmnQmsT7Sjmr1hDG0qakupkL5jwe7kfeuq64uDYgUHltZ/ppj9PnuRqda2YjTD8UBtxD5WMzyounnK8ALLenmDEjI3C4ZnJTYKheLhNDeqtZSizXlLCBvu2OR0UUOI04p/dpQVPe1IzxebPTN49JX5vN0lm0R/s9IeDmT74GDi+HSejkgSxNnaF9MqBinBrLXO/nU72MacI9cTyuaUYNSIYSTAjIv10daFfI+IYJe408Ojumo3mc+8AW7G8T6LfqeTDTCocLOg4/tTe43D+RwfY+e4JbUfuwP+UCLdslU5euw1Wnn/qg+OSaSm680mFVnIPmoTEALavO5nhlPyA81yBeko9ehC250O3DtJlJiEiRDMhWsEC+FUUoTMUXyjBhbE93Mw5Tl9um2z3tKYsvy+utG6h2SqXaxWMth2YauQcPazzxoBYKkS7KOlnVahQZDiY1uHY8Mr74uQAUp0pDaj4mMxflh0vvpG0kw4MiWc0OQiGiSNWOlNQI+xErhHuabeuWLhTDWu3gheyUTV9NXn6thFZi7bhWd0hw6P/+1XHaCL/HQwos1gUyJyqEBqhIVSZfSSXkhgsMeTehV17ZpQBG1QBCeAIuwJiufb4+guRRWmlRecHb9IMGEaBGhJfrJKBe5oR5eVCmygHFxQ3aIRUnFSIr8jICnPw8QlRAnHc88ZJeyfPkpcm1EC7nDJ6ZPZ9Z5kJuRXb6BoN6CFhFrxX+rbn8wkJ6GbKtyKa6Re3n24GCh7TgPKkxYowzYow1PUXua7wneUwttvF16RsMDIx2bx26ZykTWw4zS3JL1+v2AGXZHxljTx/QOSY8oZfcU6jyU0kQEnIHHcEB2eItscUI9gJzC5fd1ocQPT5E6Lxe1TGNwMMjTjy4yfphYRETktgjOo2D3GAW+xOpfRepoKa5Mk/CMP3UPnRKi26zs7B04ME7sdmRgC08Q8wiw9mu4+SqMciOBEMdhoR3T7rqX8jLCoz8pvNeH9Pw==</diagram></mxfile>

lib/duration/duration.go

@@ -1,9 +1,11 @@
-package runtime
+package duration
 
 import (
 	"fmt"
 	"strconv"
 	"time"
+
+	"github.com/pkg/errors"
 )
 
 // Duration is a TOML datatype
@@ -16,14 +18,8 @@ type Duration struct {
 }
 
 // UnmarshalTOML parses a duration string.
-func (d *Duration) UnmarshalTOML(dataInterface interface{}) error {
-	var data string
-	switch dataInterface.(type) {
-	case string:
-		data = dataInterface.(string)
-	default:
-		return fmt.Errorf("invalid duration: \"%s\"", dataInterface)
-	}
+func (d *Duration) UnmarshalText(data []byte) error {
+
 	// " + int + unit + "
 	if len(data) < 2 {
 		return fmt.Errorf("invalid duration: \"%s\"", data)
@@ -32,7 +28,7 @@ func (d *Duration) UnmarshalTOML(dataInterface interface{}) error {
 	unit := data[len(data)-1]
 	value, err := strconv.Atoi(string(data[:len(data)-1]))
 	if err != nil {
-		return fmt.Errorf("unable to parse duration %s: %s", data, err)
+		return errors.Wrapf(err, "unable to parse duration \"%s\"", data)
 	}
 
 	switch unit {
@@ -49,7 +45,7 @@ func (d *Duration) UnmarshalTOML(dataInterface interface{}) error {
 	case 'y':
 		d.Duration = time.Duration(value) * time.Hour * 24 * 365
 	default:
-		return fmt.Errorf("invalid duration unit: %s", string(unit))
+		return fmt.Errorf("invalid duration unit \"%s\"", string(unit))
 	}
 
 	return nil

lib/duration/duration_test.go

@@ -1,4 +1,4 @@
-package runtime
+package duration
 
 import (
 	"testing"
@@ -16,7 +16,9 @@ func TestDuration(t *testing.T) {
 		duration time.Duration
 	}{
 		{"", "invalid duration: \"\"", 0},
-		{"1x", "invalid duration unit: x", 0},
+		{"3", "invalid duration: \"3\"", 0},
+		{"am", "unable to parse duration \"am\": strconv.Atoi: parsing \"a\": invalid syntax", 0},
+		{"1x", "invalid duration unit \"x\"", 0},
 		{"1s", "", time.Second},
 		{"73s", "", time.Second * 73},
 		{"1m", "", time.Minute},
@@ -34,7 +36,7 @@ func TestDuration(t *testing.T) {
 	for _, test := range tests {
 
 		d := Duration{}
-		err := d.UnmarshalTOML(test.input)
+		err := d.UnmarshalText([]byte(test.input))
 		duration := d.Duration
 
 		if test.err == "" {

main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os"
+
+	"github.com/bdlm/log"
+	stdLogger "github.com/bdlm/std/logger"
+
+	"github.com/FreifunkBremen/yanic/cmd"
+)
+
+type hook struct{}
+
+func (h *hook) Fire(entry *log.Entry) error {
+	switch entry.Level {
+	case log.PanicLevel:
+		entry.Logger.Out = os.Stderr
+	case log.FatalLevel:
+		entry.Logger.Out = os.Stderr
+	case log.ErrorLevel:
+		entry.Logger.Out = os.Stderr
+	case log.WarnLevel:
+		entry.Logger.Out = os.Stdout
+	case log.InfoLevel:
+		entry.Logger.Out = os.Stdout
+	case log.DebugLevel:
+		entry.Logger.Out = os.Stdout
+	default:
+	}
+
+	return nil
+}
+
+func (h *hook) Levels() []stdLogger.Level {
+	return log.AllLevels
+}
+
+func main() {
+	log.AddHook(&hook{})
+
+	cmd.Execute()
+}

output/all/filter.go

@@ -0,0 +1,11 @@
+package all
+
+import (
+	_ "github.com/FreifunkBremen/yanic/output/filter/blacklist"
+	_ "github.com/FreifunkBremen/yanic/output/filter/domainappendsite"
+	_ "github.com/FreifunkBremen/yanic/output/filter/domainassite"
+	_ "github.com/FreifunkBremen/yanic/output/filter/haslocation"
+	_ "github.com/FreifunkBremen/yanic/output/filter/inarea"
+	_ "github.com/FreifunkBremen/yanic/output/filter/noowner"
+	_ "github.com/FreifunkBremen/yanic/output/filter/site"
+)

output/all/internal.go

@@ -1,37 +1,45 @@
 package all
 
 import (
+	"sync"
+	"time"
+
 	"github.com/FreifunkBremen/yanic/output"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 
-type Output struct {
-	output.Output
-	nodes *runtime.Nodes
-	list  []output.Output
+var quit chan struct{}
+var wg = sync.WaitGroup{}
+var outputA output.Output
+
+func Start(nodes *runtime.Nodes, config runtime.NodesConfig) (err error) {
+	outputA, err = Register(config.Output)
+	if err != nil {
+		return
+	}
+	quit = make(chan struct{})
+	wg.Add(1)
+	go saveWorker(nodes, config.SaveInterval.Duration)
+	return
 }
 
-func Register(nodes *runtime.Nodes, configuration interface{}) (output.Output, error) {
-	var list []output.Output
-	allOutputs := configuration.(map[string][]interface{})
-	for outputType, outputRegister := range output.Adapters {
-		outputConfigs := allOutputs[outputType]
-		for _, config := range outputConfigs {
-			output, err := outputRegister(nodes, config)
-			if err != nil {
-				return nil, err
-			}
-			if output == nil {
-				continue
-			}
-			list = append(list, output)
+func Close() {
+	close(quit)
+	wg.Wait()
+	quit = nil
+}
+
+// save periodically to output
+func saveWorker(nodes *runtime.Nodes, saveInterval time.Duration) {
+	ticker := time.NewTicker(saveInterval)
+	for {
+		select {
+		case <-ticker.C:
+			outputA.Save(nodes)
+		case <-quit:
+			ticker.Stop()
+			wg.Done()
+			return
 		}
 	}
-	return &Output{list: list, nodes: nodes}, nil
-}
-
-func (o *Output) Save() {
-	for _, item := range o.list {
-		item.Save()
-	}
 }

output/all/main.go

@@ -1,6 +1,9 @@
 package all
 
 import (
+	_ "github.com/FreifunkBremen/yanic/output/geojson"
 	_ "github.com/FreifunkBremen/yanic/output/meshviewer"
-	_ "github.com/FreifunkBremen/yanic/output/template"
+	_ "github.com/FreifunkBremen/yanic/output/meshviewer-ffrgb"
+	_ "github.com/FreifunkBremen/yanic/output/nodelist"
+	_ "github.com/FreifunkBremen/yanic/output/raw"
 )

output/all/output.go

@@ -0,0 +1,71 @@
+package all
+
+import (
+	"fmt"
+
+	"github.com/bdlm/log"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/output/filter"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	list         map[int]output.Output
+	outputFilter map[int]filter.Set
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	list := make(map[int]output.Output)
+	outputFilter := make(map[int]filter.Set)
+	i := 1
+	allOutputs := configuration
+	for outputType, outputRegister := range output.Adapters {
+		configForOutput := allOutputs[outputType]
+		if configForOutput == nil {
+			log.WithField("output", outputType).Infof("no configuration found")
+			continue
+		}
+		outputConfigs, ok := configForOutput.([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("the output type '%s' has the wrong format", outputType)
+		}
+		for _, outputConfig := range outputConfigs {
+			config, ok := outputConfig.(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("the output type '%s' has the wrong format", outputType)
+			}
+			if c, ok := config["enable"].(bool); ok && !c {
+				continue
+			}
+			output, err := outputRegister(config)
+			if err != nil {
+				return nil, err
+			}
+			if output == nil {
+				continue
+			}
+			var errs []error
+			var filterSet filter.Set
+			if c := config["filter"]; c != nil {
+				if filterConf, ok := c.(map[string]interface{}); ok {
+					filterSet, errs = filter.New(filterConf)
+				}
+				if len(errs) > 0 {
+					return nil, fmt.Errorf("filter configuration errors: %v", errs)
+				}
+				outputFilter[i] = filterSet
+			}
+			list[i] = output
+			i++
+		}
+	}
+	return &Output{list: list, outputFilter: outputFilter}, nil
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	for i, item := range o.list {
+		item.Save(o.outputFilter[i].Apply(nodes))
+	}
+}

output/all/output_test.go

@@ -0,0 +1,121 @@
+package all
+
+import (
+	"errors"
+	"sync"
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+type testOutput struct {
+	output.Output
+	countSave int
+	sync.Mutex
+}
+
+func (c *testOutput) Save(nodes *runtime.Nodes) {
+	c.Lock()
+	c.countSave++
+	c.Unlock()
+}
+func (c *testOutput) Get() int {
+	c.Lock()
+	defer c.Unlock()
+	return c.countSave
+}
+
+func TestStart(t *testing.T) {
+	assert := assert.New(t)
+
+	nodes := &runtime.Nodes{}
+
+	globalOutput := &testOutput{}
+	output.RegisterAdapter("a", func(config map[string]interface{}) (output.Output, error) {
+		return globalOutput, nil
+	})
+	output.RegisterAdapter("b", func(config map[string]interface{}) (output.Output, error) {
+		return globalOutput, nil
+	})
+	output.RegisterAdapter("c", func(config map[string]interface{}) (output.Output, error) {
+		return globalOutput, nil
+	})
+	output.RegisterAdapter("d", func(config map[string]interface{}) (output.Output, error) {
+		return nil, nil
+	})
+	output.RegisterAdapter("e", func(config map[string]interface{}) (output.Output, error) {
+		return nil, errors.New("blub")
+	})
+	allOutput, err := Register(map[string]interface{}{
+		"a": []interface{}{
+			map[string]interface{}{
+				"enable": false,
+				"path":   "a1",
+			},
+			map[string]interface{}{
+				"path": "a2",
+			},
+			map[string]interface{}{
+				"enable": true,
+				"path":   "a3",
+			},
+		},
+		"b": nil,
+		"c": []interface{}{
+			map[string]interface{}{
+				"path":   "c1",
+				"filter": map[string]interface{}{},
+			},
+		},
+		// fetch continue command in Connect
+		"d": []interface{}{
+			map[string]interface{}{
+				"path": "d0",
+			},
+		},
+	})
+	assert.NoError(err)
+
+	assert.Equal(0, globalOutput.Get())
+	allOutput.Save(nodes)
+	assert.Equal(3, globalOutput.Get())
+
+	// wrong format - map
+	_, err = Register(map[string]interface{}{
+		"e": []interface{}{
+			false,
+		},
+	})
+	assert.Error(err)
+
+	// wrong format - array
+	_, err = Register(map[string]interface{}{
+		"e": true,
+	})
+	assert.Error(err)
+
+	// output error
+	_, err = Register(map[string]interface{}{
+		"e": []interface{}{
+			map[string]interface{}{
+				"enable": true,
+			},
+		},
+	})
+	assert.Error(err)
+
+	// output error invalid config of filter
+	_, err = Register(map[string]interface{}{
+		"a": []interface{}{
+			map[string]interface{}{
+				"enable": true,
+				"filter": map[string]interface{}{
+					"blacklist": true,
+				},
+			},
+		},
+	})
+	assert.Error(err)
+}

output/filter/blacklist/blacklist.go

@@ -0,0 +1,40 @@
+package blacklist
+
+import (
+	"errors"
+
+	"github.com/FreifunkBremen/yanic/output/filter"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type blacklist map[string]interface{}
+
+func init() {
+	filter.Register("blacklist", build)
+}
+
+func build(config interface{}) (filter.Filter, error) {
+	values, ok := config.([]interface{})
+	if !ok {
+		return nil, errors.New("invalid configuration, array (of strings) expected")
+	}
+
+	list := make(blacklist)
+	for _, value := range values {
+		if nodeid, ok := value.(string); ok {
+			list[nodeid] = struct{}{}
+		} else {
+			return nil, errors.New("invalid configuration, array of strings expected")
+		}
+	}
+	return &list, nil
+}
+
+func (list blacklist) Apply(node *runtime.Node) *runtime.Node {
+	if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+		if _, ok := list[nodeinfo.NodeID]; ok {
+			return nil
+		}
+	}
+	return node
+}

output/filter/blacklist/blacklist_test.go

@@ -0,0 +1,43 @@
+package blacklist
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterBlacklist(t *testing.T) {
+	assert := assert.New(t)
+
+	// invalid config
+	filter, err := build(3)
+	assert.Error(err)
+
+	filter, err = build([]interface{}{2, "a"})
+	assert.Error(err)
+
+	// tests with empty list
+	filter, err = build([]interface{}{})
+	assert.NoError(err)
+
+	// keep node without nodeid
+	n := filter.Apply(&runtime.Node{Nodeinfo: &data.Nodeinfo{}})
+	assert.NotNil(n)
+
+	// tests with blacklist
+	filter, err = build([]interface{}{"a", "c"})
+	assert.NoError(err)
+
+	// blacklist contains node with nodeid -> drop it
+	n = filter.Apply(&runtime.Node{Nodeinfo: &data.Nodeinfo{NodeID: "a"}})
+	assert.Nil(n)
+
+	// blacklist does not contains node without nodeid -> keep it
+	n = filter.Apply(&runtime.Node{Nodeinfo: &data.Nodeinfo{}})
+	assert.NotNil(n)
+
+	n = filter.Apply(&runtime.Node{})
+	assert.NotNil(n)
+}

output/filter/domainappendsite/domainappendsite.go

@@ -0,0 +1,50 @@
+package domainappendsite
+
+import (
+	"errors"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/output/filter"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type domainAppendSite struct{ set bool }
+
+func init() {
+	filter.Register("domain_append_site", build)
+}
+
+func build(config interface{}) (filter.Filter, error) {
+	if value, ok := config.(bool); ok {
+		return &domainAppendSite{set: value}, nil
+	}
+	return nil, errors.New("invalid configuration, boolean expected")
+}
+
+func (config *domainAppendSite) Apply(node *runtime.Node) *runtime.Node {
+	if nodeinfo := node.Nodeinfo; nodeinfo != nil && config.set && nodeinfo.System.DomainCode != "" {
+		node = &runtime.Node{
+			Address:    node.Address,
+			Firstseen:  node.Firstseen,
+			Lastseen:   node.Lastseen,
+			Online:     node.Online,
+			Statistics: node.Statistics,
+			Nodeinfo: &data.Nodeinfo{
+				NodeID:  nodeinfo.NodeID,
+				Network: nodeinfo.Network,
+				System: data.System{
+					SiteCode: nodeinfo.System.SiteCode + "." + nodeinfo.System.DomainCode,
+				},
+				Owner:    nodeinfo.Owner,
+				Hostname: nodeinfo.Hostname,
+				Location: nodeinfo.Location,
+				Software: nodeinfo.Software,
+				Hardware: nodeinfo.Hardware,
+				VPN:      nodeinfo.VPN,
+				Wireless: nodeinfo.Wireless,
+			},
+			Neighbours: node.Neighbours,
+		}
+	}
+	return node
+}

output/filter/domainappendsite/domainappendsite_test.go

@@ -0,0 +1,43 @@
+package domainappendsite
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilter(t *testing.T) {
+	assert := assert.New(t)
+
+	// invalid config
+	filter, err := build("nope")
+	assert.Error(err)
+
+	// delete owner by configuration
+	filter, _ = build(true)
+	n := filter.Apply(&runtime.Node{Nodeinfo: &data.Nodeinfo{
+		System: data.System{
+			SiteCode:   "ffhb",
+			DomainCode: "city",
+		},
+	}})
+
+	assert.NotNil(n)
+	assert.Equal("ffhb.city", n.Nodeinfo.System.SiteCode)
+	assert.Equal("", n.Nodeinfo.System.DomainCode)
+
+	// keep owner configuration
+	filter, _ = build(false)
+	n = filter.Apply(&runtime.Node{Nodeinfo: &data.Nodeinfo{
+		System: data.System{
+			SiteCode:   "ffhb",
+			DomainCode: "city",
+		},
+	}})
+
+	assert.NotNil(n)
+	assert.Equal("ffhb", n.Nodeinfo.System.SiteCode)
+	assert.Equal("city", n.Nodeinfo.System.DomainCode)
+}

output/filter/domainassite/domainassite.go

@@ -0,0 +1,50 @@
+package domainassite
+
+import (
+	"errors"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/output/filter"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type domainAsSite struct{ set bool }
+
+func init() {
+	filter.Register("domain_as_site", build)
+}
+
+func build(config interface{}) (filter.Filter, error) {
+	if value, ok := config.(bool); ok {
+		return &domainAsSite{set: value}, nil
+	}
+	return nil, errors.New("invalid configuration, boolean expected")
+}
+
+func (config *domainAsSite) Apply(node *runtime.Node) *runtime.Node {
+	if nodeinfo := node.Nodeinfo; nodeinfo != nil && config.set && nodeinfo.System.DomainCode != "" {
+		node = &runtime.Node{
+			Address:    node.Address,
+			Firstseen:  node.Firstseen,
+			Lastseen:   node.Lastseen,
+			Online:     node.Online,
+			Statistics: node.Statistics,
+			Nodeinfo: &data.Nodeinfo{
+				NodeID:  nodeinfo.NodeID,
+				Network: nodeinfo.Network,
+				System: data.System{
+					SiteCode: nodeinfo.System.DomainCode,
+				},
+				Owner:    nodeinfo.Owner,
+				Hostname: nodeinfo.Hostname,
+				Location: nodeinfo.Location,
+				Software: nodeinfo.Software,
+				Hardware: nodeinfo.Hardware,
+				VPN:      nodeinfo.VPN,
+				Wireless: nodeinfo.Wireless,
+			},
+			Neighbours: node.Neighbours,
+		}
+	}
+	return node
+}

output/filter/domainassite/domainassite_test.go

@@ -0,0 +1,43 @@
+package domainassite
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilter(t *testing.T) {
+	assert := assert.New(t)
+
+	// invalid config
+	filter, err := build("nope")
+	assert.Error(err)
+
+	// delete owner by configuration
+	filter, _ = build(true)
+	n := filter.Apply(&runtime.Node{Nodeinfo: &data.Nodeinfo{
+		System: data.System{
+			SiteCode:   "ffhb",
+			DomainCode: "city",
+		},
+	}})
+
+	assert.NotNil(n)
+	assert.Equal("city", n.Nodeinfo.System.SiteCode)
+	assert.Equal("", n.Nodeinfo.System.DomainCode)
+
+	// keep owner configuration
+	filter, _ = build(false)
+	n = filter.Apply(&runtime.Node{Nodeinfo: &data.Nodeinfo{
+		System: data.System{
+			SiteCode:   "ffhb",
+			DomainCode: "city",
+		},
+	}})
+
+	assert.NotNil(n)
+	assert.Equal("ffhb", n.Nodeinfo.System.SiteCode)
+	assert.Equal("city", n.Nodeinfo.System.DomainCode)
+}