[TASK] improve output (to stats page and filtering in meshviewer)

cmd/yanic/main.go

@@ -9,8 +9,9 @@ import (
 	"time"
 
 	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/database/all"
+	allDB "github.com/FreifunkBremen/yanic/database/all"
-	"github.com/FreifunkBremen/yanic/meshviewer"
+	"github.com/FreifunkBremen/yanic/output"
+	allOutput "github.com/FreifunkBremen/yanic/output/all"
 	"github.com/FreifunkBremen/yanic/respond"
 	"github.com/FreifunkBremen/yanic/rrd"
 	"github.com/FreifunkBremen/yanic/runtime"
@@ -43,7 +44,7 @@ func main() {
 		panic(err)
 	}
 
-	connections, err = all.Connect(config.Database.Connection)
+	connections, err = allDB.Connect(config.Database.Connection)
 	if err != nil {
 		panic(err)
 	}
@@ -57,7 +58,14 @@ func main() {
 
 	nodes = runtime.NewNodes(config)
 	nodes.Start()
-	meshviewer.Start(config, nodes)
+
+	outputs, err := allOutput.Register(nodes, config.Nodes.Output)
+	if err != nil {
+		panic(err)
+	}
+
+	output.Start(outputs, config)
+	defer output.Close()
 
 	if config.Webserver.Enable {
 		log.Println("starting webserver on", config.Webserver.Bind)

config_example.toml

@@ -37,7 +37,8 @@ offline_after = "10m"
 prune_after = "7d"
 
 
-[meshviewer]
+[[nodes.output.meshviewer]]
+enable   = true
 # structur of nodes.json, which to support
 # version 1 is to support legacy meshviewer (which are in master branch)
 #    i.e. https://github.com/ffnord/meshviewer/tree/master
@@ -50,6 +51,23 @@ nodes_path     = "/var/www/html/meshviewer/data/nodes.json"
 # path where to store graph.json
 graph_path     = "/var/www/html/meshviewer/data/graph.json"
 
+[nodes.output.meshviewer.filter]
+# no_owner = true
+has_location = true
+blacklist = ["vpnid"]
+
+[nodes.output.meshviewer.filter.in_area]
+latitude_min = 34.30
+latitude_max = 71.85
+longitude_min = -24.96
+longitude_max = 39.72
+
+[[nodes.output.template]]
+enable = false
+template_path = "/var/lib/collector/html-template.tmp"
+output_path = "/var/www/html/index.html"
+
+
 [database]
 # cleaning data of measurement node,
 #   which are older than 7d

contrib/example-template.tmpl

@@ -0,0 +1,16 @@
+function ffhbCurrentStats(data) {
+	$("#freifunk").html("
+	<h1><a href="https://bremen.freifunk.net/" style="color: #444;">bremen.freifunk.net</a></h1>
+	<p>
+ 		Nutzer: <span id="freifunk_clients">0</span><br>
+		<i style="font-style: italic;">(auf  <span id="freifunk_nodes">0</span> Geräte verteilt)</i>
+	</p>
+	<p style="text-align: right;">
+		<a href="https://events.ffhb.de/meshviewer">mehr</a>
+	</p>");
+
+	$("#freifunk_clients").html(data.Clients);
+	$("#freifunk_nodes").html(data.Nodes);
+};
+
+ffhbCurrentStats({{json .GlobalStatistic}});

data/nodeinfo.go

@@ -4,7 +4,7 @@ package data
 type NodeInfo struct {
 	NodeID   string    `json:"node_id"`
 	Network  Network   `json:"network"`
-	Owner    *Owner    `json:"-"` // Removed for privacy reasons
+	Owner    *Owner    `json:"owner"`
 	System   System    `json:"system"`
 	Hostname string    `json:"hostname"`
 	Location *Location `json:"location,omitempty"`

meshviewer/nodes.go

@@ -1,50 +0,0 @@
-package meshviewer
-
-import (
-	"log"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-type nodeBuilder func(*runtime.Nodes) interface{}
-
-var nodeFormats = map[int]nodeBuilder{
-	1: BuildNodesV1,
-	2: BuildNodesV2,
-}
-
-// Start all services to manage Nodes
-func Start(config *runtime.Config, nodes *runtime.Nodes) {
-	go worker(config, nodes)
-}
-
-// Periodically saves the cached DB to json file
-func worker(config *runtime.Config, nodes *runtime.Nodes) {
-	c := time.Tick(config.Nodes.SaveInterval.Duration)
-
-	for range c {
-		saveMeshviewer(config, nodes)
-	}
-}
-
-func saveMeshviewer(config *runtime.Config, nodes *runtime.Nodes) {
-	// Locking foo
-	nodes.RLock()
-	defer nodes.RUnlock()
-	if path := config.Meshviewer.NodesPath; path != "" {
-		version := config.Meshviewer.Version
-		builder := nodeFormats[version]
-
-		if builder != nil {
-			runtime.SaveJSON(builder(nodes), path)
-		} else {
-			log.Panicf("invalid nodes version: %d", version)
-		}
-
-	}
-
-	if path := config.Meshviewer.GraphPath; path != "" {
-		runtime.SaveJSON(BuildGraph(nodes), path)
-	}
-}

output/all/internal.go

@@ -0,0 +1,37 @@
+package all
+
+import (
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	nodes *runtime.Nodes
+	list  []output.Output
+}
+
+func Register(nodes *runtime.Nodes, configuration interface{}) (output.Output, error) {
+	var list []output.Output
+	allOutputs := configuration.(map[string][]interface{})
+	for outputType, outputRegister := range output.Adapters {
+		outputConfigs := allOutputs[outputType]
+		for _, config := range outputConfigs {
+			output, err := outputRegister(nodes, config)
+			if err != nil {
+				return nil, err
+			}
+			if output == nil {
+				continue
+			}
+			list = append(list, output)
+		}
+	}
+	return &Output{list: list, nodes: nodes}, nil
+}
+
+func (o *Output) Save() {
+	for _, item := range o.list {
+		item.Save()
+	}
+}

output/all/main.go

@@ -0,0 +1,6 @@
+package all
+
+import (
+	_ "github.com/FreifunkBremen/yanic/output/meshviewer"
+	_ "github.com/FreifunkBremen/yanic/output/template"
+)

output/internal.go

@@ -0,0 +1,37 @@
+package output
+
+import (
+	"time"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+var quit chan struct{}
+
+// Start workers of database
+// WARNING: Do not override this function
+//  you should use New()
+func Start(output Output, config *runtime.Config) {
+	quit = make(chan struct{})
+	go saveWorker(output, config.Nodes.SaveInterval.Duration)
+}
+
+func Close() {
+	if quit != nil {
+		close(quit)
+	}
+}
+
+// save periodically to output
+func saveWorker(output Output, saveInterval time.Duration) {
+	ticker := time.NewTicker(saveInterval)
+	for {
+		select {
+		case <-ticker.C:
+			output.Save()
+		case <-quit:
+			ticker.Stop()
+			return
+		}
+	}
+}

output/meshviewer/filter.go

@@ -0,0 +1,157 @@
+package meshviewer
+
+import (
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type filter func(node *runtime.Node) *runtime.Node
+
+// Config Filter
+type filterConfig map[string]interface{}
+
+func (f filterConfig) Blacklist() *map[string]interface{} {
+	if v, ok := f["blacklist"]; ok {
+		list := make(map[string]interface{})
+		for _, nodeid := range v.([]interface{}) {
+			list[nodeid.(string)] = true
+		}
+		return &list
+	}
+	return nil
+}
+
+func (f filterConfig) NoOwner() bool {
+	if v, ok := f["no_owner"]; ok {
+		return v.(bool)
+	}
+	return true
+}
+func (f filterConfig) HasLocation() *bool {
+	if v, ok := f["has_location"].(bool); ok {
+		return &v
+	}
+	return nil
+}
+
+type area struct {
+	xA float64
+	xB float64
+	yA float64
+	yB float64
+}
+
+func (f filterConfig) InArea() *area {
+	if areaConfigInt, ok := f["in_area"]; ok {
+		areaConfig := areaConfigInt.(map[string]interface{})
+		a := area{}
+		if v, ok := areaConfig["latitude_min"]; ok {
+			a.xA = v.(float64)
+		}
+		if v, ok := areaConfig["latitude_max"]; ok {
+			a.xB = v.(float64)
+		}
+		if v, ok := areaConfig["longitude_min"]; ok {
+			a.yA = v.(float64)
+		}
+		if v, ok := areaConfig["longitude_max"]; ok {
+			a.yB = v.(float64)
+		}
+		return &a
+	}
+	return nil
+}
+
+// Create Filter
+func createFilter(config filterConfig) filter {
+	return func(n *runtime.Node) *runtime.Node {
+		//maybe cloning of this object is better?
+		node := n
+
+		if config.NoOwner() {
+			node = filterNoOwner(node)
+		}
+		if ok := config.HasLocation(); ok != nil {
+			node = filterHasLocation(node, *ok)
+		}
+		if area := config.InArea(); area != nil {
+			node = filterLocationInArea(node, *area)
+		}
+		if list := config.Blacklist(); list != nil {
+			node = filterBlacklist(node, *list)
+		}
+
+		return node
+	}
+}
+
+func filterBlacklist(node *runtime.Node, list map[string]interface{}) *runtime.Node {
+	if node != nil {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			if _, ok := list[nodeinfo.NodeID]; !ok {
+				return node
+			}
+		}
+	}
+	return nil
+}
+
+func filterNoOwner(node *runtime.Node) *runtime.Node {
+	if node == nil {
+		return nil
+	}
+	return &runtime.Node{
+		Address:    node.Address,
+		Firstseen:  node.Firstseen,
+		Lastseen:   node.Lastseen,
+		Online:     node.Online,
+		Statistics: node.Statistics,
+		Nodeinfo: &data.NodeInfo{
+			NodeID:   node.Nodeinfo.NodeID,
+			Network:  node.Nodeinfo.Network,
+			System:   node.Nodeinfo.System,
+			Owner:    nil,
+			Hostname: node.Nodeinfo.Hostname,
+			Location: node.Nodeinfo.Location,
+			Software: node.Nodeinfo.Software,
+			Hardware: node.Nodeinfo.Hardware,
+			VPN:      node.Nodeinfo.VPN,
+			Wireless: node.Nodeinfo.Wireless,
+		},
+		Neighbours: node.Neighbours,
+	}
+}
+
+func filterHasLocation(node *runtime.Node, withLocation bool) *runtime.Node {
+	if node != nil {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			if withLocation {
+				if location := nodeinfo.Location; location != nil {
+					return node
+				}
+			} else {
+				if location := nodeinfo.Location; location == nil {
+					return node
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func filterLocationInArea(node *runtime.Node, a area) *runtime.Node {
+	if node != nil {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			if location := nodeinfo.Location; location != nil {
+				if location.Latitude >= a.xA && location.Latitude <= a.xB {
+					if location.Longtitude >= a.yA && location.Longtitude <= a.yB {
+						return node
+					}
+				}
+			} else {
+				return node
+			}
+		}
+	}
+	return nil
+}

output/meshviewer/graph_test.go

@@ -58,7 +58,7 @@ func testGetNodeByFile(filename string) *runtime.Node {
 }
 
 func testfile(name string, obj interface{}) {
-	file, err := ioutil.ReadFile("../runtime/testdata/" + name)
+	file, err := ioutil.ReadFile("../../runtime/testdata/" + name)
 	if err != nil {
 		panic(err)
 	}

output/meshviewer/nodes_test.go

@@ -10,13 +10,13 @@ import (
 )
 
 func TestNodesV1(t *testing.T) {
-	nodes := BuildNodesV1(createTestNodes()).(*NodesV1)
+	nodes := BuildNodesV1(func(n *runtime.Node) *runtime.Node { return n }, createTestNodes()).(*NodesV1)
 
 	assert := assert.New(t)
 	assert.Len(nodes.List, 2)
 }
 func TestNodesV2(t *testing.T) {
-	nodes := BuildNodesV2(createTestNodes()).(*NodesV2)
+	nodes := BuildNodesV2(func(n *runtime.Node) *runtime.Node { return n }, createTestNodes()).(*NodesV2)
 
 	assert := assert.New(t)
 	assert.Len(nodes.List, 2)

output/meshviewer/nodes_v1.go

@@ -14,7 +14,7 @@ type NodesV1 struct {
 }
 
 // BuildNodesV1 transforms data to legacy meshviewer
-func BuildNodesV1(nodes *runtime.Nodes) interface{} {
+func BuildNodesV1(toFilter filter, nodes *runtime.Nodes) interface{} {
 	meshviewerNodes := &NodesV1{
 		Version:   1,
 		List:      make(map[string]*Node),
@@ -23,21 +23,21 @@ func BuildNodesV1(nodes *runtime.Nodes) interface{} {
 
 	for nodeID := range nodes.List {
 		nodeOrigin := nodes.List[nodeID]
-
+		nodeFiltere := toFilter(nodeOrigin)
-		if nodeOrigin.Statistics == nil {
+		if nodeOrigin.Statistics == nil || nodeFiltere == nil {
 			continue
 		}
 
 		node := &Node{
-			Firstseen: nodeOrigin.Firstseen,
+			Firstseen: nodeFiltere.Firstseen,
-			Lastseen:  nodeOrigin.Lastseen,
+			Lastseen:  nodeFiltere.Lastseen,
 			Flags: Flags{
-				Online:  nodeOrigin.Online,
+				Online:  nodeFiltere.Online,
-				Gateway: nodeOrigin.IsGateway(),
+				Gateway: nodeFiltere.IsGateway(),
 			},
-			Nodeinfo: nodeOrigin.Nodeinfo,
+			Nodeinfo: nodeFiltere.Nodeinfo,
 		}
-		node.Statistics = NewStatistics(nodeOrigin.Statistics)
+		node.Statistics = NewStatistics(nodeFiltere.Statistics)
 		meshviewerNodes.List[nodeID] = node
 	}
 	return meshviewerNodes

output/meshviewer/nodes_v2.go

@@ -14,7 +14,7 @@ type NodesV2 struct {
 }
 
 // BuildNodesV2 transforms data to modern meshviewers
-func BuildNodesV2(nodes *runtime.Nodes) interface{} {
+func BuildNodesV2(toFilter filter, nodes *runtime.Nodes) interface{} {
 	meshviewerNodes := &NodesV2{
 		Version:   2,
 		Timestamp: jsontime.Now(),
@@ -22,19 +22,20 @@ func BuildNodesV2(nodes *runtime.Nodes) interface{} {
 
 	for nodeID := range nodes.List {
 		nodeOrigin := nodes.List[nodeID]
-		if nodeOrigin.Statistics == nil {
+		nodeFiltere := toFilter(nodeOrigin)
+		if nodeOrigin.Statistics == nil || nodeFiltere == nil {
 			continue
 		}
 		node := &Node{
-			Firstseen: nodeOrigin.Firstseen,
+			Firstseen: nodeFiltere.Firstseen,
-			Lastseen:  nodeOrigin.Lastseen,
+			Lastseen:  nodeFiltere.Lastseen,
 			Flags: Flags{
-				Online:  nodeOrigin.Online,
+				Online:  nodeFiltere.Online,
-				Gateway: nodeOrigin.IsGateway(),
+				Gateway: nodeFiltere.IsGateway(),
 			},
-			Nodeinfo: nodeOrigin.Nodeinfo,
+			Nodeinfo: nodeFiltere.Nodeinfo,
 		}
-		node.Statistics = NewStatistics(nodeOrigin.Statistics)
+		node.Statistics = NewStatistics(nodeFiltere.Statistics)
 		meshviewerNodes.List = append(meshviewerNodes.List, node)
 	}
 	return meshviewerNodes

output/meshviewer/output.go

@@ -0,0 +1,88 @@
+package meshviewer
+
+import (
+	"log"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	config  Config
+	nodes   *runtime.Nodes
+	builder nodeBuilder
+	filter  filter
+}
+
+type Config map[string]interface{}
+
+func (c Config) Enable() bool {
+	return c["enable"].(bool)
+}
+
+func (c Config) Version() int64 {
+	return c["version"].(int64)
+}
+func (c Config) NodesPath() string {
+	if c["nodes_path"] == nil {
+		log.Panic("in configuration of [[nodes.output.meshviewer]] was no nodes_path defined", c)
+	}
+	return c["nodes_path"].(string)
+}
+func (c Config) GraphPath() string {
+	return c["graph_path"].(string)
+}
+
+func (c Config) FilterOption() filterConfig {
+	if v, ok := c["filter"]; ok {
+		var filterMap filterConfig
+		filterMap = v.(map[string]interface{})
+		return filterMap
+	}
+	return nil
+}
+
+type nodeBuilder func(filter, *runtime.Nodes) interface{}
+
+var nodeFormats = map[int64]nodeBuilder{
+	1: BuildNodesV1,
+	2: BuildNodesV2,
+}
+
+func init() {
+	output.RegisterAdapter("meshviewer", Register)
+}
+
+func Register(nodes *runtime.Nodes, configuration interface{}) (output.Output, error) {
+	var config Config
+	config = configuration.(map[string]interface{})
+	if !config.Enable() {
+		return nil, nil
+	}
+
+	builder := nodeFormats[config.Version()]
+	if builder == nil {
+		log.Panicf("invalid nodes version: %d", config.Version())
+	}
+
+	return &Output{
+		nodes:   nodes,
+		config:  config,
+		builder: builder,
+		filter:  createFilter(config.FilterOption()),
+	}, nil
+}
+
+func (o *Output) Save() {
+	o.nodes.RLock()
+	defer o.nodes.RUnlock()
+
+	if path := o.config.NodesPath(); path != "" {
+		runtime.SaveJSON(o.builder(o.filter, o.nodes), path)
+	}
+
+	if path := o.config.GraphPath(); path != "" {
+		runtime.SaveJSON(BuildGraph(o.nodes), path)
+	}
+}

output/output.go

@@ -0,0 +1,19 @@
+package output
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+// Output interface to use for implementation in e.g. influxdb
+type Output interface {
+	// InsertNode stores statistics per node
+	Save()
+}
+
+// Register function with config to get a output interface
+type Register func(nodes *runtime.Nodes, config interface{}) (Output, error)
+
+// Adapters is the list of registered output adapters
+var Adapters = map[string]Register{}
+
+func RegisterAdapter(name string, n Register) {
+	Adapters[name] = n
+}

output/template/main.go

@@ -0,0 +1,85 @@
+package template
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"log"
+	"os"
+	goTemplate "text/template"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	config   Config
+	nodes    *runtime.Nodes
+	template *goTemplate.Template
+}
+
+type Config map[string]interface{}
+
+func (c Config) Enable() bool {
+	return c["enable"].(bool)
+}
+
+func (c Config) TemplatePath() string {
+	return c["template_path"].(string)
+}
+func (c Config) ResultPath() string {
+	return c["result_path"].(string)
+}
+
+func init() {
+	output.RegisterAdapter("template", Register)
+}
+
+func Register(nodes *runtime.Nodes, configuration interface{}) (output.Output, error) {
+	var config Config
+	config = configuration.(map[string]interface{})
+	if !config.Enable() {
+		return nil, nil
+	}
+	t := goTemplate.New("some")
+	t = t.Funcs(goTemplate.FuncMap{"json": func(v interface{}) string {
+		a, _ := json.Marshal(v)
+		return string(a)
+	}})
+	buf := bytes.NewBuffer(nil)
+	f, err := os.Open(config.TemplatePath()) // Error handling elided for brevity.
+	if err != nil {
+		log.Panic(err)
+	}
+	io.Copy(buf, f) // Error handling elided for brevity.
+	f.Close()
+
+	s := string(buf.Bytes())
+	t.Parse(s)
+	return &Output{
+		config:   config,
+		nodes:    nodes,
+		template: t,
+	}, nil
+}
+
+func (o *Output) Save() {
+	stats := runtime.NewGlobalStats(o.nodes)
+	if stats == nil {
+		log.Panic("update of [output.template] not possible invalid data for the template generated")
+	}
+	tmpFile := o.config.ResultPath() + ".tmp"
+	f, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		log.Panic(err)
+	}
+	o.template.Execute(f, map[string]interface{}{"GlobalStatistic": stats})
+	if err != nil {
+		log.Panic(err)
+	}
+	f.Close()
+	if err := os.Rename(tmpFile, o.config.ResultPath()); err != nil {
+		log.Panic(err)
+	}
+}

runtime/config.go

@@ -26,6 +26,7 @@ type Config struct {
 		SaveInterval Duration `toml:"save_interval"` // Save nodes periodically
 		OfflineAfter Duration `toml:"offline_after"` // Set node to offline if not seen within this period
 		PruneAfter   Duration `toml:"prune_after"`   // Remove nodes after n days of inactivity
+		Output       map[string][]interface{}
 	}
 	Meshviewer struct {
 		Version   int    `toml:"version"`

runtime/config_test.go

@@ -18,13 +18,17 @@ func TestReadConfig(t *testing.T) {
 	assert.Equal("eth0", config.Respondd.Interface)
 	assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
 
-	assert.Equal(2, config.Meshviewer.Version)
-	assert.Equal("/var/www/html/meshviewer/data/nodes.json", config.Meshviewer.NodesPath)
-
 	assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
 
 	assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration)
 
+	var meshviewer map[string]interface{}
+	outputs := config.Nodes.Output["meshviewer"]
+	assert.Len(outputs, 1, "more outputs are given")
+	meshviewer = outputs[0].(map[string]interface{})
+	assert.Equal(int64(2), meshviewer["version"])
+	assert.Equal("/var/www/html/meshviewer/data/nodes.json", meshviewer["nodes_path"])
+
 	var influxdb map[string]interface{}
 	dbs := config.Database.Connection["influxdb"]
 	assert.Len(dbs, 1, "more influxdb are given")

runtime/stats.go

@@ -23,7 +23,7 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
 		Models:    make(CounterMap),
 	}
 
-	nodes.Lock()
+	nodes.RLock()
 	for _, node := range nodes.List {
 		if node.Online {
 			result.Nodes++
@@ -42,7 +42,7 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
 			}
 		}
 	}
-	nodes.Unlock()
+	nodes.RUnlock()
 	return
 }