diff --git a/INSTALL.md b/INSTALL.md index a7547cb..b071db2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -59,7 +59,7 @@ accessible under `/var/www/html/meshviewer`. #### With webserver (Apache, nginx) The meshviewer needs the output files like `nodes_path` and `graph_path` inside the same directory as the `dataPath`. Change the path in the section -`[meshviewer]` accordingly. +`[[nodes.output.meshviewer]]` accordingly. ### Service ```bash diff --git a/cmd/serve.go b/cmd/serve.go index 82341ae..60d33a9 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -8,8 +8,9 @@ import ( "time" "github.com/FreifunkBremen/yanic/database" - "github.com/FreifunkBremen/yanic/database/all" - "github.com/FreifunkBremen/yanic/meshviewer" + allDatabase "github.com/FreifunkBremen/yanic/database/all" + "github.com/FreifunkBremen/yanic/output" + allOutput "github.com/FreifunkBremen/yanic/output/all" "github.com/FreifunkBremen/yanic/respond" "github.com/FreifunkBremen/yanic/runtime" "github.com/FreifunkBremen/yanic/webserver" @@ -24,7 +25,7 @@ var serveCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { config := loadConfig() - connections, err := all.Connect(config.Database.Connection) + connections, err := allDatabase.Connect(config.Database.Connection) if err != nil { panic(err) } @@ -33,7 +34,13 @@ var serveCmd = &cobra.Command{ nodes = runtime.NewNodes(config) nodes.Start() - meshviewer.Start(config, nodes) + + outputs, err := allOutput.Register(config.Nodes.Output) + if err != nil { + panic(err) + } + output.Start(outputs, nodes, config) + defer output.Close() if config.Webserver.Enable { log.Println("starting webserver on", config.Webserver.Bind) diff --git a/config_example.toml b/config_example.toml index 759bea6..60027df 100644 --- a/config_example.toml +++ b/config_example.toml @@ -34,7 +34,17 @@ save_interval = "5s" offline_after = "10m" -[meshviewer] +## [[nodes.output.-]] +# every output: +# needs to be enabled just adding: +# enable = true +# could filter the nodes by using a there filter entry (see output meshviewer) +# [nodes.output.-.filter] +# could be used multiple times (suggested by the "[[...]]" instatt of "[...]") +# it is useful for e.g. filter by different array and use multiple meshviewers + +[[nodes.output.meshviewer]] +enable = true # The structure version of the output which should be generated (i.e. nodes.json) # version 1 is accepted by the legacy meshviewer (which is the master branch) # i.e. https://github.com/ffnord/meshviewer/tree/master @@ -48,6 +58,26 @@ nodes_path = "/var/www/html/meshviewer/data/nodes.json" graph_path = "/var/www/html/meshviewer/data/graph.json" +[nodes.output.meshviewer.filter] +# no_owner = true +has_location = true +blacklist = ["vpnid"] + +[nodes.output.meshviewer.filter.in_area] +latitude_min = 34.30 +latitude_max = 71.85 +longitude_min = -24.96 +longitude_max = 39.72 + +[[nodes.output.nodelist]] +enable = true +path = "/var/www/html/meshviewer/data/nodelist.json" + +[[nodes.output.meshviewer-ffrgb]] +enable = true +path = "/var/www/html/meshviewer/data/meshviewer.json" + + [database] # this will send delete commands to the database to prune data # which is older than: @@ -55,6 +85,13 @@ delete_after = "7d" # how often run the cleaning delete_interval = "1h" +## [[database.connection.-]] +# every output: +# needs to be enabled just adding: +# enable = true +# could be used multiple times (suggested by the "[[...]]" instatt of "[...]") +# it is useful for e.g. save into a database before and behind a firewall + # Save collected data to InfluxDB. # There are the following measurments: # node: store node specific data i.e. clients memory, airtime diff --git a/data/nodeinfo.go b/data/nodeinfo.go index 204b044..2b0099a 100644 --- a/data/nodeinfo.go +++ b/data/nodeinfo.go @@ -4,7 +4,7 @@ package data type NodeInfo struct { NodeID string `json:"node_id"` Network Network `json:"network"` - Owner *Owner `json:"-"` // Removed for privacy reasons + Owner *Owner `json:"owner"` System System `json:"system"` Hostname string `json:"hostname"` Location *Location `json:"location,omitempty"` diff --git a/database/influxdb/global_test.go b/database/influxdb/global_test.go index 946c134..f1ced75 100644 --- a/database/influxdb/global_test.go +++ b/database/influxdb/global_test.go @@ -22,37 +22,43 @@ func TestGlobalStats(t *testing.T) { func createTestNodes() *runtime.Nodes { nodes := runtime.NewNodes(&runtime.Config{}) - nodeData := &data.ResponseData{ + nodeData := &runtime.Node{ + Online: true, Statistics: &data.Statistics{ Clients: data.Clients{ Total: 23, }, }, - NodeInfo: &data.NodeInfo{ + Nodeinfo: &data.NodeInfo{ + NodeID: "abcdef012345", Hardware: data.Hardware{ Model: "TP-Link 841", }, }, } - nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1" - nodes.Update("abcdef012345", nodeData) + nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1" + nodes.AddNode(nodeData) - nodes.Update("112233445566", &data.ResponseData{ + nodes.AddNode(&runtime.Node{ + Online: true, Statistics: &data.Statistics{ Clients: data.Clients{ Total: 2, }, }, - NodeInfo: &data.NodeInfo{ + Nodeinfo: &data.NodeInfo{ + NodeID: "112233445566", Hardware: data.Hardware{ Model: "TP-Link 841", }, }, }) - nodes.Update("0xdeadbeef0x", &data.ResponseData{ - NodeInfo: &data.NodeInfo{ - VPN: true, + nodes.AddNode(&runtime.Node{ + Online: true, + Nodeinfo: &data.NodeInfo{ + NodeID: "0xdeadbeef0x", + VPN: true, Hardware: data.Hardware{ Model: "Xeon Multi-Core", }, diff --git a/database/influxdb/node_test.go b/database/influxdb/node_test.go index 6b594a3..6bda061 100644 --- a/database/influxdb/node_test.go +++ b/database/influxdb/node_test.go @@ -144,7 +144,7 @@ func testPoints(nodes ...*runtime.Node) (points []*client.Point) { } for _, node := range nodes { - nodesList.Update(node.Nodeinfo.NodeID, &data.ResponseData{NodeInfo: node.Nodeinfo}) + nodesList.AddNode(node) } // Process data diff --git a/meshviewer/nodes.go b/meshviewer/nodes.go deleted file mode 100644 index bbb09cb..0000000 --- a/meshviewer/nodes.go +++ /dev/null @@ -1,50 +0,0 @@ -package meshviewer - -import ( - "log" - "time" - - "github.com/FreifunkBremen/yanic/runtime" -) - -type nodeBuilder func(*runtime.Nodes) interface{} - -var nodeFormats = map[int]nodeBuilder{ - 1: BuildNodesV1, - 2: BuildNodesV2, -} - -// Start all services to manage Nodes -func Start(config *runtime.Config, nodes *runtime.Nodes) { - go worker(config, nodes) -} - -// Periodically saves the cached DB to json file -func worker(config *runtime.Config, nodes *runtime.Nodes) { - c := time.Tick(config.Nodes.SaveInterval.Duration) - - for range c { - saveMeshviewer(config, nodes) - } -} - -func saveMeshviewer(config *runtime.Config, nodes *runtime.Nodes) { - // Locking foo - nodes.RLock() - defer nodes.RUnlock() - if path := config.Meshviewer.NodesPath; path != "" { - version := config.Meshviewer.Version - builder := nodeFormats[version] - - if builder != nil { - runtime.SaveJSON(builder(nodes), path) - } else { - log.Panicf("invalid nodes version: %d", version) - } - - } - - if path := config.Meshviewer.GraphPath; path != "" { - runtime.SaveJSON(BuildGraph(nodes), path) - } -} diff --git a/output/all/filter.go b/output/all/filter.go new file mode 100644 index 0000000..d32eb58 --- /dev/null +++ b/output/all/filter.go @@ -0,0 +1,39 @@ +package all + +import "github.com/FreifunkBremen/yanic/runtime" + +// Config Filter +type filterConfig map[string]interface{} + +type filterFunc func(*runtime.Node) *runtime.Node + +func noFilter(node *runtime.Node) *runtime.Node { + return node +} + +// Create Filter +func (f filterConfig) filtering(nodesOrigin *runtime.Nodes) *runtime.Nodes { + nodes := runtime.NewNodes(&runtime.Config{}) + filterfuncs := []filterFunc{ + f.HasLocation(), + f.Blacklist(), + f.InArea(), + f.NoOwner(), + } + + for _, nodeOrigin := range nodesOrigin.List { + //maybe cloning of this object is better? + node := nodeOrigin + for _, f := range filterfuncs { + node = f(node) + if node == nil { + break + } + } + + if node != nil { + nodes.AddNode(node) + } + } + return nodes +} diff --git a/output/all/filter_blacklist.go b/output/all/filter_blacklist.go new file mode 100644 index 0000000..cd1c0b7 --- /dev/null +++ b/output/all/filter_blacklist.go @@ -0,0 +1,24 @@ +package all + +import "github.com/FreifunkBremen/yanic/runtime" + +func (f filterConfig) Blacklist() filterFunc { + v, ok := f["blacklist"] + if !ok { + return noFilter + } + + list := make(map[string]interface{}) + for _, nodeid := range v.([]interface{}) { + list[nodeid.(string)] = true + } + + return func(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + if _, ok := list[nodeinfo.NodeID]; ok { + return nil + } + } + return node + } +} diff --git a/output/all/filter_blacklist_test.go b/output/all/filter_blacklist_test.go new file mode 100644 index 0000000..c7c73d9 --- /dev/null +++ b/output/all/filter_blacklist_test.go @@ -0,0 +1,34 @@ +package all + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterBlacklist(t *testing.T) { + assert := assert.New(t) + var config filterConfig + + config = map[string]interface{}{} + + filterBlacklist := config.Blacklist() + + n := filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + config["blacklist"] = []interface{}{"a", "c"} + filterBlacklist = config.Blacklist() + + n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{NodeID: "a"}}) + assert.Nil(n) + + n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + n = filterBlacklist(&runtime.Node{}) + assert.NotNil(n) + +} diff --git a/output/all/filter_haslocation.go b/output/all/filter_haslocation.go new file mode 100644 index 0000000..2f3580c --- /dev/null +++ b/output/all/filter_haslocation.go @@ -0,0 +1,26 @@ +package all + +import "github.com/FreifunkBremen/yanic/runtime" + +func (f filterConfig) HasLocation() filterFunc { + withLocation, ok := f["has_location"].(bool) + if !ok { + return noFilter + } + return func(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + if withLocation { + if location := nodeinfo.Location; location != nil { + return node + } + } else { + if location := nodeinfo.Location; location == nil { + return node + } + } + } else if !withLocation { + return node + } + return nil + } +} diff --git a/output/all/filter_haslocation_test.go b/output/all/filter_haslocation_test.go new file mode 100644 index 0000000..61e6b44 --- /dev/null +++ b/output/all/filter_haslocation_test.go @@ -0,0 +1,50 @@ +package all + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterHasLocation(t *testing.T) { + assert := assert.New(t) + var config filterConfig + + config = map[string]interface{}{} + + filterHasLocation := config.HasLocation() + n := filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{}, + }}) + assert.NotNil(n) + + config["has_location"] = true + filterHasLocation = config.HasLocation() + + n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{}, + }}) + assert.NotNil(n) + + n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.Nil(n) + + n = filterHasLocation(&runtime.Node{}) + assert.Nil(n) + + config["has_location"] = false + filterHasLocation = config.HasLocation() + + n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{}, + }}) + assert.Nil(n) + + n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + n = filterHasLocation(&runtime.Node{}) + assert.NotNil(n) +} diff --git a/output/all/filter_inarea.go b/output/all/filter_inarea.go new file mode 100644 index 0000000..bcea6f4 --- /dev/null +++ b/output/all/filter_inarea.go @@ -0,0 +1,42 @@ +package all + +import "github.com/FreifunkBremen/yanic/runtime" + +type area struct { + latitudeMin float64 + latitudeMax float64 + longitudeMin float64 + longitudeMax float64 +} + +func (f filterConfig) InArea() filterFunc { + if areaConfigInt, ok := f["in_area"]; ok { + areaConfig := areaConfigInt.(map[string]interface{}) + a := area{} + if v, ok := areaConfig["latitude_min"]; ok { + a.latitudeMin = v.(float64) + } + if v, ok := areaConfig["latitude_max"]; ok { + a.latitudeMax = v.(float64) + } + if v, ok := areaConfig["longitude_min"]; ok { + a.longitudeMin = v.(float64) + } + if v, ok := areaConfig["longitude_max"]; ok { + a.longitudeMax = v.(float64) + } + return func(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + location := nodeinfo.Location + if location == nil { + return node + } + if location.Latitude >= a.latitudeMin && location.Latitude <= a.latitudeMax && location.Longtitude >= a.longitudeMin && location.Longtitude <= a.longitudeMax { + return node + } + } + return nil + } + } + return noFilter +} diff --git a/output/all/filter_inarea_test.go b/output/all/filter_inarea_test.go new file mode 100644 index 0000000..a9b28e3 --- /dev/null +++ b/output/all/filter_inarea_test.go @@ -0,0 +1,65 @@ +package all + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterInArea(t *testing.T) { + assert := assert.New(t) + var config filterConfig + areaConfig := map[string]interface{}{ + "latitude_min": 3.0, + "latitude_max": 5.0, + "longitude_min": 10.0, + "longitude_max": 12.0, + } + config = map[string]interface{}{} + + filterLocationInArea := config.InArea() + n := filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 4.0, Longtitude: 11.0}, + }}) + assert.NotNil(n) + + config["in_area"] = areaConfig + filterLocationInArea = config.InArea() + + // drop area without nodeinfo + n = filterLocationInArea(&runtime.Node{}) + assert.Nil(n) + + // keep without location + n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + // zeros not in area (0, 0) + n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{}, + }}) + assert.Nil(n) + + // in area + n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 4.0, Longtitude: 11.0}, + }}) + assert.NotNil(n) + + n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 4.0, Longtitude: 13.0}, + }}) + assert.Nil(n) + + n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 6.0, Longtitude: 11.0}, + }}) + assert.Nil(n) + + n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 1.0, Longtitude: 2.0}, + }}) + assert.Nil(n) +} diff --git a/output/all/filter_noowner.go b/output/all/filter_noowner.go new file mode 100644 index 0000000..76bc47a --- /dev/null +++ b/output/all/filter_noowner.go @@ -0,0 +1,37 @@ +package all + +import ( + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" +) + +func (f filterConfig) NoOwner() filterFunc { + if v, ok := f["no_owner"]; ok && v.(bool) == false { + return noFilter + } + return func(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + return &runtime.Node{ + Address: node.Address, + Firstseen: node.Firstseen, + Lastseen: node.Lastseen, + Online: node.Online, + Statistics: node.Statistics, + Nodeinfo: &data.NodeInfo{ + NodeID: nodeinfo.NodeID, + Network: nodeinfo.Network, + System: nodeinfo.System, + Owner: nil, + Hostname: nodeinfo.Hostname, + Location: nodeinfo.Location, + Software: nodeinfo.Software, + Hardware: nodeinfo.Hardware, + VPN: nodeinfo.VPN, + Wireless: nodeinfo.Wireless, + }, + Neighbours: node.Neighbours, + } + } + return node + } +} diff --git a/output/all/filter_noowner_test.go b/output/all/filter_noowner_test.go new file mode 100644 index 0000000..1aa496f --- /dev/null +++ b/output/all/filter_noowner_test.go @@ -0,0 +1,49 @@ +package all + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterNoOwner(t *testing.T) { + assert := assert.New(t) + var config filterConfig + + config = map[string]interface{}{} + + filterNoOwner := config.NoOwner() + n := filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Owner: &data.Owner{ + Contact: "blub", + }, + }}) + assert.NotNil(n) + assert.Nil(n.Nodeinfo.Owner) + + n = filterNoOwner(&runtime.Node{}) + assert.NotNil(n) + + config["no_owner"] = true + filterNoOwner = config.NoOwner() + n = filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Owner: &data.Owner{ + Contact: "blub", + }, + }}) + assert.NotNil(n) + assert.Nil(n.Nodeinfo.Owner) + + config["no_owner"] = false + filterNoOwner = config.NoOwner() + + n = filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Owner: &data.Owner{ + Contact: "blub", + }, + }}) + assert.NotNil(n) + assert.NotNil(n.Nodeinfo.Owner) +} diff --git a/output/all/filter_test.go b/output/all/filter_test.go new file mode 100644 index 0000000..9a55333 --- /dev/null +++ b/output/all/filter_test.go @@ -0,0 +1,41 @@ +package all + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilter(t *testing.T) { + assert := assert.New(t) + + // filtered - do not run all + nodes := &runtime.Nodes{ + List: map[string]*runtime.Node{ + "a": &runtime.Node{ + Nodeinfo: &data.NodeInfo{NodeID: "a"}, + }, + }, + } + config := filterConfig{ + "has_location": true, + } + nodes = config.filtering(nodes) + assert.Len(nodes.List, 0) + + // run to end + nodes = &runtime.Nodes{ + List: map[string]*runtime.Node{ + "a": &runtime.Node{ + Nodeinfo: &data.NodeInfo{NodeID: "a"}, + }, + }, + } + config = filterConfig{ + "has_location": false, + } + nodes = config.filtering(nodes) + assert.Len(nodes.List, 1) +} diff --git a/output/all/internal.go b/output/all/internal.go new file mode 100644 index 0000000..f871990 --- /dev/null +++ b/output/all/internal.go @@ -0,0 +1,63 @@ +package all + +import ( + "log" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + list map[int]output.Output + filter map[int]filterConfig +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + list := make(map[int]output.Output) + filter := make(map[int]filterConfig) + i := 1 + allOutputs := configuration + for outputType, outputRegister := range output.Adapters { + configForOutput := allOutputs[outputType] + if configForOutput == nil { + log.Printf("the output type '%s' has no configuration\n", outputType) + continue + } + outputConfigs, ok := configForOutput.([]map[string]interface{}) + if !ok { + log.Panicf("the output type '%s' has the wrong format\n", outputType) + } + for _, config := range outputConfigs { + if c, ok := config["enable"].(bool); ok && !c { + continue + } + output, err := outputRegister(config) + if err != nil { + return nil, err + } + if output == nil { + continue + } + list[i] = output + if c := config["filter"]; c != nil { + filter[i] = config["filter"].(map[string]interface{}) + } + i++ + } + } + return &Output{list: list, filter: filter}, nil +} + +func (o *Output) Save(nodes *runtime.Nodes) { + for i, item := range o.list { + var filteredNodes *runtime.Nodes + if config := o.filter[i]; config != nil { + filteredNodes = config.filtering(nodes) + } else { + filteredNodes = filterConfig{}.filtering(nodes) + } + + item.Save(filteredNodes) + } +} diff --git a/output/all/internal_test.go b/output/all/internal_test.go new file mode 100644 index 0000000..ad067a3 --- /dev/null +++ b/output/all/internal_test.go @@ -0,0 +1,89 @@ +package all + +import ( + "errors" + "testing" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +type testOutput struct { + output.Output + CountSave int +} + +func (c *testOutput) Save(nodes *runtime.Nodes) { + c.CountSave++ +} + +func TestStart(t *testing.T) { + assert := assert.New(t) + + nodes := &runtime.Nodes{} + + globalOutput := &testOutput{} + output.RegisterAdapter("a", func(config map[string]interface{}) (output.Output, error) { + return globalOutput, nil + }) + output.RegisterAdapter("b", func(config map[string]interface{}) (output.Output, error) { + return globalOutput, nil + }) + output.RegisterAdapter("c", func(config map[string]interface{}) (output.Output, error) { + return globalOutput, nil + }) + output.RegisterAdapter("d", func(config map[string]interface{}) (output.Output, error) { + return nil, nil + }) + output.RegisterAdapter("e", func(config map[string]interface{}) (output.Output, error) { + return nil, errors.New("blub") + }) + allOutput, err := Register(map[string]interface{}{ + "a": []map[string]interface{}{ + map[string]interface{}{ + "enable": false, + "path": "a1", + }, + map[string]interface{}{ + "path": "a2", + }, + map[string]interface{}{ + "enable": true, + "path": "a3", + }, + }, + "b": nil, + "c": []map[string]interface{}{ + map[string]interface{}{ + "path": "c1", + "filter": map[string]interface{}{}, + }, + }, + // fetch continue command in Connect + "d": []map[string]interface{}{ + map[string]interface{}{ + "path": "d0", + }, + }, + }) + assert.NoError(err) + + assert.Equal(0, globalOutput.CountSave) + allOutput.Save(nodes) + assert.Equal(3, globalOutput.CountSave) + + _, err = Register(map[string]interface{}{ + "e": []map[string]interface{}{ + map[string]interface{}{}, + }, + }) + assert.Error(err) + + // wrong format -> the only panic in Register + assert.Panics(func() { + Register(map[string]interface{}{ + "e": true, + }) + }) +} diff --git a/output/all/main.go b/output/all/main.go new file mode 100644 index 0000000..07c9dae --- /dev/null +++ b/output/all/main.go @@ -0,0 +1,7 @@ +package all + +import ( + _ "github.com/FreifunkBremen/yanic/output/meshviewer" + _ "github.com/FreifunkBremen/yanic/output/meshviewer-ffrgb" + _ "github.com/FreifunkBremen/yanic/output/nodelist" +) diff --git a/output/internal.go b/output/internal.go new file mode 100644 index 0000000..dcf462b --- /dev/null +++ b/output/internal.go @@ -0,0 +1,40 @@ +package output + +import ( + "sync" + "time" + + "github.com/FreifunkBremen/yanic/runtime" +) + +var quit chan struct{} +var wg = sync.WaitGroup{} + +// Start workers of database +// WARNING: Do not override this function +// you should use New() +func Start(output Output, nodes *runtime.Nodes, config *runtime.Config) { + quit = make(chan struct{}) + wg.Add(1) + go saveWorker(output, nodes, config.Nodes.SaveInterval.Duration) +} + +func Close() { + close(quit) + wg.Wait() +} + +// save periodically to output +func saveWorker(output Output, nodes *runtime.Nodes, saveInterval time.Duration) { + ticker := time.NewTicker(saveInterval) + for { + select { + case <-ticker.C: + output.Save(nodes) + case <-quit: + wg.Done() + ticker.Stop() + return + } + } +} diff --git a/output/internal_test.go b/output/internal_test.go new file mode 100644 index 0000000..9d9e9d0 --- /dev/null +++ b/output/internal_test.go @@ -0,0 +1,49 @@ +package output + +import ( + "testing" + "time" + + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +type testConn struct { + Output + CountSave int +} + +func (c *testConn) Save(nodes *runtime.Nodes) { + c.CountSave++ +} + +func TestStart(t *testing.T) { + assert := assert.New(t) + + conn := &testConn{} + config := &runtime.Config{ + Nodes: struct { + Enable bool `toml:"enable"` + StatePath string `toml:"state_path"` + SaveInterval runtime.Duration `toml:"save_interval"` + OfflineAfter runtime.Duration `toml:"offline_after"` + PruneAfter runtime.Duration `toml:"prune_after"` + Output map[string]interface{} + }{ + SaveInterval: runtime.Duration{Duration: time.Millisecond * 10}, + }, + } + assert.Nil(quit) + + Start(conn, nil, config) + assert.NotNil(quit) + + assert.Equal(0, conn.CountSave) + time.Sleep(time.Millisecond * 12) + assert.Equal(1, conn.CountSave) + + time.Sleep(time.Millisecond * 12) + Close() + assert.Equal(2, conn.CountSave) + +} diff --git a/output/meshviewer-ffrgb/meshviewer.go b/output/meshviewer-ffrgb/meshviewer.go new file mode 100644 index 0000000..668f0d8 --- /dev/null +++ b/output/meshviewer-ffrgb/meshviewer.go @@ -0,0 +1,86 @@ +package meshviewerFFRGB + +import ( + "fmt" + "strings" + + "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/runtime" +) + +func transform(nodes *runtime.Nodes) *Meshviewer { + + meshviewer := &Meshviewer{ + Timestamp: jsontime.Now(), + Nodes: make([]*Node, 0), + Links: make([]*Link, 0), + } + + links := make(map[string]*Link) + + nodes.RLock() + defer nodes.RUnlock() + + for _, nodeOrigin := range nodes.List { + node := NewNode(nodes, nodeOrigin) + meshviewer.Nodes = append(meshviewer.Nodes, node) + + typeList := make(map[string]string) + + if nodeinfo := nodeOrigin.Nodeinfo; nodeinfo != nil { + if meshes := nodeinfo.Network.Mesh; meshes != nil { + for _, mesh := range meshes { + for _, mac := range mesh.Interfaces.Wireless { + typeList[mac] = "wifi" + } + for _, mac := range mesh.Interfaces.Tunnel { + typeList[mac] = "vpn" + } + } + } + } + + for _, linkOrigin := range nodes.NodeLinks(nodeOrigin) { + var key string + // keep source and target in the same order + switchSourceTarget := strings.Compare(linkOrigin.SourceMAC, linkOrigin.TargetMAC) > 0 + if switchSourceTarget { + key = fmt.Sprintf("%s-%s", linkOrigin.SourceMAC, linkOrigin.TargetMAC) + } else { + key = fmt.Sprintf("%s-%s", linkOrigin.TargetMAC, linkOrigin.SourceMAC) + } + if link := links[key]; link != nil { + if switchSourceTarget { + link.TargetTQ = float32(linkOrigin.TQ) / 255.0 + } else { + link.SourceTQ = float32(linkOrigin.TQ) / 255.0 + } + continue + } + linkType := typeList[linkOrigin.SourceMAC] + if linkType == "" { + linkType = "other" + } + tq := float32(linkOrigin.TQ) / 255.0 + link := &Link{ + Type: linkType, + Source: linkOrigin.SourceID, + SourceMAC: linkOrigin.SourceMAC, + Target: linkOrigin.TargetID, + TargetMAC: linkOrigin.TargetMAC, + SourceTQ: tq, + TargetTQ: tq, + } + if switchSourceTarget { + link.Source = linkOrigin.TargetID + link.SourceMAC = linkOrigin.TargetMAC + link.Target = linkOrigin.SourceID + link.TargetMAC = linkOrigin.SourceMAC + } + links[key] = link + meshviewer.Links = append(meshviewer.Links, link) + } + } + + return meshviewer +} diff --git a/output/meshviewer-ffrgb/meshviewer_test.go b/output/meshviewer-ffrgb/meshviewer_test.go new file mode 100644 index 0000000..447ef58 --- /dev/null +++ b/output/meshviewer-ffrgb/meshviewer_test.go @@ -0,0 +1,144 @@ +package meshviewerFFRGB + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestTransform(t *testing.T) { + assert := assert.New(t) + + nodes := runtime.NewNodes(&runtime.Config{}) + nodes.AddNode(&runtime.Node{ + Nodeinfo: &data.NodeInfo{ + NodeID: "node_a", + Network: data.Network{ + Mac: "node:a:mac", + Mesh: map[string]*data.BatInterface{ + "bat0": &data.BatInterface{ + Interfaces: struct { + Wireless []string `json:"wireless,omitempty"` + Other []string `json:"other,omitempty"` + Tunnel []string `json:"tunnel,omitempty"` + }{ + Wireless: []string{"node:a:mac:wifi"}, + Tunnel: []string{"node:a:mac:vpn"}, + Other: []string{"node:a:mac:lan"}, + }, + }, + }, + }, + }, + Neighbours: &data.Neighbours{ + NodeID: "node_a", + Batadv: map[string]data.BatadvNeighbours{ + "node:a:mac:wifi": data.BatadvNeighbours{ + Neighbours: map[string]data.BatmanLink{ + "node:b:mac:wifi": data.BatmanLink{Tq: 153}, + }, + }, + "node:a:mac:lan": data.BatadvNeighbours{ + Neighbours: map[string]data.BatmanLink{ + "node:b:mac:lan": data.BatmanLink{Tq: 51}, + }, + }, + }, + }, + }) + nodes.AddNode(&runtime.Node{ + Nodeinfo: &data.NodeInfo{ + NodeID: "node_c", + Network: data.Network{ + Mac: "node:c:mac", + Mesh: map[string]*data.BatInterface{ + "bat0": &data.BatInterface{ + Interfaces: struct { + Wireless []string `json:"wireless,omitempty"` + Other []string `json:"other,omitempty"` + Tunnel []string `json:"tunnel,omitempty"` + }{ + Other: []string{"node:c:mac:lan"}, + }, + }, + }, + }, + }, + Neighbours: &data.Neighbours{ + NodeID: "node_b", + Batadv: map[string]data.BatadvNeighbours{ + "node:c:mac:lan": data.BatadvNeighbours{ + Neighbours: map[string]data.BatmanLink{ + "node:b:mac:lan": data.BatmanLink{Tq: 102}, + }, + }, + }, + }, + }) + nodes.AddNode(&runtime.Node{ + Nodeinfo: &data.NodeInfo{ + NodeID: "node_b", + Network: data.Network{ + Mac: "node:b:mac", + Mesh: map[string]*data.BatInterface{ + "bat0": &data.BatInterface{ + Interfaces: struct { + Wireless []string `json:"wireless,omitempty"` + Other []string `json:"other,omitempty"` + Tunnel []string `json:"tunnel,omitempty"` + }{ + Wireless: []string{"node:b:mac:wifi"}, + Other: []string{"node:b:mac:lan"}, + }, + }, + }, + }, + }, + Neighbours: &data.Neighbours{ + NodeID: "node_b", + Batadv: map[string]data.BatadvNeighbours{ + "node:b:mac:lan": data.BatadvNeighbours{ + Neighbours: map[string]data.BatmanLink{ + "node:c:mac:lan": data.BatmanLink{Tq: 204}, + }, + }, + "node:b:mac:wifi": data.BatadvNeighbours{ + Neighbours: map[string]data.BatmanLink{ + "node:a:mac:wifi": data.BatmanLink{Tq: 204}, + }, + }, + }, + }, + }) + + meshviewer := transform(nodes) + assert.NotNil(meshviewer) + assert.Len(meshviewer.Nodes, 3) + links := meshviewer.Links + assert.Len(links, 3) + + for _, link := range links { + switch link.SourceMAC { + case "node:a:mac:lan": + assert.Equal("other", link.Type) + assert.Equal("node:b:mac:lan", link.TargetMAC) + assert.Equal(float32(0.2), link.SourceTQ) + assert.Equal(float32(0.2), link.TargetTQ) + break + + case "node:a:mac:wifi": + assert.Equal("wifi", link.Type) + assert.Equal("node:b:mac:wifi", link.TargetMAC) + assert.Equal(float32(0.6), link.SourceTQ) + assert.Equal(float32(0.8), link.TargetTQ) + default: + assert.Equal("other", link.Type) + assert.Equal("node:c:mac:lan", link.TargetMAC) + assert.Equal(float32(0.8), link.SourceTQ) + assert.Equal(float32(0.4), link.TargetTQ) + break + } + } +} diff --git a/output/meshviewer-ffrgb/output.go b/output/meshviewer-ffrgb/output.go new file mode 100644 index 0000000..ad6ab43 --- /dev/null +++ b/output/meshviewer-ffrgb/output.go @@ -0,0 +1,43 @@ +package meshviewerFFRGB + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + path string +} + +type Config map[string]interface{} + +func (c Config) Path() string { + if path, ok := c["path"]; ok { + return path.(string) + } + return "" +} + +func init() { + output.RegisterAdapter("meshviewer-ffrgb", Register) +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + var config Config + config = configuration + + if path := config.Path(); path != "" { + return &Output{ + path: path, + }, nil + } + return nil, errors.New("no path given") + +} + +func (o *Output) Save(nodes *runtime.Nodes) { + runtime.SaveJSON(transform(nodes), o.path) +} diff --git a/output/meshviewer-ffrgb/output_test.go b/output/meshviewer-ffrgb/output_test.go new file mode 100644 index 0000000..082a585 --- /dev/null +++ b/output/meshviewer-ffrgb/output_test.go @@ -0,0 +1,27 @@ +package meshviewerFFRGB + +import ( + "os" + "testing" + + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestOutput(t *testing.T) { + assert := assert.New(t) + out, err := Register(map[string]interface{}{}) + assert.Error(err) + assert.Nil(out) + + out, err = Register(map[string]interface{}{ + "path": "/tmp/meshviewer.json", + }) + os.Remove("/tmp/meshviewer.json") + assert.NoError(err) + assert.NotNil(out) + + out.Save(&runtime.Nodes{}) + _, err = os.Stat("/tmp/meshviewer.json") + assert.NoError(err) +} diff --git a/output/meshviewer-ffrgb/struct.go b/output/meshviewer-ffrgb/struct.go new file mode 100644 index 0000000..8c881c2 --- /dev/null +++ b/output/meshviewer-ffrgb/struct.go @@ -0,0 +1,153 @@ +package meshviewerFFRGB + +import ( + "time" + + "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Meshviewer struct { + Timestamp jsontime.Time `json:"timestamp"` + Nodes []*Node `json:"nodes"` + Links []*Link `json:"links"` +} + +type Node struct { + Firstseen jsontime.Time `json:"firstseen"` + Lastseen jsontime.Time `json:"lastseen"` + IsOnline bool `json:"is_online"` + IsGateway bool `json:"is_gateway"` + Clients uint32 `json:"clients"` + ClientsWifi24 uint32 `json:"clients_wifi24"` + ClientsWifi5 uint32 `json:"clients_wifi5"` + ClientsOthers uint32 `json:"clients_other"` + RootFSUsage float64 `json:"rootfs_usage,omitempty"` + LoadAverage float64 `json:"loadavg,omitempty"` + MemoryUsage *float64 `json:"memory_usage,omitempty"` + Uptime jsontime.Time `json:"uptime,omitempty"` + GatewayNexthop string `json:"gateway_nexthop,omitempty"` + GatewayIPv4 string `json:"gateway,omitempty"` + GatewayIPv6 string `json:"gateway6,omitempty"` + NodeID string `json:"node_id"` + Network Network `json:"network"` + SiteCode string `json:"site_code,omitempty"` + Hostname string `json:"hostname"` + Location *Location `json:"location,omitempty"` + Firmware Firmware `json:"firmware,omitempty"` + Autoupdater Autoupdater `json:"autoupdater"` + Nproc int `json:"nproc"` + Model string `json:"model,omitempty"` + VPN bool `json:"vpn"` +} + +// Firmware out of software +type Firmware struct { + Base string `json:"base,omitempty"` + Release string `json:"release,omitempty"` +} + +// Autoupdater +type Autoupdater struct { + Enabled bool `json:"enabled"` + Branch string `json:"branch,omitempty"` +} + +// Network struct +type Network struct { + MAC string `json:"mac"` + Addresses []string `json:"addresses"` +} + +// Location struct +type Location struct { + Longtitude float64 `json:"longitude,omitempty"` + Latitude float64 `json:"latitude,omitempty"` +} + +// Link +type Link struct { + Type string `json:"type"` + Source string `json:"source"` + Target string `json:"target"` + SourceTQ float32 `json:"source_tq"` + TargetTQ float32 `json:"target_tq"` + // keep the logic for maybe later implementation + SourceMAC string `json:"-"` + TargetMAC string `json:"-"` +} + +func NewNode(nodes *runtime.Nodes, n *runtime.Node) *Node { + node := &Node{ + Firstseen: n.Firstseen, + Lastseen: n.Lastseen, + IsOnline: n.Online, + IsGateway: n.IsGateway(), + } + + if nodeinfo := n.Nodeinfo; nodeinfo != nil { + node.NodeID = nodeinfo.NodeID + node.Network = Network{ + MAC: nodeinfo.Network.Mac, + Addresses: nodeinfo.Network.Addresses, + } + node.SiteCode = nodeinfo.System.SiteCode + node.Hostname = nodeinfo.Hostname + if location := nodeinfo.Location; location != nil { + node.Location = &Location{ + Longtitude: location.Longtitude, + Latitude: location.Latitude, + } + } + node.Firmware = nodeinfo.Software.Firmware + node.Autoupdater = Autoupdater{ + Enabled: nodeinfo.Software.Autoupdater.Enabled, + Branch: nodeinfo.Software.Autoupdater.Branch, + } + node.Nproc = nodeinfo.Hardware.Nproc + node.Model = nodeinfo.Hardware.Model + node.VPN = nodeinfo.VPN + } + if statistic := n.Statistics; statistic != nil { + node.Clients = statistic.Clients.Total + if node.Clients == 0 { + node.Clients = statistic.Clients.Wifi24 + statistic.Clients.Wifi5 + } + node.ClientsWifi24 = statistic.Clients.Wifi24 + node.ClientsWifi5 = statistic.Clients.Wifi5 + + wifi := node.ClientsWifi24 - node.ClientsWifi5 + if node.Clients >= wifi { + node.ClientsOthers = node.Clients - wifi + } + + node.RootFSUsage = statistic.RootFsUsage + node.LoadAverage = statistic.LoadAverage + + /* The Meshviewer could not handle absolute memory output + * calc the used memory as a float which 100% equal 1.0 + * calc is coppied from node statuspage (look discussion: + * https://github.com/FreifunkBremen/yanic/issues/35) + */ + if statistic.Memory.Total > 0 { + usage := 1 - (float64(statistic.Memory.Free)+float64(statistic.Memory.Buffers)+float64(statistic.Memory.Cached))/float64(statistic.Memory.Total) + node.MemoryUsage = &usage + } + + node.Uptime = jsontime.Now().Add(time.Duration(statistic.Uptime) * -time.Second) + node.GatewayNexthop = nodes.GetNodeIDbyMAC(statistic.GatewayNexthop) + if node.GatewayNexthop == "" { + node.GatewayNexthop = statistic.GatewayNexthop + } + node.GatewayIPv4 = nodes.GetNodeIDbyMAC(statistic.GatewayIPv4) + if node.GatewayIPv4 == "" { + node.GatewayIPv4 = statistic.GatewayIPv4 + } + node.GatewayIPv6 = nodes.GetNodeIDbyMAC(statistic.GatewayIPv6) + if node.GatewayIPv6 == "" { + node.GatewayIPv6 = statistic.GatewayIPv6 + } + } + + return node +} diff --git a/output/meshviewer-ffrgb/struct_test.go b/output/meshviewer-ffrgb/struct_test.go new file mode 100644 index 0000000..cf93790 --- /dev/null +++ b/output/meshviewer-ffrgb/struct_test.go @@ -0,0 +1,50 @@ +package meshviewerFFRGB + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestRegister(t *testing.T) { + assert := assert.New(t) + nodes := runtime.NewNodes(&runtime.Config{}) + node := NewNode(nodes, &runtime.Node{ + Nodeinfo: &data.NodeInfo{ + Network: data.Network{ + Mac: "blub", + }, + Location: &data.Location{ + Longtitude: 13.3, + Latitude: 8.7, + }, + }, + Statistics: &data.Statistics{ + Memory: data.Memory{ + Free: 13, + Total: 50, + }, + Wireless: []*data.WirelessAirtime{ + &data.WirelessAirtime{ + ChanUtil: 0.3, + Frequency: 2512, + }, + &data.WirelessAirtime{ + ChanUtil: 0.4, + Frequency: 2612, + }, + &data.WirelessAirtime{ + ChanUtil: 0.5, + Frequency: 5200, + }, + }, + }, + }) + assert.NotNil(node) + assert.Equal("blub", node.Network.MAC) + assert.Equal(13.3, node.Location.Longtitude) + assert.Equal(8.7, node.Location.Latitude) + assert.Equal(0.74, *node.MemoryUsage) +} diff --git a/meshviewer/graph.go b/output/meshviewer/graph.go similarity index 83% rename from meshviewer/graph.go rename to output/meshviewer/graph.go index 0b18b2f..00211e3 100644 --- a/meshviewer/graph.go +++ b/output/meshviewer/graph.go @@ -35,10 +35,9 @@ type GraphLink struct { // GraphBuilder a temporaty struct during fill the graph from the node neighbours type graphBuilder struct { - macToID map[string]string // mapping from MAC address to node id - idToMac map[string]string // mapping from node id to one MAC address - links map[string]*GraphLink // mapping from $idA-$idB to existing link - vpn map[string]interface{} // IDs/addresses of VPN servers + macToID map[string]string // mapping from MAC address to node id + idToMac map[string]string // mapping from node id to one MAC address + links map[string]*GraphLink // mapping from $idA-$idB to existing link } // BuildGraph transform from nodes (Neighbours) to Graph @@ -47,7 +46,6 @@ func BuildGraph(nodes *runtime.Nodes) *Graph { macToID: make(map[string]string), idToMac: make(map[string]string), links: make(map[string]*GraphLink), - vpn: make(map[string]interface{}), } builder.readNodes(nodes.List) @@ -62,12 +60,8 @@ func (builder *graphBuilder) readNodes(nodes map[string]*runtime.Node) { // Fill mac->id map for sourceID, node := range nodes { if nodeinfo := node.Nodeinfo; nodeinfo != nil { - // is VPN address? - if nodeinfo.VPN { - builder.vpn[sourceID] = nil - } - if len(nodeinfo.Network.Mac) > 0 { + if nodeinfo.Network.Mac != "" { builder.idToMac[sourceID] = nodeinfo.Network.Mac } @@ -92,12 +86,21 @@ func (builder *graphBuilder) readNodes(nodes map[string]*runtime.Node) { // Add links for sourceID, node := range nodes { if node.Online { + vpnInterface := make(map[string]interface{}) + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + for _, batinterface := range nodeinfo.Network.Mesh { + for _, vpn := range batinterface.Interfaces.Tunnel { + vpnInterface[vpn] = nil + } + } + } if neighbours := node.Neighbours; neighbours != nil { // Batman neighbours - for _, batadvNeighbours := range neighbours.Batadv { + for sourceMAC, batadvNeighbours := range neighbours.Batadv { for targetAddress, link := range batadvNeighbours.Neighbours { if targetID, found := builder.macToID[targetAddress]; found { - builder.addLink(targetID, sourceID, link.Tq) + _, vpn := vpnInterface[sourceMAC] + builder.addLink(targetID, sourceID, link.Tq, vpn) } } } @@ -105,7 +108,7 @@ func (builder *graphBuilder) readNodes(nodes map[string]*runtime.Node) { for _, neighbours := range neighbours.LLDP { for targetAddress := range neighbours { if targetID, found := builder.macToID[targetAddress]; found { - builder.addLink(targetID, sourceID, 255) + builder.addLink(targetID, sourceID, 255, false) } } } @@ -159,16 +162,7 @@ func (builder *graphBuilder) extract() ([]*GraphNode, []*GraphLink) { return cache.Nodes, links } -func (builder *graphBuilder) isVPN(ids ...string) bool { - for _, id := range ids { - if _, found := builder.vpn[id]; found { - return true - } - } - return false -} - -func (builder *graphBuilder) addLink(targetID string, sourceID string, linkTq int) { +func (builder *graphBuilder) addLink(targetID string, sourceID string, linkTq int, vpn bool) { // Sort IDs to generate the key var key string if strings.Compare(sourceID, targetID) > 0 { @@ -184,7 +178,7 @@ func (builder *graphBuilder) addLink(targetID string, sourceID string, linkTq in if link, ok := builder.links[key]; !ok { builder.links[key] = &GraphLink{ - VPN: builder.isVPN(sourceID, targetID), + VPN: vpn, TQ: tq, } } else { diff --git a/meshviewer/graph_test.go b/output/meshviewer/graph_test.go similarity index 96% rename from meshviewer/graph_test.go rename to output/meshviewer/graph_test.go index ceb6430..6c1313f 100644 --- a/meshviewer/graph_test.go +++ b/output/meshviewer/graph_test.go @@ -56,7 +56,7 @@ func testGetNodeByFile(filename string) *runtime.Node { } func testfile(name string, obj interface{}) { - file, err := ioutil.ReadFile("../runtime/testdata/" + name) + file, err := ioutil.ReadFile("../../runtime/testdata/" + name) if err != nil { panic(err) } diff --git a/meshviewer/node.go b/output/meshviewer/node.go similarity index 100% rename from meshviewer/node.go rename to output/meshviewer/node.go diff --git a/meshviewer/nodes_test.go b/output/meshviewer/nodes_test.go similarity index 67% rename from meshviewer/nodes_test.go rename to output/meshviewer/nodes_test.go index bd08d5e..27f5385 100644 --- a/meshviewer/nodes_test.go +++ b/output/meshviewer/nodes_test.go @@ -25,37 +25,45 @@ func TestNodesV2(t *testing.T) { func createTestNodes() *runtime.Nodes { nodes := runtime.NewNodes(&runtime.Config{}) - nodeData := &data.ResponseData{ + nodeData := &runtime.Node{ Statistics: &data.Statistics{ Clients: data.Clients{ Total: 23, }, }, - NodeInfo: &data.NodeInfo{ + Nodeinfo: &data.NodeInfo{ + NodeID: "abcdef012345", Hardware: data.Hardware{ Model: "TP-Link 841", }, }, } - nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1" - nodes.Update("abcdef012345", nodeData) + nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1" + nodes.AddNode(nodeData) - nodes.Update("112233445566", &data.ResponseData{ + nodes.AddNode(&runtime.Node{ Statistics: &data.Statistics{ Clients: data.Clients{ - Total: 2, + Wifi24: 2, + Wifi5: 3, + }, + Memory: data.Memory{ + Total: 32, + Free: 8, }, }, - NodeInfo: &data.NodeInfo{ + Nodeinfo: &data.NodeInfo{ + NodeID: "112233445566", Hardware: data.Hardware{ Model: "TP-Link 841", }, }, }) - nodes.Update("0xdeadbeef0x", &data.ResponseData{ - NodeInfo: &data.NodeInfo{ - VPN: true, + nodes.AddNode(&runtime.Node{ + Nodeinfo: &data.NodeInfo{ + NodeID: "0xdeadbeef0x", + VPN: true, Hardware: data.Hardware{ Model: "Xeon Multi-Core", }, diff --git a/meshviewer/nodes_v1.go b/output/meshviewer/nodes_v1.go similarity index 94% rename from meshviewer/nodes_v1.go rename to output/meshviewer/nodes_v1.go index 1ffadb0..cccda96 100644 --- a/meshviewer/nodes_v1.go +++ b/output/meshviewer/nodes_v1.go @@ -21,9 +21,7 @@ func BuildNodesV1(nodes *runtime.Nodes) interface{} { Timestamp: jsontime.Now(), } - for nodeID := range nodes.List { - nodeOrigin := nodes.List[nodeID] - + for nodeID, nodeOrigin := range nodes.List { if nodeOrigin.Statistics == nil { continue } diff --git a/meshviewer/nodes_v2.go b/output/meshviewer/nodes_v2.go similarity index 94% rename from meshviewer/nodes_v2.go rename to output/meshviewer/nodes_v2.go index 0500b39..2d4eb6e 100644 --- a/meshviewer/nodes_v2.go +++ b/output/meshviewer/nodes_v2.go @@ -20,8 +20,7 @@ func BuildNodesV2(nodes *runtime.Nodes) interface{} { Timestamp: jsontime.Now(), } - for nodeID := range nodes.List { - nodeOrigin := nodes.List[nodeID] + for _, nodeOrigin := range nodes.List { if nodeOrigin.Statistics == nil { continue } diff --git a/output/meshviewer/output.go b/output/meshviewer/output.go new file mode 100644 index 0000000..df1a94c --- /dev/null +++ b/output/meshviewer/output.go @@ -0,0 +1,72 @@ +package meshviewer + +import ( + "fmt" + "log" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + config Config + builder nodeBuilder +} + +type Config map[string]interface{} + +func (c Config) Version() int64 { + if v := c["version"]; v != nil { + return v.(int64) + } + return -1 +} +func (c Config) NodesPath() string { + if c["nodes_path"] == nil { + log.Panic("in configuration of [[nodes.output.meshviewer]] was no nodes_path defined:\n", c) + } + return c["nodes_path"].(string) +} +func (c Config) GraphPath() string { + return c["graph_path"].(string) +} + +type nodeBuilder func(*runtime.Nodes) interface{} + +var nodeFormats = map[int64]nodeBuilder{ + 1: BuildNodesV1, + 2: BuildNodesV2, +} + +func init() { + output.RegisterAdapter("meshviewer", Register) +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + var config Config + config = configuration + + builder := nodeFormats[config.Version()] + if builder == nil { + return nil, fmt.Errorf("invalid nodes version: %d", config.Version()) + } + + return &Output{ + config: config, + builder: builder, + }, nil +} + +func (o *Output) Save(nodes *runtime.Nodes) { + nodes.RLock() + defer nodes.RUnlock() + + if path := o.config.NodesPath(); path != "" { + runtime.SaveJSON(o.builder(nodes), path) + } + + if path := o.config.GraphPath(); path != "" { + runtime.SaveJSON(BuildGraph(nodes), path) + } +} diff --git a/output/meshviewer/output_test.go b/output/meshviewer/output_test.go new file mode 100644 index 0000000..32b26e8 --- /dev/null +++ b/output/meshviewer/output_test.go @@ -0,0 +1,44 @@ +package meshviewer + +import ( + "os" + "testing" + + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestOutput(t *testing.T) { + assert := assert.New(t) + + // no version defined + out, err := Register(map[string]interface{}{}) + assert.Error(err) + assert.Nil(out) + + // no nodes path defined + out, err = Register(map[string]interface{}{ + "version": int64(1), + }) + assert.NoError(err) + assert.NotNil(out) + assert.Panics(func() { + out.Save(&runtime.Nodes{}) + }) + + out, err = Register(map[string]interface{}{ + "version": int64(2), + "nodes_path": "/tmp/nodes.json", + "graph_path": "/tmp/graph.json", + }) + os.Remove("/tmp/nodes.json") + os.Remove("/tmp/graph.json") + assert.NoError(err) + assert.NotNil(out) + + out.Save(&runtime.Nodes{}) + _, err = os.Stat("/tmp/nodes.json") + assert.NoError(err) + _, err = os.Stat("/tmp/graph.json") + assert.NoError(err) +} diff --git a/output/nodelist/nodelist.go b/output/nodelist/nodelist.go new file mode 100644 index 0000000..0566a75 --- /dev/null +++ b/output/nodelist/nodelist.go @@ -0,0 +1,63 @@ +package nodelist + +import ( + "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/runtime" +) + +// NodeList rewritten after: https://github.com/ffnord/ffmap-backend/blob/c33ebf62f013e18bf71b5a38bd058847340db6b7/lib/nodelist.py +type NodeList struct { + Version string `json:"version"` + Timestamp jsontime.Time `json:"updated_at"` // Timestamp of the generation + List []*Node `json:"nodes"` +} + +type Node struct { + ID string `json:"id"` + Name string `json:"name"` + Position *Position `json:"position,omitempty"` + Status struct { + Online bool `json:"online"` + LastContact jsontime.Time `json:"lastcontact"` + Clients uint32 `json:"clients"` + } `json:"status"` +} + +type Position struct { + Lat float64 `json:"lat"` + Long float64 `json:"long"` +} + +func NewNode(n *runtime.Node) (node *Node) { + if nodeinfo := n.Nodeinfo; nodeinfo != nil { + node = &Node{ + ID: nodeinfo.NodeID, + Name: nodeinfo.Hostname, + } + if location := nodeinfo.Location; location != nil { + node.Position = &Position{Lat: location.Latitude, Long: location.Longtitude} + } + + node.Status.Online = n.Online + node.Status.LastContact = n.Lastseen + if statistics := n.Statistics; statistics != nil { + node.Status.Clients = statistics.Clients.Total + } + } + return +} + +func transform(nodes *runtime.Nodes) *NodeList { + nodelist := &NodeList{ + Version: "1.0.1", + Timestamp: jsontime.Now(), + } + + for _, nodeOrigin := range nodes.List { + node := NewNode(nodeOrigin) + if node != nil { + nodelist.List = append(nodelist.List, node) + } + } + return nodelist +} diff --git a/output/nodelist/nodelist_test.go b/output/nodelist/nodelist_test.go new file mode 100644 index 0000000..c923423 --- /dev/null +++ b/output/nodelist/nodelist_test.go @@ -0,0 +1,67 @@ +package nodelist + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" +) + +func TestTransform(t *testing.T) { + nodes := transform(createTestNodes()) + + assert := assert.New(t) + assert.Len(nodes.List, 3) +} + +func createTestNodes() *runtime.Nodes { + nodes := runtime.NewNodes(&runtime.Config{}) + + nodeData := &runtime.Node{ + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 23, + }, + }, + Nodeinfo: &data.NodeInfo{ + NodeID: "abcdef012345", + Hardware: data.Hardware{ + Model: "TP-Link 841", + }, + }, + } + nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1" + nodes.AddNode(nodeData) + + nodes.AddNode(&runtime.Node{ + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 2, + }, + }, + Nodeinfo: &data.NodeInfo{ + NodeID: "112233445566", + Hardware: data.Hardware{ + Model: "TP-Link 841", + }, + Location: &data.Location{ + Latitude: 23, + Longtitude: 2, + }, + }, + }) + + nodes.AddNode(&runtime.Node{ + Nodeinfo: &data.NodeInfo{ + NodeID: "0xdeadbeef0x", + VPN: true, + Hardware: data.Hardware{ + Model: "Xeon Multi-Core", + }, + }, + }) + + return nodes +} diff --git a/output/nodelist/output.go b/output/nodelist/output.go new file mode 100644 index 0000000..3ed9eb7 --- /dev/null +++ b/output/nodelist/output.go @@ -0,0 +1,46 @@ +package nodelist + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + path string +} + +type Config map[string]interface{} + +func (c Config) Path() string { + if path, ok := c["path"]; ok { + return path.(string) + } + return "" +} + +func init() { + output.RegisterAdapter("nodelist", Register) +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + var config Config + config = configuration + + if path := config.Path(); path != "" { + return &Output{ + path: path, + }, nil + } + return nil, errors.New("no path given") + +} + +func (o *Output) Save(nodes *runtime.Nodes) { + nodes.RLock() + defer nodes.RUnlock() + + runtime.SaveJSON(transform(nodes), o.path) +} diff --git a/output/nodelist/output_test.go b/output/nodelist/output_test.go new file mode 100644 index 0000000..d5bc690 --- /dev/null +++ b/output/nodelist/output_test.go @@ -0,0 +1,28 @@ +package nodelist + +import ( + "os" + "testing" + + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestOutput(t *testing.T) { + assert := assert.New(t) + + out, err := Register(map[string]interface{}{}) + assert.Error(err) + assert.Nil(out) + + out, err = Register(map[string]interface{}{ + "path": "/tmp/nodelist.json", + }) + os.Remove("/tmp/nodelist.json") + assert.NoError(err) + assert.NotNil(out) + + out.Save(&runtime.Nodes{}) + _, err = os.Stat("/tmp/nodelist.json") + assert.NoError(err) +} diff --git a/output/output.go b/output/output.go new file mode 100644 index 0000000..e040505 --- /dev/null +++ b/output/output.go @@ -0,0 +1,19 @@ +package output + +import "github.com/FreifunkBremen/yanic/runtime" + +// Output interface to use for implementation in e.g. influxdb +type Output interface { + // InsertNode stores statistics per node + Save(nodes *runtime.Nodes) +} + +// Register function with config to get a output interface +type Register func(config map[string]interface{}) (Output, error) + +// Adapters is the list of registered output adapters +var Adapters = map[string]Register{} + +func RegisterAdapter(name string, n Register) { + Adapters[name] = n +} diff --git a/output/output_test.go b/output/output_test.go new file mode 100644 index 0000000..24612ab --- /dev/null +++ b/output/output_test.go @@ -0,0 +1,18 @@ +package output + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRegister(t *testing.T) { + assert := assert.New(t) + assert.Len(Adapters, 0) + + RegisterAdapter("blub", func(config map[string]interface{}) (Output, error) { + return nil, nil + }) + + assert.Len(Adapters, 1) +} diff --git a/respond/collector.go b/respond/collector.go index 1098abf..8f7c6eb 100644 --- a/respond/collector.go +++ b/respond/collector.go @@ -256,9 +256,11 @@ func (coll *Collector) saveResponse(addr *net.UDPAddr, res *data.ResponseData) { // Store link data if neighbours := node.Neighbours; neighbours != nil { + coll.nodes.RLock() for _, link := range coll.nodes.NodeLinks(node) { db.InsertLink(&link, node.Lastseen.GetTime()) } + coll.nodes.RUnlock() } } } diff --git a/runtime/config.go b/runtime/config.go index 45c258f..f9f88d8 100644 --- a/runtime/config.go +++ b/runtime/config.go @@ -26,6 +26,7 @@ type Config struct { SaveInterval Duration `toml:"save_interval"` // Save nodes periodically OfflineAfter Duration `toml:"offline_after"` // Set node to offline if not seen within this period PruneAfter Duration `toml:"prune_after"` // Remove nodes after n days of inactivity + Output map[string]interface{} } Meshviewer struct { Version int `toml:"version"` diff --git a/runtime/config_test.go b/runtime/config_test.go index 9450186..c5ae9d7 100644 --- a/runtime/config_test.go +++ b/runtime/config_test.go @@ -18,13 +18,18 @@ func TestReadConfig(t *testing.T) { assert.Equal([]string{"eth0"}, config.Respondd.Interfaces) assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration) - assert.Equal(2, config.Meshviewer.Version) - assert.Equal("/var/www/html/meshviewer/data/nodes.json", config.Meshviewer.NodesPath) - assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration) assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration) + var meshviewer map[string]interface{} + var outputs []map[string]interface{} + outputs = config.Nodes.Output["meshviewer"].([]map[string]interface{}) + assert.Len(outputs, 1, "more outputs are given") + meshviewer = outputs[0] + assert.Equal(int64(2), meshviewer["version"]) + assert.Equal("/var/www/html/meshviewer/data/nodes.json", meshviewer["nodes_path"]) + var influxdb map[string]interface{} dbs := config.Database.Connection["influxdb"] assert.Len(dbs, 1, "more influxdb are given") diff --git a/runtime/nodes.go b/runtime/nodes.go index 51f27df..b963974 100644 --- a/runtime/nodes.go +++ b/runtime/nodes.go @@ -39,6 +39,17 @@ func (nodes *Nodes) Start() { go nodes.worker() } +func (nodes *Nodes) AddNode(node *Node) { + nodeinfo := node.Nodeinfo + if nodeinfo == nil || nodeinfo.NodeID == "" { + return + } + nodes.Lock() + defer nodes.Unlock() + nodes.List[nodeinfo.NodeID] = node + nodes.readIfaces(nodeinfo) +} + // Update a Node func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node { now := jsontime.Now() @@ -52,6 +63,9 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node { } nodes.List[nodeID] = node } + if res.NodeInfo != nil { + nodes.readIfaces(res.NodeInfo) + } nodes.Unlock() // Update wireless statistics @@ -69,10 +83,6 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node { node.Nodeinfo = res.NodeInfo node.Statistics = res.Statistics - if node.Nodeinfo != nil { - nodes.readIfaces(node.Nodeinfo) - } - return node } @@ -90,6 +100,10 @@ func (nodes *Nodes) Select(f func(*Node) bool) []*Node { return result } +func (nodes *Nodes) GetNodeIDbyMAC(mac string) string { + return nodes.ifaceToNodeID[mac] +} + // NodeLinks returns a list of links to known neighbours func (nodes *Nodes) NodeLinks(node *Node) (result []Link) { // Store link data @@ -98,9 +112,6 @@ func (nodes *Nodes) NodeLinks(node *Node) (result []Link) { return } - nodes.RLock() - defer nodes.RUnlock() - for sourceMAC, batadv := range neighbours.Batadv { for neighbourMAC, link := range batadv.Neighbours { if neighbourID := nodes.ifaceToNodeID[neighbourMAC]; neighbourID != "" { @@ -165,8 +176,6 @@ func (nodes *Nodes) readIfaces(nodeinfo *data.NodeInfo) { log.Println("nodeID missing in nodeinfo") return } - nodes.Lock() - defer nodes.Unlock() addresses := []string{network.Mac} @@ -191,11 +200,13 @@ func (nodes *Nodes) load() { if err = json.NewDecoder(f).Decode(nodes); err == nil { log.Println("loaded", len(nodes.List), "nodes") + nodes.Lock() for _, node := range nodes.List { if node.Nodeinfo != nil { nodes.readIfaces(node.Nodeinfo) } } + nodes.Unlock() } else { log.Println("failed to unmarshal nodes:", err) diff --git a/runtime/nodes_test.go b/runtime/nodes_test.go index 7104b0f..a42ddcd 100644 --- a/runtime/nodes_test.go +++ b/runtime/nodes_test.go @@ -137,6 +137,20 @@ func TestSelectNodes(t *testing.T) { assert.Equal(time, selectedNodes[0].Firstseen) } +func TestAddNode(t *testing.T) { + assert := assert.New(t) + nodes := NewNodes(&Config{}) + + nodes.AddNode(&Node{}) + assert.Len(nodes.List, 0) + + nodes.AddNode(&Node{Nodeinfo: &data.NodeInfo{}}) + assert.Len(nodes.List, 0) + + nodes.AddNode(&Node{Nodeinfo: &data.NodeInfo{NodeID: "blub"}}) + assert.Len(nodes.List, 1) +} + func TestLinksNodes(t *testing.T) { assert := assert.New(t) @@ -188,4 +202,7 @@ func TestLinksNodes(t *testing.T) { assert.Equal(link.TargetID, "f4f26dd7a30a") assert.Equal(link.TargetMAC, "f4:f2:6d:d7:a3:0a") assert.Equal(link.TQ, 200) + + nodeid := nodes.GetNodeIDbyMAC("f4:f2:6d:d7:a3:0a") + assert.Equal("f4f26dd7a30a", nodeid) } diff --git a/runtime/stats.go b/runtime/stats.go index 423a5f9..a477303 100644 --- a/runtime/stats.go +++ b/runtime/stats.go @@ -23,7 +23,7 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) { Models: make(CounterMap), } - nodes.Lock() + nodes.RLock() for _, node := range nodes.List { if node.Online { result.Nodes++ @@ -42,7 +42,7 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) { } } } - nodes.Unlock() + nodes.RUnlock() return } diff --git a/runtime/stats_test.go b/runtime/stats_test.go index fd0bfcb..0a4ace4 100644 --- a/runtime/stats_test.go +++ b/runtime/stats_test.go @@ -29,37 +29,43 @@ func TestGlobalStats(t *testing.T) { func createTestNodes() *Nodes { nodes := NewNodes(&Config{}) - nodeData := &data.ResponseData{ + nodeData := &Node{ + Online: true, Statistics: &data.Statistics{ Clients: data.Clients{ Total: 23, }, }, - NodeInfo: &data.NodeInfo{ + Nodeinfo: &data.NodeInfo{ + NodeID: "abcdef012345", Hardware: data.Hardware{ Model: "TP-Link 841", }, }, } - nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1" - nodes.Update("abcdef012345", nodeData) + nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1" + nodes.AddNode(nodeData) - nodes.Update("112233445566", &data.ResponseData{ + nodes.AddNode(&Node{ + Online: true, Statistics: &data.Statistics{ Clients: data.Clients{ Total: 2, }, }, - NodeInfo: &data.NodeInfo{ + Nodeinfo: &data.NodeInfo{ + NodeID: "112233445566", Hardware: data.Hardware{ Model: "TP-Link 841", }, }, }) - nodes.Update("0xdeadbeef0x", &data.ResponseData{ - NodeInfo: &data.NodeInfo{ - VPN: true, + nodes.AddNode(&Node{ + Online: true, + Nodeinfo: &data.NodeInfo{ + NodeID: "0xdeadbeef0x", + VPN: true, Hardware: data.Hardware{ Model: "Xeon Multi-Core", }, diff --git a/runtime/testdata/node1.json b/runtime/testdata/node1.json index 379e5b9..eea2d3b 100644 --- a/runtime/testdata/node1.json +++ b/runtime/testdata/node1.json @@ -2,10 +2,12 @@ "nodeinfo":{ "node_id":"node1.json", "network":{ + "mac": "a", "mesh":{ "bat0":{ "interfaces":{ - "wireless":["a"] + "wireless":["a"], + "other":["a2"] } } } @@ -15,10 +17,13 @@ "batadv":{ "a":{ "neighbours":{ - "b":{"tq":250,"lastseen":0.42}, + "b":{"tq":150,"lastseen":0.42}, "c":{"tq":250,"lastseen":0.42} } } + }, + "lldp":{ + "a2": {"c2": {}} } } diff --git a/runtime/testdata/node2.json b/runtime/testdata/node2.json index de1b546..141039a 100644 --- a/runtime/testdata/node2.json +++ b/runtime/testdata/node2.json @@ -15,7 +15,7 @@ "batadv":{ "b":{ "neighbours":{ - "a":{"tq":150,"lastseen":0.42} + "a":{"tq":250,"lastseen":0.42} } } } diff --git a/runtime/testdata/node3.json b/runtime/testdata/node3.json index 7dcabd3..0a24051 100644 --- a/runtime/testdata/node3.json +++ b/runtime/testdata/node3.json @@ -5,7 +5,8 @@ "mesh":{ "bat0":{ "interfaces":{ - "wireless":["c"] + "wireless":["c"], + "other":["c2"] } } } @@ -18,6 +19,9 @@ "a":{"tq":200,"lastseen":0.42} } } + }, + "lldp":{ + "c2": {"a2": {}} } }