Merge pull request #5 from FreifunkBremen/nodes_expiration

Expire nodes after n days of inactivity
(Ist okay so -> gleich mal ein Test auf meiner Maschine ...)

config_example.yml

@@ -2,7 +2,9 @@
 respondd:
   enable: true
   interface: eth0
-  collectinterval: 15
+
+  # Collected data every n seconds
+  collectinterval: 60
 webserver:
   enable: false
   port: 8080
@@ -16,9 +18,15 @@ nodes:
   nodes_path: /var/www/html/meshviewer/data/nodes_all.json
   nodesmini_path: /var/www/html/meshviewer/data/nodes.json
   graphs_path: /var/www/html/meshviewer/data/graph.json
-  saveinterval: 5
   aliases_enable: false
   aliases_path: /var/www/html/meshviewer/data/aliases.json
+
+  # Export nodes and graph every n seconds
+  saveinterval: 5
+
+  # Expire offline nodes after n days
+  max_age: 7
+
 influxdb:
   enable: false
   host: http://localhost:8086

jsontime/jsontime.go

@@ -29,7 +29,7 @@ func (t *Time) UnmarshalJSON(data []byte) (err error) {
 	}
 	return
 }
-func (t Time) GetTime() time.Time{
+func (t Time) GetTime() time.Time {
 	return t.time
 }
 func (t Time) Unix() int64 {
@@ -46,3 +46,7 @@ func (t Time) Add(d time.Duration) Time {
 func (t Time) After(u Time) bool {
 	return t.time.After(u.GetTime())
 }
+
+func (t Time) Before(u Time) bool {
+	return t.time.Before(u.GetTime())
+}

meshviewer/meshviewer.go

@@ -11,7 +11,7 @@ import (
 type Node struct {
 	Firstseen  jsontime.Time    `json:"firstseen"`
 	Lastseen   jsontime.Time    `json:"lastseen"`
-	Flags      *Flags           `json:"flags,omitempty"`
+	Flags      Flags            `json:"flags"`
 	Statistics *Statistics      `json:"statistics"`
 	Nodeinfo   *data.NodeInfo   `json:"nodeinfo"`
 	Neighbours *data.Neighbours `json:"-"`

models/config.go

@@ -31,7 +31,8 @@ type Config struct {
 		NodesMiniPath string `yaml:"nodesmini_path"`
 		GraphsPath    string `yaml:"graphs_path"`
 		AliasesPath   string `yaml:"aliases_path"`
-		SaveInterval  int    `yaml:"saveinterval"`
+		SaveInterval  int    `yaml:"saveinterval"` // Save nodes every n seconds
+		MaxAge        int    `yaml:"max_age"`      // Remove nodes after n days of inactivity
 	} `yaml:"nodes"`
 	Influxdb struct {
 		Enable   bool   `yaml:"enable"`

models/graph.go

@@ -78,7 +78,7 @@ func (builder *GraphBuilder) readNodes(nodes map[string]*Node) {
 
 	// Add links
 	for sourceID, node := range nodes {
-		if flag := node.Flags; flag == nil || flag.Online {
+		if node.Flags.Online {
 			if neighbours := node.Neighbours; neighbours != nil {
 				// Batman neighbours
 				for _, batadvNeighbours := range neighbours.Batadv {

models/graph_test.go

@@ -35,7 +35,11 @@ func testGetNodesByFile(files ...string) *Nodes {
 	}
 
 	for _, file := range files {
-		nodes.List[file] = testGetNodeByFile(file)
+		node := testGetNodeByFile(file)
+		nodes.Update(file, &data.ResponseData{
+			NodeInfo:   node.Nodeinfo,
+			Neighbours: node.Neighbours,
+		})
 	}
 
 	return nodes

models/node.go

@@ -13,7 +13,7 @@ import (
 type Node struct {
 	Firstseen  jsontime.Time    `json:"firstseen"`
 	Lastseen   jsontime.Time    `json:"lastseen"`
-	Flags      *meshviewer.Flags `json:"flags,omitempty"`
+	Flags      meshviewer.Flags `json:"flags"`
 	Statistics *data.Statistics `json:"statistics"`
 	Nodeinfo   *data.NodeInfo   `json:"nodeinfo"`
 	Neighbours *data.Neighbours `json:"-"`

models/nodes.go

@@ -56,20 +56,13 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
 	if node == nil {
 		node = &Node{
 			Firstseen: now,
-			Flags: &meshviewer.Flags{
-				Online:  true,
-				Gateway: false,
-			},
 		}
 		nodes.List[nodeID] = node
 	}
 	nodes.Unlock()
 
 	node.Lastseen = now
-
-	if node.Flags != nil {
 	node.Flags.Online = true
-	}
 
 	// Update neighbours
 	if val := res.Neighbours; val != nil {
@@ -145,18 +138,45 @@ func (nodes *Nodes) worker() {
 	c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval))
 
 	for range c {
-		log.Println("saving", len(nodes.List), "nodes")
+		nodes.expire()
+		nodes.save()
+	}
+}
+
+// Expires nodes and set nodes offline
+func (nodes *Nodes) expire() {
 	nodes.Timestamp = jsontime.Now()
+
+	// Nodes last seen before expireTime will be removed
+	maxAge := nodes.config.Nodes.MaxAge
+	if maxAge <= 0 {
+		maxAge = 7 // our default
+	}
+	expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24)
+
+	// Nodes last seen before offlineTime are changed to 'offline'
+	offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
+
+	// Locking foo
 	nodes.Lock()
-		//
+	defer nodes.Unlock()
-		// set node as offline (without statistics)
+
-		for _, node := range nodes.List {
+	for id, node := range nodes.List {
-			if node.Statistics != nil && nodes.Timestamp.After(node.Lastseen.Add(time.Second*time.Duration(10*nodes.config.Respondd.CollectInterval))) {
+		if node.Lastseen.Before(expireTime) {
-				if node.Flags != nil {
+			// expire
+			delete(nodes.List, id)
+		} else if node.Lastseen.Before(offlineTime) {
+			// set to offline
 			node.Flags.Online = false
 		}
 	}
-		}
+}
+
+func (nodes *Nodes) save() {
+	// Locking foo
+	nodes.RLock()
+	defer nodes.RUnlock()
+
 	// serialize nodes
 	save(nodes, nodes.config.Nodes.NodesPath)
 	save(nodes.GetNodesMini(), nodes.config.Nodes.NodesMiniPath)
@@ -164,9 +184,6 @@ func (nodes *Nodes) worker() {
 	if path := nodes.config.Nodes.GraphsPath; path != "" {
 		save(nodes.BuildGraph(), path)
 	}
-
-		nodes.Unlock()
-	}
 }
 
 // Returns global statistics for InfluxDB

models/nodes_test.go

@@ -4,11 +4,45 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/FreifunkBremen/respond-collector/data"
 	"github.com/stretchr/testify/assert"
 )
 
+func TestExpire(t *testing.T) {
+	assert := assert.New(t)
+	config := &Config{}
+	config.Nodes.MaxAge = 6
+	nodes := &Nodes{
+		config: config,
+		List:   make(map[string]*Node),
+	}
+
+	nodes.Update("expire", &data.ResponseData{})  // should expire
+	nodes.Update("offline", &data.ResponseData{}) // should become offline
+	nodes.Update("online", &data.ResponseData{})  // should stay online
+
+	expire := nodes.List["expire"]
+	expire.Lastseen = expire.Lastseen.Add((-6 * time.Hour * 24) - time.Minute)
+	offline := nodes.List["offline"]
+	offline.Lastseen = offline.Lastseen.Add((-6 * time.Hour * 24) + time.Minute)
+
+	nodes.expire()
+
+	// one expired?
+	assert.Equal(2, len(nodes.List))
+	assert.Nil(nodes.List["expire"])
+
+	// one offline?
+	assert.NotNil(nodes.List["offline"])
+	assert.False(nodes.List["offline"].Flags.Online)
+
+	// one online?
+	assert.NotNil(nodes.List["online"])
+	assert.True(nodes.List["online"].Flags.Online)
+}
+
 func TestLoadAndSave(t *testing.T) {
 	assert := assert.New(t)