[TASK] add global statistics for individual sites

cmd/import.go

@@ -12,12 +12,13 @@ import (
 
 // importCmd represents the import command
 var importCmd = &cobra.Command{
-	Use:     "import <file.rrd>",
+	Use:     "import <file.rrd> <site>",
 	Short:   "Imports global statistics from the given RRD files, requires InfluxDB",
-	Example: "yanic import --config /etc/yanic.toml olddata.rrd",
+	Example: "yanic import --config /etc/yanic.toml olddata.rrd global",
-	Args:    cobra.ExactArgs(1),
+	Args:    cobra.ExactArgs(2),
 	Run: func(cmd *cobra.Command, args []string) {
 		path := args[0]
+		site := args[1]
 		config := loadConfig()
 
 		connections, err := all.Connect(config.Database.Connection)
@@ -36,6 +37,7 @@ var importCmd = &cobra.Command{
 					Clients: uint32(ds.Clients),
 				},
 				ds.Time,
+				site,
 			)
 		}
 	},

cmd/query.go

@@ -26,7 +26,7 @@ var queryCmd = &cobra.Command{
 
 		nodes := runtime.NewNodes(&runtime.Config{})
 
-		collector := respond.NewCollector(nil, nodes, []string{iface}, 0)
+		collector := respond.NewCollector(nil, nodes, []string{}, []string{iface}, 0)
 		defer collector.Close()
 		collector.SendPacket(dstAddress)
 

cmd/serve.go

@@ -58,7 +58,7 @@ var serveCmd = &cobra.Command{
 				time.Sleep(delay)
 			}
 
-			collector = respond.NewCollector(connections, nodes, config.Respondd.Interfaces, config.Respondd.Port)
+			collector = respond.NewCollector(connections, nodes, config.Respondd.Sites, config.Respondd.Interfaces, config.Respondd.Port)
 			collector.Start(config.Respondd.CollectInterval.Duration)
 			defer collector.Close()
 		}

config_example.toml

@@ -11,6 +11,8 @@ synchronize      = "1m"
 collect_interval = "1m"
 # interface that has an IP in your mesh network
 interfaces       = ["br-ffhb"]
+# list of sites to save stats for (empty for global only)
+sites            = []
 # define a port to listen
 # if not set or set to 0 the kernel will use a random free port at its own
 #port = 10001

database/all/internal.go

@@ -43,9 +43,9 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
 	}
 }
 
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
 	for _, item := range conn.list {
-		item.InsertGlobals(stats, time)
+		item.InsertGlobals(stats, time, site)
 	}
 }
 

database/database.go

@@ -15,7 +15,7 @@ type Connection interface {
 	InsertLink(*runtime.Link, time.Time)
 
 	// InsertGlobals stores global statistics
-	InsertGlobals(*runtime.GlobalStats, time.Time)
+	InsertGlobals(*runtime.GlobalStats, time.Time, string)
 
 	// PruneNodes prunes historical per-node data
 	PruneNodes(deleteAfter time.Duration)

database/graphite/global.go

@@ -7,20 +7,30 @@ import (
 	"github.com/fgrosse/graphigo"
 )
 
-func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
+func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
-	c.addPoint(GlobalStatsFields(stats))
+	measurementGlobal := MeasurementGlobal
-	c.addCounterMap(CounterMeasurementModel, stats.Models, time)
+	counterMeasurementModel := CounterMeasurementModel
-	c.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time)
+	counterMeasurementFirmware := CounterMeasurementFirmware
+
+	if site != runtime.GLOBAL_SITE {
+		measurementGlobal += "_" + site
+		counterMeasurementModel += "_" + site
+		counterMeasurementFirmware += "_" + site
+	}
+
+	c.addPoint(GlobalStatsFields(measurementGlobal, stats))
+	c.addCounterMap(counterMeasurementModel, stats.Models, time)
+	c.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time)
 }
 
-func GlobalStatsFields(stats *runtime.GlobalStats) []graphigo.Metric {
+func GlobalStatsFields(name string, stats *runtime.GlobalStats) []graphigo.Metric {
 	return []graphigo.Metric{
-		{Name: MeasurementGlobal + ".nodes", Value: stats.Nodes},
+		{Name: name + ".nodes", Value: stats.Nodes},
-		{Name: MeasurementGlobal + ".gateways", Value: stats.Gateways},
+		{Name: name + ".gateways", Value: stats.Gateways},
-		{Name: MeasurementGlobal + ".clients.total", Value: stats.Clients},
+		{Name: name + ".clients.total", Value: stats.Clients},
-		{Name: MeasurementGlobal + ".clients.wifi", Value: stats.ClientsWifi},
+		{Name: name + ".clients.wifi", Value: stats.ClientsWifi},
-		{Name: MeasurementGlobal + ".clients.wifi24", Value: stats.ClientsWifi24},
+		{Name: name + ".clients.wifi24", Value: stats.ClientsWifi24},
-		{Name: MeasurementGlobal + ".clients.wifi5", Value: stats.ClientsWifi5},
+		{Name: name + ".clients.wifi5", Value: stats.ClientsWifi5},
 	}
 }
 

database/influxdb/global.go

@@ -8,11 +8,29 @@ import (
 )
 
 // InsertGlobals implementation of database
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
-	conn.addPoint(MeasurementGlobal, nil, GlobalStatsFields(stats), time)
+	var tags models.Tags
-	conn.addCounterMap(CounterMeasurementModel, stats.Models, time)
+
-	conn.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time)
+	measurementGlobal := MeasurementGlobal
-	conn.addCounterMap(CounterMeasurementAutoupdater, stats.Autoupdater, time)
+	counterMeasurementModel := CounterMeasurementModel
+	counterMeasurementFirmware := CounterMeasurementFirmware
+	counterMeasurementAutoupdater := CounterMeasurementAutoupdater
+
+	if site != runtime.GLOBAL_SITE {
+		tags = models.Tags{
+			models.Tag{Key: []byte("site"), Value: []byte(site)},
+		}
+
+		measurementGlobal += "_site"
+		counterMeasurementModel += "_site"
+		counterMeasurementFirmware += "_site"
+		counterMeasurementAutoupdater += "_site"
+	}
+
+	conn.addPoint(measurementGlobal, tags, GlobalStatsFields(stats), time)
+	conn.addCounterMap(counterMeasurementModel, stats.Models, time, site)
+	conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site)
+	conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site)
 }
 
 // GlobalStatsFields returns fields for InfluxDB
@@ -30,12 +48,13 @@ func GlobalStatsFields(stats *runtime.GlobalStats) map[string]interface{} {
 // Saves the values of a CounterMap in the database.
 // The key are used as 'value' tag.
 // The value is used as 'counter' field.
-func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time) {
+func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string) {
 	for key, count := range m {
 		conn.addPoint(
 			name,
 			models.Tags{
 				models.Tag{Key: []byte("value"), Value: []byte(key)},
+				models.Tag{Key: []byte("site"), Value: []byte(site)},
 			},
 			models.Fields{"count": count},
 			t,

database/influxdb/global_test.go

@@ -9,14 +9,20 @@ import (
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 
+const TEST_SITE = "ffxx"
+
 func TestGlobalStats(t *testing.T) {
-	stats := runtime.NewGlobalStats(createTestNodes())
+	stats := runtime.NewGlobalStats(createTestNodes(), []string{TEST_SITE})
 
 	assert := assert.New(t)
-	fields := GlobalStatsFields(stats)
 
-	// check fields
+	// check SITE_GLOBAL fields
+	fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE])
 	assert.EqualValues(3, fields["nodes"])
+
+	// check TEST_SITE fields
+	fields = GlobalStatsFields(stats[TEST_SITE])
+	assert.EqualValues(2, fields["nodes"])
 }
 
 func createTestNodes() *runtime.Nodes {
@@ -34,6 +40,9 @@ func createTestNodes() *runtime.Nodes {
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
+			System: data.System{
+				SiteCode: TEST_SITE,
+			},
 		},
 	}
 	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
@@ -64,6 +73,9 @@ func createTestNodes() *runtime.Nodes {
 			Hardware: data.Hardware{
 				Model: "Xeon Multi-Core",
 			},
+			System: data.System{
+				SiteCode: TEST_SITE,
+			},
 		},
 	})
 

database/logging/file.go

@@ -56,8 +56,8 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
 	conn.log("InsertLink: ", link)
 }
 
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
-	conn.log("InsertGlobals: [", time.String(), "] nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
+	conn.log("InsertGlobals: [", time.String(), "] site: ", site, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
 }
 
 func (conn *Connection) PruneNodes(deleteAfter time.Duration) {

respond/collector.go

@@ -24,16 +24,18 @@ type Collector struct {
 	queue    chan *Response // received responses
 	db       database.Connection
 	nodes    *runtime.Nodes
+	sites    []string
 	interval time.Duration // Interval for multicast packets
 	stop     chan interface{}
 }
 
 // NewCollector creates a Collector struct
-func NewCollector(db database.Connection, nodes *runtime.Nodes, ifaces []string, port int) *Collector {
+func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string , ifaces []string, port int) *Collector {
 
 	coll := &Collector{
 		db:          db,
 		nodes:       nodes,
+		sites:       sites,
 		port:        port,
 		queue:       make(chan *Response, 400),
 		stop:        make(chan interface{}),
@@ -300,7 +302,9 @@ func (coll *Collector) globalStatsWorker() {
 
 // saves global statistics
 func (coll *Collector) saveGlobalStats() {
-	stats := runtime.NewGlobalStats(coll.nodes)
+	stats := runtime.NewGlobalStats(coll.nodes, coll.sites)
 
-	coll.db.InsertGlobals(stats, time.Now())
+	for site, stat := range stats {
+		coll.db.InsertGlobals(stat, time.Now(), site)
+	}
 }

respond/collector_test.go

@@ -9,10 +9,12 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const SITE_TEST = "ffxx"
+
 func TestCollector(t *testing.T) {
 	nodes := runtime.NewNodes(&runtime.Config{})
 
-	collector := NewCollector(nil, nodes, []string{}, 10001)
+	collector := NewCollector(nil, nodes, []string{SITE_TEST}, []string{}, 10001)
 	collector.Start(time.Millisecond)
 	time.Sleep(time.Millisecond * 10)
 	collector.Close()

runtime/config.go

@@ -12,6 +12,7 @@ type Config struct {
 		Enable          bool     `toml:"enable"`
 		Synchronize     Duration `toml:"synchronize"`
 		Interfaces      []string `toml:"interfaces"`
+		Sites           []string `toml:"sites"`
 		Port            int      `toml:"port"`
 		CollectInterval Duration `toml:"collect_interval"`
 	}

runtime/stats.go

@@ -1,6 +1,9 @@
 package runtime
 
-const DISABLED_AUTOUPDATER = "disabled"
+const (
+	DISABLED_AUTOUPDATER = "disabled"
+	GLOBAL_SITE = "global"
+)
 
 // CounterMap to manage multiple values
 type CounterMap map[string]uint32
@@ -20,33 +23,32 @@ type GlobalStats struct {
 }
 
 //NewGlobalStats returns global statistics for InfluxDB
-func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
+func NewGlobalStats(nodes *Nodes, sites []string) (result map[string]*GlobalStats) {
-	result = &GlobalStats{
+	result = make(map[string]*GlobalStats)
+
+	result[GLOBAL_SITE] = &GlobalStats{
 		Firmwares:   make(CounterMap),
 		Models:      make(CounterMap),
 		Autoupdater: make(CounterMap),
 	}
 
+	for _, site := range sites {
+		result[site] = &GlobalStats{
+			Firmwares:   make(CounterMap),
+			Models:      make(CounterMap),
+			Autoupdater: make(CounterMap),
+		}
+	}
+
 	nodes.RLock()
 	for _, node := range nodes.List {
 		if node.Online {
-			result.Nodes++
+			result[GLOBAL_SITE].Add(node)
-			if stats := node.Statistics; stats != nil {
+
-				result.Clients += stats.Clients.Total
-				result.ClientsWifi24 += stats.Clients.Wifi24
-				result.ClientsWifi5 += stats.Clients.Wifi5
-				result.ClientsWifi += stats.Clients.Wifi
-			}
-			if node.IsGateway() {
-				result.Gateways++
-			}
 			if info := node.Nodeinfo; info != nil {
-				result.Models.Increment(info.Hardware.Model)
+				site := info.System.SiteCode
-				result.Firmwares.Increment(info.Software.Firmware.Release)
+				if _, exist := result[site]; exist {
-				if info.Software.Autoupdater.Enabled {
+					result[site].Add(node)
-					result.Autoupdater.Increment(info.Software.Autoupdater.Branch)
-				} else {
-					result.Autoupdater.Increment(DISABLED_AUTOUPDATER)
 				}
 			}
 		}
@@ -55,6 +57,30 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
 	return
 }
 
+// Add values to GlobalStats
+// if node is online
+func (s *GlobalStats) Add(node *Node) {
+	s.Nodes++
+	if stats := node.Statistics; stats != nil {
+		s.Clients += stats.Clients.Total
+		s.ClientsWifi24 += stats.Clients.Wifi24
+		s.ClientsWifi5 += stats.Clients.Wifi5
+		s.ClientsWifi += stats.Clients.Wifi
+	}
+	if node.IsGateway() {
+		s.Gateways++
+	}
+	if info := node.Nodeinfo; info != nil {
+		s.Models.Increment(info.Hardware.Model)
+		s.Firmwares.Increment(info.Software.Firmware.Release)
+		if info.Software.Autoupdater.Enabled {
+			s.Autoupdater.Increment(info.Software.Autoupdater.Branch)
+		} else {
+			s.Autoupdater.Increment(DISABLED_AUTOUPDATER)
+		}
+	}
+}
+
 // Increment counter in the map by one
 // if the value is not empty
 func (m CounterMap) Increment(key string) {

runtime/stats_test.go

@@ -8,26 +8,49 @@ import (
 	"github.com/FreifunkBremen/yanic/data"
 )
 
+const TEST_SITE = "ffxx"
+
 func TestGlobalStats(t *testing.T) {
-	stats := NewGlobalStats(createTestNodes())
+	stats := NewGlobalStats(createTestNodes(), []string{TEST_SITE})
 
 	assert := assert.New(t)
-	assert.EqualValues(1, stats.Gateways)
+	assert.Len(stats, 2)
-	assert.EqualValues(3, stats.Nodes)
+
-	assert.EqualValues(25, stats.Clients)
+	//check GLOBAL_SITE stats
+	assert.EqualValues(1, stats[GLOBAL_SITE].Gateways)
+	assert.EqualValues(3, stats[GLOBAL_SITE].Nodes)
+	assert.EqualValues(25, stats[GLOBAL_SITE].Clients)
 
 	// check models
-	assert.Len(stats.Models, 2)
+	assert.Len(stats[GLOBAL_SITE].Models, 2)
-	assert.EqualValues(2, stats.Models["TP-Link 841"])
+	assert.EqualValues(2, stats[GLOBAL_SITE].Models["TP-Link 841"])
-	assert.EqualValues(1, stats.Models["Xeon Multi-Core"])
+	assert.EqualValues(1, stats[GLOBAL_SITE].Models["Xeon Multi-Core"])
 
 	// check firmwares
-	assert.Len(stats.Firmwares, 1)
+	assert.Len(stats[GLOBAL_SITE].Firmwares, 1)
-	assert.EqualValues(1, stats.Firmwares["2016.1.6+entenhausen1"])
+	assert.EqualValues(1, stats[GLOBAL_SITE].Firmwares["2016.1.6+entenhausen1"])
 
 	// check autoupdater
-	assert.Len(stats.Autoupdater, 2)
+	assert.Len(stats[GLOBAL_SITE].Autoupdater, 2)
-	assert.EqualValues(1, stats.Autoupdater["stable"])
+	assert.EqualValues(1, stats[GLOBAL_SITE].Autoupdater["stable"])
+
+	// check TEST_SITE stats
+	assert.EqualValues(1, stats[TEST_SITE].Gateways)
+	assert.EqualValues(2, stats[TEST_SITE].Nodes)
+	assert.EqualValues(23, stats[TEST_SITE].Clients)
+
+	// check models
+	assert.Len(stats[TEST_SITE].Models, 2)
+	assert.EqualValues(1, stats[TEST_SITE].Models["TP-Link 841"])
+	assert.EqualValues(1, stats[TEST_SITE].Models["Xeon Multi-Core"])
+
+	// check firmwares
+	assert.Len(stats[TEST_SITE].Firmwares, 1)
+	assert.EqualValues(1, stats[TEST_SITE].Firmwares["2016.1.6+entenhausen1"])
+
+	// check autoupdater
+	assert.Len(stats[TEST_SITE].Autoupdater, 1)
+	assert.EqualValues(0, stats[TEST_SITE].Autoupdater["stable"])
 }
 
 func createTestNodes() *Nodes {
@@ -45,6 +68,9 @@ func createTestNodes() *Nodes {
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
+			System: data.System{
+				SiteCode: TEST_SITE,
+			},
 		},
 	}
 	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
@@ -82,6 +108,9 @@ func createTestNodes() *Nodes {
 			Hardware: data.Hardware{
 				Model: "Xeon Multi-Core",
 			},
+			System: data.System{
+				SiteCode: TEST_SITE,
+			},
 		},
 	})