Freifunk-Troisdorf/yanic
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[TASK] add global statistics for individual sites

Browse Source
This commit is contained in:
Julian Labus 2017-11-21 15:12:06 +01:00 committed by Geno
parent fd9a1e1101
commit 7324567f91
15 changed files with 171 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/import.go
View File

@ -12,12 +12,13 @@ import (
// importCmd represents the import command
var importCmd = &cobra.Command{
Use: "import <file.rrd>",
Use: "import <file.rrd> <site>",
Short: "Imports global statistics from the given RRD files, requires InfluxDB",
Example: "yanic import --config /etc/yanic.toml olddata.rrd",
Args: cobra.ExactArgs(1),
Example: "yanic import --config /etc/yanic.toml olddata.rrd global",
Args: cobra.ExactArgs(2),
Run: func(cmd *cobra.Command, args []string) {
path := args[0]
site := args[1]
config := loadConfig()
connections, err := all.Connect(config.Database.Connection)
@ -36,6 +37,7 @@ var importCmd = &cobra.Command{
Clients: uint32(ds.Clients),
},
ds.Time,
site,
)
}
},

2
cmd/query.go
View File

@ -26,7 +26,7 @@ var queryCmd = &cobra.Command{
nodes := runtime.NewNodes(&runtime.Config{})
collector := respond.NewCollector(nil, nodes, []string{iface}, 0)
collector := respond.NewCollector(nil, nodes, []string{}, []string{iface}, 0)
defer collector.Close()
collector.SendPacket(dstAddress)

2
cmd/serve.go
View File

@ -58,7 +58,7 @@ var serveCmd = &cobra.Command{
time.Sleep(delay)
}
collector = respond.NewCollector(connections, nodes, config.Respondd.Interfaces, config.Respondd.Port)
collector = respond.NewCollector(connections, nodes, config.Respondd.Sites, config.Respondd.Interfaces, config.Respondd.Port)
collector.Start(config.Respondd.CollectInterval.Duration)
defer collector.Close()
}

2
config_example.toml
View File

@ -11,6 +11,8 @@ synchronize = "1m"
collect_interval = "1m"
# interface that has an IP in your mesh network
interfaces = ["br-ffhb"]
# list of sites to save stats for (empty for global only)
sites = []
# define a port to listen
# if not set or set to 0 the kernel will use a random free port at its own
#port = 10001

4
database/all/internal.go
View File

@ -43,9 +43,9 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
}
}
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
for _, item := range conn.list {
item.InsertGlobals(stats, time)
item.InsertGlobals(stats, time, site)
}
}

2
database/database.go
View File

@ -15,7 +15,7 @@ type Connection interface {
InsertLink(*runtime.Link, time.Time)
// InsertGlobals stores global statistics
InsertGlobals(*runtime.GlobalStats, time.Time)
InsertGlobals(*runtime.GlobalStats, time.Time, string)
// PruneNodes prunes historical per-node data
PruneNodes(deleteAfter time.Duration)

32
database/graphite/global.go
View File

@ -7,20 +7,30 @@ import (
"github.com/fgrosse/graphigo"
)
func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
c.addPoint(GlobalStatsFields(stats))
c.addCounterMap(CounterMeasurementModel, stats.Models, time)
c.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time)
func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
measurementGlobal := MeasurementGlobal
counterMeasurementModel := CounterMeasurementModel
counterMeasurementFirmware := CounterMeasurementFirmware
if site != runtime.GLOBAL_SITE {
measurementGlobal += "_" + site
counterMeasurementModel += "_" + site
counterMeasurementFirmware += "_" + site
}
func GlobalStatsFields(stats *runtime.GlobalStats) []graphigo.Metric {
c.addPoint(GlobalStatsFields(measurementGlobal, stats))
c.addCounterMap(counterMeasurementModel, stats.Models, time)
c.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time)
}
func GlobalStatsFields(name string, stats *runtime.GlobalStats) []graphigo.Metric {
return []graphigo.Metric{
{Name: MeasurementGlobal + ".nodes", Value: stats.Nodes},
{Name: MeasurementGlobal + ".gateways", Value: stats.Gateways},
{Name: MeasurementGlobal + ".clients.total", Value: stats.Clients},
{Name: MeasurementGlobal + ".clients.wifi", Value: stats.ClientsWifi},
{Name: MeasurementGlobal + ".clients.wifi24", Value: stats.ClientsWifi24},
{Name: MeasurementGlobal + ".clients.wifi5", Value: stats.ClientsWifi5},
{Name: name + ".nodes", Value: stats.Nodes},
{Name: name + ".gateways", Value: stats.Gateways},
{Name: name + ".clients.total", Value: stats.Clients},
{Name: name + ".clients.wifi", Value: stats.ClientsWifi},
{Name: name + ".clients.wifi24", Value: stats.ClientsWifi24},
{Name: name + ".clients.wifi5", Value: stats.ClientsWifi5},
}
}

31
database/influxdb/global.go
View File

@ -8,11 +8,29 @@ import (
)
// InsertGlobals implementation of database
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
conn.addPoint(MeasurementGlobal, nil, GlobalStatsFields(stats), time)
conn.addCounterMap(CounterMeasurementModel, stats.Models, time)
conn.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time)
conn.addCounterMap(CounterMeasurementAutoupdater, stats.Autoupdater, time)
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
var tags models.Tags
measurementGlobal := MeasurementGlobal
counterMeasurementModel := CounterMeasurementModel
counterMeasurementFirmware := CounterMeasurementFirmware
counterMeasurementAutoupdater := CounterMeasurementAutoupdater
if site != runtime.GLOBAL_SITE {
tags = models.Tags{
models.Tag{Key: []byte("site"), Value: []byte(site)},
}
measurementGlobal += "_site"
counterMeasurementModel += "_site"
counterMeasurementFirmware += "_site"
counterMeasurementAutoupdater += "_site"
}
conn.addPoint(measurementGlobal, tags, GlobalStatsFields(stats), time)
conn.addCounterMap(counterMeasurementModel, stats.Models, time, site)
conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site)
conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site)
}
// GlobalStatsFields returns fields for InfluxDB
@ -30,12 +48,13 @@ func GlobalStatsFields(stats *runtime.GlobalStats) map[string]interface{} {
// Saves the values of a CounterMap in the database.
// The key are used as 'value' tag.
// The value is used as 'counter' field.
func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time) {
func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string) {
for key, count := range m {
conn.addPoint(
name,
models.Tags{
models.Tag{Key: []byte("value"), Value: []byte(key)},
models.Tag{Key: []byte("site"), Value: []byte(site)},
},
models.Fields{"count": count},
t,

18
database/influxdb/global_test.go
View File

@ -9,14 +9,20 @@ import (
"github.com/FreifunkBremen/yanic/runtime"
)
const TEST_SITE = "ffxx"
func TestGlobalStats(t *testing.T) {
stats := runtime.NewGlobalStats(createTestNodes())
stats := runtime.NewGlobalStats(createTestNodes(), []string{TEST_SITE})
assert := assert.New(t)
fields := GlobalStatsFields(stats)
// check fields
// check SITE_GLOBAL fields
fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE])
assert.EqualValues(3, fields["nodes"])
// check TEST_SITE fields
fields = GlobalStatsFields(stats[TEST_SITE])
assert.EqualValues(2, fields["nodes"])
}
func createTestNodes() *runtime.Nodes {
@ -34,6 +40,9 @@ func createTestNodes() *runtime.Nodes {
Hardware: data.Hardware{
Model: "TP-Link 841",
},
System: data.System{
SiteCode: TEST_SITE,
},
},
}
nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
@ -64,6 +73,9 @@ func createTestNodes() *runtime.Nodes {
Hardware: data.Hardware{
Model: "Xeon Multi-Core",
},
System: data.System{
SiteCode: TEST_SITE,
},
},
})

4
database/logging/file.go
View File

@ -56,8 +56,8 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
conn.log("InsertLink: ", link)
}
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) {
conn.log("InsertGlobals: [", time.String(), "] nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
conn.log("InsertGlobals: [", time.String(), "] site: ", site, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
}
func (conn *Connection) PruneNodes(deleteAfter time.Duration) {

10
respond/collector.go
View File

@ -24,16 +24,18 @@ type Collector struct {
queue chan *Response // received responses
db database.Connection
nodes *runtime.Nodes
sites []string
interval time.Duration // Interval for multicast packets
stop chan interface{}
}
// NewCollector creates a Collector struct
func NewCollector(db database.Connection, nodes *runtime.Nodes, ifaces []string, port int) *Collector {
func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string , ifaces []string, port int) *Collector {
coll := &Collector{
db: db,
nodes: nodes,
sites: sites,
port: port,
queue: make(chan *Response, 400),
stop: make(chan interface{}),
@ -300,7 +302,9 @@ func (coll *Collector) globalStatsWorker() {
// saves global statistics
func (coll *Collector) saveGlobalStats() {
stats := runtime.NewGlobalStats(coll.nodes)
stats := runtime.NewGlobalStats(coll.nodes, coll.sites)
coll.db.InsertGlobals(stats, time.Now())
for site, stat := range stats {
coll.db.InsertGlobals(stat, time.Now(), site)
}
}

4
respond/collector_test.go
View File

@ -9,10 +9,12 @@ import (
"github.com/stretchr/testify/assert"
)
const SITE_TEST = "ffxx"
func TestCollector(t *testing.T) {
nodes := runtime.NewNodes(&runtime.Config{})
collector := NewCollector(nil, nodes, []string{}, 10001)
collector := NewCollector(nil, nodes, []string{SITE_TEST}, []string{}, 10001)
collector.Start(time.Millisecond)
time.Sleep(time.Millisecond * 10)
collector.Close()

1
runtime/config.go
View File

@ -12,6 +12,7 @@ type Config struct {
Enable bool `toml:"enable"`
Synchronize Duration `toml:"synchronize"`
Interfaces []string `toml:"interfaces"`
Sites []string `toml:"sites"`
Port int `toml:"port"`
CollectInterval Duration `toml:"collect_interval"`
}

64
runtime/stats.go
View File

@ -1,6 +1,9 @@
package runtime
const DISABLED_AUTOUPDATER = "disabled"
const (
DISABLED_AUTOUPDATER = "disabled"
GLOBAL_SITE = "global"
)
// CounterMap to manage multiple values
type CounterMap map[string]uint32
@ -20,33 +23,32 @@ type GlobalStats struct {
}
//NewGlobalStats returns global statistics for InfluxDB
func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
result = &GlobalStats{
func NewGlobalStats(nodes *Nodes, sites []string) (result map[string]*GlobalStats) {
result = make(map[string]*GlobalStats)
result[GLOBAL_SITE] = &GlobalStats{
Firmwares: make(CounterMap),
Models: make(CounterMap),
Autoupdater: make(CounterMap),
}
for _, site := range sites {
result[site] = &GlobalStats{
Firmwares: make(CounterMap),
Models: make(CounterMap),
Autoupdater: make(CounterMap),
}
}
nodes.RLock()
for _, node := range nodes.List {
if node.Online {
result.Nodes++
if stats := node.Statistics; stats != nil {
result.Clients += stats.Clients.Total
result.ClientsWifi24 += stats.Clients.Wifi24
result.ClientsWifi5 += stats.Clients.Wifi5
result.ClientsWifi += stats.Clients.Wifi
}
if node.IsGateway() {
result.Gateways++
}
result[GLOBAL_SITE].Add(node)
if info := node.Nodeinfo; info != nil {
result.Models.Increment(info.Hardware.Model)
result.Firmwares.Increment(info.Software.Firmware.Release)
if info.Software.Autoupdater.Enabled {
result.Autoupdater.Increment(info.Software.Autoupdater.Branch)
} else {
result.Autoupdater.Increment(DISABLED_AUTOUPDATER)
site := info.System.SiteCode
if _, exist := result[site]; exist {
result[site].Add(node)
}
}
}
@ -55,6 +57,30 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
return
}
// Add values to GlobalStats
// if node is online
func (s *GlobalStats) Add(node *Node) {
s.Nodes++
if stats := node.Statistics; stats != nil {
s.Clients += stats.Clients.Total
s.ClientsWifi24 += stats.Clients.Wifi24
s.ClientsWifi5 += stats.Clients.Wifi5
s.ClientsWifi += stats.Clients.Wifi
}
if node.IsGateway() {
s.Gateways++
}
if info := node.Nodeinfo; info != nil {
s.Models.Increment(info.Hardware.Model)
s.Firmwares.Increment(info.Software.Firmware.Release)
if info.Software.Autoupdater.Enabled {
s.Autoupdater.Increment(info.Software.Autoupdater.Branch)
} else {
s.Autoupdater.Increment(DISABLED_AUTOUPDATER)
}
}
}
// Increment counter in the map by one
// if the value is not empty
func (m CounterMap) Increment(key string) {

51
runtime/stats_test.go
View File

@ -8,26 +8,49 @@ import (
"github.com/FreifunkBremen/yanic/data"
)
const TEST_SITE = "ffxx"
func TestGlobalStats(t *testing.T) {
stats := NewGlobalStats(createTestNodes())
stats := NewGlobalStats(createTestNodes(), []string{TEST_SITE})
assert := assert.New(t)
assert.EqualValues(1, stats.Gateways)
assert.EqualValues(3, stats.Nodes)
assert.EqualValues(25, stats.Clients)
assert.Len(stats, 2)
//check GLOBAL_SITE stats
assert.EqualValues(1, stats[GLOBAL_SITE].Gateways)
assert.EqualValues(3, stats[GLOBAL_SITE].Nodes)
assert.EqualValues(25, stats[GLOBAL_SITE].Clients)
// check models
assert.Len(stats.Models, 2)
assert.EqualValues(2, stats.Models["TP-Link 841"])
assert.EqualValues(1, stats.Models["Xeon Multi-Core"])
assert.Len(stats[GLOBAL_SITE].Models, 2)
assert.EqualValues(2, stats[GLOBAL_SITE].Models["TP-Link 841"])
assert.EqualValues(1, stats[GLOBAL_SITE].Models["Xeon Multi-Core"])
// check firmwares
assert.Len(stats.Firmwares, 1)
assert.EqualValues(1, stats.Firmwares["2016.1.6+entenhausen1"])
assert.Len(stats[GLOBAL_SITE].Firmwares, 1)
assert.EqualValues(1, stats[GLOBAL_SITE].Firmwares["2016.1.6+entenhausen1"])
// check autoupdater
assert.Len(stats.Autoupdater, 2)
assert.EqualValues(1, stats.Autoupdater["stable"])
assert.Len(stats[GLOBAL_SITE].Autoupdater, 2)
assert.EqualValues(1, stats[GLOBAL_SITE].Autoupdater["stable"])
// check TEST_SITE stats
assert.EqualValues(1, stats[TEST_SITE].Gateways)
assert.EqualValues(2, stats[TEST_SITE].Nodes)
assert.EqualValues(23, stats[TEST_SITE].Clients)
// check models
assert.Len(stats[TEST_SITE].Models, 2)
assert.EqualValues(1, stats[TEST_SITE].Models["TP-Link 841"])
assert.EqualValues(1, stats[TEST_SITE].Models["Xeon Multi-Core"])
// check firmwares
assert.Len(stats[TEST_SITE].Firmwares, 1)
assert.EqualValues(1, stats[TEST_SITE].Firmwares["2016.1.6+entenhausen1"])
// check autoupdater
assert.Len(stats[TEST_SITE].Autoupdater, 1)
assert.EqualValues(0, stats[TEST_SITE].Autoupdater["stable"])
}
func createTestNodes() *Nodes {
@ -45,6 +68,9 @@ func createTestNodes() *Nodes {
Hardware: data.Hardware{
Model: "TP-Link 841",
},
System: data.System{
SiteCode: TEST_SITE,
},
},
}
nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
@ -82,6 +108,9 @@ func createTestNodes() *Nodes {
Hardware: data.Hardware{
Model: "Xeon Multi-Core",
},
System: data.System{
SiteCode: TEST_SITE,
},
},
})