Merge pull request #110 from FreifunkBremen/fix-sleep

[TASK] refactoring

cmd/config.go

@@ -2,13 +2,24 @@ package cmd
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 
 	"github.com/FreifunkBremen/yanic/database"
 	"github.com/FreifunkBremen/yanic/respond"
 	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/FreifunkBremen/yanic/webserver"
+	"github.com/naoina/toml"
 )
 
+// Config represents the whole configuration
+type Config struct {
+	Respondd  respond.Config
+	Webserver webserver.Config
+	Nodes     runtime.NodesConfig
+	Database  database.Config
+}
+
 var (
 	configPath  string
 	collector   *respond.Collector
@@ -16,11 +27,28 @@ var (
 	nodes       *runtime.Nodes
 )
 
-func loadConfig() *runtime.Config {
-	config, err := runtime.ReadConfigFile(configPath)
+func loadConfig() *Config {
+	config, err := ReadConfigFile(configPath)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "unable to load config file:", err)
 		os.Exit(2)
 	}
 	return config
 }
+
+// ReadConfigFile reads a config model from path of a yml file
+func ReadConfigFile(path string) (config *Config, err error) {
+	config = &Config{}
+
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = toml.Unmarshal(file, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}

cmd/config_test.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	config, err := ReadConfigFile("../config_example.toml")
+	assert.NoError(err)
+	assert.NotNil(config)
+
+	assert.True(config.Respondd.Enable)
+	assert.Equal([]string{"br-ffhb"}, config.Respondd.Interfaces)
+	assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
+	assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
+	assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration)
+
+	// Test output plugins
+	assert.Len(config.Nodes.Output, 3)
+	outputs := config.Nodes.Output["meshviewer"].([]interface{})
+	assert.Len(outputs, 1)
+	meshviewer := outputs[0]
+
+	assert.EqualValues(map[string]interface{}{
+		"version":    int64(2),
+		"enable":     false,
+		"nodes_path": "/var/www/html/meshviewer/data/nodes.json",
+		"graph_path": "/var/www/html/meshviewer/data/graph.json",
+	}, meshviewer)
+
+	_, err = ReadConfigFile("testdata/config_invalid.toml")
+	assert.Error(err, "not unmarshalable")
+	assert.Contains(err.Error(), "invalid TOML syntax")
+
+	_, err = ReadConfigFile("testdata/adsa.toml")
+	assert.Error(err, "not found able")
+	assert.Contains(err.Error(), "no such file or directory")
+}

cmd/import.go

@@ -3,8 +3,7 @@ package cmd
 import (
 	"log"
 
-	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/database/all"
+	allDatabase "github.com/FreifunkBremen/yanic/database/all"
 	"github.com/FreifunkBremen/yanic/rrd"
 	"github.com/FreifunkBremen/yanic/runtime"
 	"github.com/spf13/cobra"
@@ -21,12 +20,11 @@ var importCmd = &cobra.Command{
 		site := args[1]
 		config := loadConfig()
 
-		connections, err := all.Connect(config.Database.Connection)
+		err := allDatabase.Start(config.Database)
 		if err != nil {
 			panic(err)
 		}
-		database.Start(connections, config)
-		defer database.Close(connections)
+		defer allDatabase.Close()
 
 		log.Println("importing RRD from", path)
 

cmd/query.go

@@ -24,7 +24,7 @@ var queryCmd = &cobra.Command{
 
 		log.Printf("Sending request address=%s iface=%s", dstAddress, iface)
 
-		nodes := runtime.NewNodes(&runtime.Config{})
+		nodes := runtime.NewNodes(&runtime.NodesConfig{})
 
 		collector := respond.NewCollector(nil, nodes, []string{}, []string{iface}, 0)
 		defer collector.Close()

cmd/serve.go

@@ -7,9 +7,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/FreifunkBremen/yanic/database"
 	allDatabase "github.com/FreifunkBremen/yanic/database/all"
-	"github.com/FreifunkBremen/yanic/output"
 	allOutput "github.com/FreifunkBremen/yanic/output/all"
 	"github.com/FreifunkBremen/yanic/respond"
 	"github.com/FreifunkBremen/yanic/runtime"
@@ -25,22 +23,20 @@ var serveCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		config := loadConfig()
 
-		connections, err := allDatabase.Connect(config.Database.Connection)
+		err := allDatabase.Start(config.Database)
 		if err != nil {
 			panic(err)
 		}
-		database.Start(connections, config)
-		defer database.Close(connections)
+		defer allDatabase.Close()
 
-		nodes = runtime.NewNodes(config)
+		nodes = runtime.NewNodes(&config.Nodes)
 		nodes.Start()
 
-		outputs, err := allOutput.Register(config.Nodes.Output)
+		err = allOutput.Start(nodes, config.Nodes)
 		if err != nil {
 			panic(err)
 		}
-		output.Start(outputs, nodes, config)
-		defer output.Close()
+		defer allOutput.Close()
 
 		if config.Webserver.Enable {
 			log.Println("starting webserver on", config.Webserver.Bind)

cmd/testdata/config_invalid.toml

@@ -0,0 +1 @@
+foobar

database/all/connection.go

@@ -0,0 +1,75 @@
+package all
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/FreifunkBremen/yanic/database"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Connection struct {
+	database.Connection
+	list []database.Connection
+}
+
+func Connect(allConnection map[string]interface{}) (database.Connection, error) {
+	var list []database.Connection
+	for dbType, conn := range database.Adapters {
+		configForType := allConnection[dbType]
+		if configForType == nil {
+			log.Printf("the output type '%s' has no configuration", dbType)
+			continue
+		}
+		dbConfigs, ok := configForType.([]map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("the output type '%s' has the wrong format", dbType)
+		}
+
+		for _, config := range dbConfigs {
+			if c, ok := config["enable"].(bool); ok && !c {
+				continue
+			}
+			connected, err := conn(config)
+			if err != nil {
+				return nil, err
+			}
+			if connected == nil {
+				continue
+			}
+			list = append(list, connected)
+		}
+	}
+	return &Connection{list: list}, nil
+}
+
+func (conn *Connection) InsertNode(node *runtime.Node) {
+	for _, item := range conn.list {
+		item.InsertNode(node)
+	}
+}
+
+func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
+	for _, item := range conn.list {
+		item.InsertLink(link, time)
+	}
+}
+
+func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
+	for _, item := range conn.list {
+		item.InsertGlobals(stats, time, site)
+	}
+}
+
+func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
+	for _, item := range conn.list {
+		item.PruneNodes(deleteAfter)
+	}
+}
+
+func (conn *Connection) Close() {
+	for _, item := range conn.list {
+		item.Close()
+	}
+}

database/all/internal.go

@@ -1,74 +1,45 @@
 package all
 
 import (
-	"log"
+	"sync"
 	"time"
 
 	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/runtime"
 )
 
-type Connection struct {
-	database.Connection
-	list []database.Connection
-}
+var conn database.Connection
+var wg = sync.WaitGroup{}
+var quit chan struct{}
 
-func Connect(allConnection map[string]interface{}) (database.Connection, error) {
-	var list []database.Connection
-	for dbType, conn := range database.Adapters {
-		configForType := allConnection[dbType]
-		if configForType == nil {
-			log.Printf("the output type '%s' has no configuration\n", dbType)
-			continue
-		}
-		dbConfigs, ok := configForType.([]map[string]interface{})
-		if !ok {
-			log.Panicf("the output type '%s' has the wrong format\n", dbType)
-		}
-
-		for _, config := range dbConfigs {
-			if c, ok := config["enable"].(bool); ok && !c {
-				continue
-			}
-			connected, err := conn(config)
+func Start(config database.Config) (err error) {
+	conn, err = Connect(config.Connection)
 	if err != nil {
-				return nil, err
+		return
 	}
-			if connected == nil {
-				continue
-			}
-			list = append(list, connected)
-		}
-	}
-	return &Connection{list: list}, nil
+	quit = make(chan struct{})
+	wg.Add(1)
+	go deleteWorker(config.DeleteInterval.Duration, config.DeleteAfter.Duration)
+	return
 }
 
-func (conn *Connection) InsertNode(node *runtime.Node) {
-	for _, item := range conn.list {
-		item.InsertNode(node)
-	}
+func Close() {
+	close(quit)
+	wg.Wait()
+	conn.Close()
+	quit = nil
 }
 
-func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
-	for _, item := range conn.list {
-		item.InsertLink(link, time)
-	}
-}
-
-func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
-	for _, item := range conn.list {
-		item.InsertGlobals(stats, time, site)
-	}
-}
-
-func (conn *Connection) PruneNodes(deleteAfter time.Duration) {
-	for _, item := range conn.list {
-		item.PruneNodes(deleteAfter)
-	}
-}
-
-func (conn *Connection) Close() {
-	for _, item := range conn.list {
-		item.Close()
+// prunes node-specific data periodically
+func deleteWorker(deleteInterval time.Duration, deleteAfter time.Duration) {
+	ticker := time.NewTicker(deleteInterval)
+	for {
+		select {
+		case <-ticker.C:
+			conn.PruneNodes(deleteAfter)
+		case <-quit:
+			ticker.Stop()
+			wg.Done()
+			return
+		}
 	}
 }

database/all/internel_test.go

@@ -2,96 +2,28 @@ package all
 
 import (
 	"errors"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/FreifunkBremen/yanic/lib/duration"
 	"github.com/stretchr/testify/assert"
 )
 
-type testConn struct {
-	database.Connection
-	countNode    int
-	countLink    int
-	countGlobals int
-	countPrune   int
-	countClose   int
-	sync.Mutex
-}
-
-func (c *testConn) InsertNode(node *runtime.Node) {
-	c.Lock()
-	c.countNode++
-	c.Unlock()
-}
-func (c *testConn) GetNode() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countNode
-}
-func (c *testConn) InsertLink(link *runtime.Link, time time.Time) {
-	c.Lock()
-	c.countLink++
-	c.Unlock()
-}
-func (c *testConn) GetLink() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countLink
-}
-func (c *testConn) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
-	c.Lock()
-	c.countGlobals++
-	c.Unlock()
-}
-func (c *testConn) GetGlobal() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countGlobals
-}
-func (c *testConn) PruneNodes(time.Duration) {
-	c.Lock()
-	c.countPrune++
-	c.Unlock()
-}
-func (c *testConn) GetPrune() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countPrune
-}
-func (c *testConn) Close() {
-	c.Lock()
-	c.countClose++
-	c.Unlock()
-}
-func (c *testConn) GetClose() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countClose
-}
-
 func TestStart(t *testing.T) {
 	assert := assert.New(t)
 
-	globalConn := &testConn{}
-	database.RegisterAdapter("a", func(config map[string]interface{}) (database.Connection, error) {
-		return globalConn, nil
-	})
-	database.RegisterAdapter("b", func(config map[string]interface{}) (database.Connection, error) {
-		return globalConn, nil
-	})
-	database.RegisterAdapter("c", func(config map[string]interface{}) (database.Connection, error) {
-		return globalConn, nil
-	})
 	database.RegisterAdapter("d", func(config map[string]interface{}) (database.Connection, error) {
 		return nil, nil
 	})
 	database.RegisterAdapter("e", func(config map[string]interface{}) (database.Connection, error) {
 		return nil, errors.New("blub")
 	})
-	allConn, err := Connect(map[string]interface{}{
+	// Test for PruneNodes (by start)
+	assert.Nil(quit)
+	err := Start(database.Config{
+		DeleteInterval: duration.Duration{Duration: time.Millisecond},
+		Connection: map[string]interface{}{
 			"a": []map[string]interface{}{
 				map[string]interface{}{
 					"enable": false,
@@ -117,29 +49,12 @@ func TestStart(t *testing.T) {
 					"path": "d0",
 				},
 			},
+		},
 	})
 	assert.NoError(err)
+	assert.NotNil(quit)
 
-	assert.Equal(0, globalConn.GetNode())
-	allConn.InsertNode(nil)
-	assert.Equal(3, globalConn.GetNode())
-
-	assert.Equal(0, globalConn.GetLink())
-	allConn.InsertLink(nil, time.Now())
-	assert.Equal(3, globalConn.GetLink())
-
-	assert.Equal(0, globalConn.GetGlobal())
-	allConn.InsertGlobals(nil, time.Now(), runtime.GLOBAL_SITE)
-	assert.Equal(3, globalConn.GetGlobal())
-
-	assert.Equal(0, globalConn.GetPrune())
-	allConn.PruneNodes(time.Second)
-	assert.Equal(3, globalConn.GetPrune())
-
-	assert.Equal(0, globalConn.GetClose())
-	allConn.Close()
-	assert.Equal(3, globalConn.GetClose())
-
+	// connection type not found
 	_, err = Connect(map[string]interface{}{
 		"e": []map[string]interface{}{
 			map[string]interface{}{},
@@ -147,10 +62,14 @@ func TestStart(t *testing.T) {
 	})
 	assert.Error(err)
 
-	// wrong format -> the only panic in Register
-	assert.Panics(func() {
-		Connect(map[string]interface{}{
+	// test close
+	Close()
+
+	// wrong format
+	err = Start(database.Config{
+		Connection: map[string]interface{}{
 			"e": true,
+		},
 	})
-	})
+	assert.Error(err)
 }

database/config.go

@@ -0,0 +1,9 @@
+package database
+
+import "github.com/FreifunkBremen/yanic/lib/duration"
+
+type Config struct {
+	DeleteInterval duration.Duration `toml:"delete_interval"` // Delete stats of nodes every n minutes
+	DeleteAfter    duration.Duration `toml:"delete_after"`    // Delete stats of nodes till now-deletetill n minutes
+	Connection     map[string]interface{}
+}

database/influxdb/global_test.go

@@ -88,7 +88,7 @@ func TestGlobalStats(t *testing.T) {
 }
 
 func createTestNodes() *runtime.Nodes {
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 
 	nodeData := &runtime.Node{
 		Online: true,

database/influxdb/node_test.go

@@ -177,7 +177,7 @@ func testPoints(nodes ...*runtime.Node) (points []*client.Point) {
 		panic(err)
 	}
 
-	nodesList := runtime.NewNodes(&runtime.Config{})
+	nodesList := runtime.NewNodes(&runtime.NodesConfig{})
 
 	// Create dummy connection
 	conn := &Connection{

database/internal.go

@@ -1,40 +0,0 @@
-package database
-
-import (
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-var quit chan struct{}
-
-// Start workers of database
-// WARNING: Do not override this function
-//  you should use New()
-func Start(conn Connection, config *runtime.Config) {
-	quit = make(chan struct{})
-	go deleteWorker(conn, config.Database.DeleteInterval.Duration, config.Database.DeleteAfter.Duration)
-}
-
-func Close(conn Connection) {
-	if quit != nil {
-		close(quit)
-	}
-	if conn != nil {
-		conn.Close()
-	}
-}
-
-// prunes node-specific data periodically
-func deleteWorker(conn Connection, deleteInterval time.Duration, deleteAfter time.Duration) {
-	ticker := time.NewTicker(deleteInterval)
-	for {
-		select {
-		case <-ticker.C:
-			conn.PruneNodes(deleteAfter)
-		case <-quit:
-			ticker.Stop()
-			return
-		}
-	}
-}

database/internal_test.go

@@ -1,69 +0,0 @@
-package database
-
-import (
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-	"github.com/stretchr/testify/assert"
-)
-
-type testConn struct {
-	Connection
-	countClose int
-	countPrune int
-	sync.Mutex
-}
-
-func (c *testConn) Close() {
-	c.Lock()
-	c.countClose++
-	c.Unlock()
-}
-func (c *testConn) GetClose() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countClose
-}
-func (c *testConn) PruneNodes(time.Duration) {
-	c.Lock()
-	c.countPrune++
-	c.Unlock()
-}
-func (c *testConn) GetPruneNodes() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countPrune
-}
-
-func TestStart(t *testing.T) {
-	assert := assert.New(t)
-
-	conn := &testConn{}
-	config := &runtime.Config{
-		Database: struct {
-			DeleteInterval runtime.Duration `toml:"delete_interval"`
-			DeleteAfter    runtime.Duration `toml:"delete_after"`
-			Connection     map[string]interface{}
-		}{
-			DeleteInterval: runtime.Duration{Duration: time.Millisecond * 10},
-		},
-	}
-	assert.Nil(quit)
-
-	Start(conn, config)
-	assert.NotNil(quit)
-
-	assert.Equal(0, conn.GetPruneNodes())
-	time.Sleep(time.Millisecond * 12)
-	assert.Equal(1, conn.GetPruneNodes())
-
-	assert.Equal(0, conn.GetClose())
-	Close(conn)
-	assert.NotNil(quit)
-	assert.Equal(1, conn.GetClose())
-
-	time.Sleep(time.Millisecond * 12) // to reach timer.Stop() line
-
-}

lib/duration/duration.go

@@ -1,4 +1,4 @@
-package runtime
+package duration
 
 import (
 	"fmt"
@@ -18,14 +18,8 @@ type Duration struct {
 }
 
 // UnmarshalTOML parses a duration string.
-func (d *Duration) UnmarshalTOML(dataInterface interface{}) error {
-	var data string
-	switch dataInterface.(type) {
-	case string:
-		data = dataInterface.(string)
-	default:
-		return fmt.Errorf("invalid duration: \"%s\"", dataInterface)
-	}
+func (d *Duration) UnmarshalText(data []byte) error {
+
 	// " + int + unit + "
 	if len(data) < 2 {
 		return fmt.Errorf("invalid duration: \"%s\"", data)

lib/duration/duration_test.go

@@ -1,4 +1,4 @@
-package runtime
+package duration
 
 import (
 	"testing"
@@ -16,6 +16,8 @@ func TestDuration(t *testing.T) {
 		duration time.Duration
 	}{
 		{"", "invalid duration: \"\"", 0},
+		{"3", "invalid duration: \"3\"", 0},
+		{"am", "unable to parse duration \"am\": strconv.Atoi: parsing \"a\": invalid syntax", 0},
 		{"1x", "invalid duration unit \"x\"", 0},
 		{"1s", "", time.Second},
 		{"73s", "", time.Second * 73},
@@ -34,7 +36,7 @@ func TestDuration(t *testing.T) {
 	for _, test := range tests {
 
 		d := Duration{}
-		err := d.UnmarshalTOML(test.input)
+		err := d.UnmarshalText([]byte(test.input))
 		duration := d.Duration
 
 		if test.err == "" {
@@ -44,13 +46,4 @@ func TestDuration(t *testing.T) {
 			assert.EqualError(err, test.err)
 		}
 	}
-
-	d := Duration{}
-	err := d.UnmarshalTOML(3)
-	assert.Error(err)
-	assert.Contains(err.Error(), "invalid duration")
-
-	err = d.UnmarshalTOML("am")
-	assert.Error(err)
-	assert.EqualError(err, "unable to parse duration \"am\": strconv.Atoi: parsing \"a\": invalid syntax")
 }

output/all/filter.go

@@ -1,6 +1,8 @@
 package all
 
-import "github.com/FreifunkBremen/yanic/runtime"
+import (
+	"github.com/FreifunkBremen/yanic/runtime"
+)
 
 // Config Filter
 type filterConfig map[string]interface{}
@@ -13,7 +15,7 @@ func noFilter(node *runtime.Node) *runtime.Node {
 
 // Create Filter
 func (f filterConfig) filtering(nodesOrigin *runtime.Nodes) *runtime.Nodes {
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 	filterfuncs := []filterFunc{
 		f.HasLocation(),
 		f.Blacklist(),

output/all/internal.go

@@ -1,63 +1,45 @@
 package all
 
 import (
-	"log"
+	"sync"
+	"time"
 
 	"github.com/FreifunkBremen/yanic/output"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 
-type Output struct {
-	output.Output
-	list   map[int]output.Output
-	filter map[int]filterConfig
-}
+var quit chan struct{}
+var wg = sync.WaitGroup{}
+var outputA output.Output
 
-func Register(configuration map[string]interface{}) (output.Output, error) {
-	list := make(map[int]output.Output)
-	filter := make(map[int]filterConfig)
-	i := 1
-	allOutputs := configuration
-	for outputType, outputRegister := range output.Adapters {
-		configForOutput := allOutputs[outputType]
-		if configForOutput == nil {
-			log.Printf("the output type '%s' has no configuration\n", outputType)
-			continue
-		}
-		outputConfigs, ok := configForOutput.([]map[string]interface{})
-		if !ok {
-			log.Panicf("the output type '%s' has the wrong format\n", outputType)
-		}
-		for _, config := range outputConfigs {
-			if c, ok := config["enable"].(bool); ok && !c {
-				continue
-			}
-			output, err := outputRegister(config)
+func Start(nodes *runtime.Nodes, config runtime.NodesConfig) (err error) {
+	outputA, err = Register(config.Output)
 	if err != nil {
-				return nil, err
+		return
 	}
-			if output == nil {
-				continue
-			}
-			list[i] = output
-			if c := config["filter"]; c != nil {
-				filter[i] = config["filter"].(map[string]interface{})
-			}
-			i++
-		}
-	}
-	return &Output{list: list, filter: filter}, nil
+	quit = make(chan struct{})
+	wg.Add(1)
+	go saveWorker(nodes, config.SaveInterval.Duration)
+	return
 }
 
-func (o *Output) Save(nodes *runtime.Nodes) {
-	for i, item := range o.list {
-		var filteredNodes *runtime.Nodes
-		if config := o.filter[i]; config != nil {
-			filteredNodes = config.filtering(nodes)
-		} else {
-			filteredNodes = filterConfig{}.filtering(nodes)
-		}
+func Close() {
+	close(quit)
+	wg.Wait()
+	quit = nil
+}
 
-		item.Save(filteredNodes)
+// save periodically to output
+func saveWorker(nodes *runtime.Nodes, saveInterval time.Duration) {
+	ticker := time.NewTicker(saveInterval)
+	for {
+		select {
+		case <-ticker.C:
+			outputA.Save(nodes)
+		case <-quit:
+			ticker.Stop()
+			wg.Done()
+			return
+		}
 	}
 }

output/all/output.go

@@ -0,0 +1,64 @@
+package all
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	list   map[int]output.Output
+	filter map[int]filterConfig
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	list := make(map[int]output.Output)
+	filter := make(map[int]filterConfig)
+	i := 1
+	allOutputs := configuration
+	for outputType, outputRegister := range output.Adapters {
+		configForOutput := allOutputs[outputType]
+		if configForOutput == nil {
+			log.Printf("the output type '%s' has no configuration\n", outputType)
+			continue
+		}
+		outputConfigs, ok := configForOutput.([]map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("the output type '%s' has the wrong format", outputType)
+		}
+		for _, config := range outputConfigs {
+			if c, ok := config["enable"].(bool); ok && !c {
+				continue
+			}
+			output, err := outputRegister(config)
+			if err != nil {
+				return nil, err
+			}
+			if output == nil {
+				continue
+			}
+			list[i] = output
+			if c := config["filter"]; c != nil {
+				filter[i] = config["filter"].(map[string]interface{})
+			}
+			i++
+		}
+	}
+	return &Output{list: list, filter: filter}, nil
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	for i, item := range o.list {
+		var filteredNodes *runtime.Nodes
+		if config := o.filter[i]; config != nil {
+			filteredNodes = config.filtering(nodes)
+		} else {
+			filteredNodes = filterConfig{}.filtering(nodes)
+		}
+
+		item.Save(filteredNodes)
+	}
+}

output/all/output_test.go

@@ -89,10 +89,9 @@ func TestStart(t *testing.T) {
 	})
 	assert.Error(err)
 
-	// wrong format -> the only panic in Register
-	assert.Panics(func() {
-		Register(map[string]interface{}{
+	// wrong format
+	_, err = Register(map[string]interface{}{
 		"e": true,
 	})
-	})
+	assert.Error(err)
 }

output/internal.go

@@ -1,40 +0,0 @@
-package output
-
-import (
-	"sync"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-var quit chan struct{}
-var wg = sync.WaitGroup{}
-
-// Start workers of database
-// WARNING: Do not override this function
-//  you should use New()
-func Start(output Output, nodes *runtime.Nodes, config *runtime.Config) {
-	quit = make(chan struct{})
-	wg.Add(1)
-	go saveWorker(output, nodes, config.Nodes.SaveInterval.Duration)
-}
-
-func Close() {
-	close(quit)
-	wg.Wait()
-}
-
-// save periodically to output
-func saveWorker(output Output, nodes *runtime.Nodes, saveInterval time.Duration) {
-	ticker := time.NewTicker(saveInterval)
-	for {
-		select {
-		case <-ticker.C:
-			output.Save(nodes)
-		case <-quit:
-			wg.Done()
-			ticker.Stop()
-			return
-		}
-	}
-}

output/internal_test.go

@@ -1,57 +0,0 @@
-package output
-
-import (
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-	"github.com/stretchr/testify/assert"
-)
-
-type testConn struct {
-	Output
-	countSave int
-	sync.Mutex
-}
-
-func (c *testConn) Save(nodes *runtime.Nodes) {
-	c.Lock()
-	c.countSave++
-	c.Unlock()
-}
-func (c *testConn) Get() int {
-	c.Lock()
-	defer c.Unlock()
-	return c.countSave
-}
-
-func TestStart(t *testing.T) {
-	assert := assert.New(t)
-
-	conn := &testConn{}
-	config := &runtime.Config{
-		Nodes: struct {
-			StatePath    string           `toml:"state_path"`
-			SaveInterval runtime.Duration `toml:"save_interval"`
-			OfflineAfter runtime.Duration `toml:"offline_after"`
-			PruneAfter   runtime.Duration `toml:"prune_after"`
-			Output       map[string]interface{}
-		}{
-			SaveInterval: runtime.Duration{Duration: time.Millisecond * 10},
-		},
-	}
-	assert.Nil(quit)
-
-	Start(conn, nil, config)
-	assert.NotNil(quit)
-
-	assert.Equal(0, conn.Get())
-	time.Sleep(time.Millisecond * 12)
-	assert.Equal(1, conn.Get())
-
-	time.Sleep(time.Millisecond * 12)
-	Close()
-	assert.Equal(2, conn.Get())
-
-}

output/meshviewer-ffrgb/meshviewer.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 

output/meshviewer-ffrgb/meshviewer_test.go

@@ -11,7 +11,7 @@ import (
 func TestTransform(t *testing.T) {
 	assert := assert.New(t)
 
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 	nodes.AddNode(&runtime.Node{
 		Online: true,
 		Nodeinfo: &data.NodeInfo{

output/meshviewer-ffrgb/struct.go

@@ -3,7 +3,7 @@ package meshviewerFFRGB
 import (
 	"time"
 
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 

output/meshviewer-ffrgb/struct_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestRegister(t *testing.T) {
 	assert := assert.New(t)
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 	node := NewNode(nodes, &runtime.Node{
 		Nodeinfo: &data.NodeInfo{
 			Owner: &data.Owner{

output/meshviewer/graph_test.go

@@ -33,7 +33,7 @@ func TestGenerateGraph(t *testing.T) {
 
 func testGetNodesByFile(files ...string) *runtime.Nodes {
 
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 
 	for _, file := range files {
 		node := testGetNodeByFile(file)

output/meshviewer/node.go

@@ -2,7 +2,7 @@ package meshviewer
 
 import (
 	"github.com/FreifunkBremen/yanic/data"
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 )
 
 // Node struct

output/meshviewer/nodes_test.go

@@ -23,7 +23,7 @@ func TestNodesV2(t *testing.T) {
 }
 
 func createTestNodes() *runtime.Nodes {
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 
 	nodeData := &runtime.Node{
 		Statistics: &data.Statistics{

output/meshviewer/nodes_v1.go

@@ -1,7 +1,7 @@
 package meshviewer
 
 import (
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 

output/meshviewer/nodes_v2.go

@@ -1,7 +1,7 @@
 package meshviewer
 
 import (
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 

output/nodelist/nodelist.go

@@ -1,7 +1,7 @@
 package nodelist
 
 import (
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 

output/nodelist/nodelist_test.go

@@ -17,7 +17,7 @@ func TestTransform(t *testing.T) {
 }
 
 func createTestNodes() *runtime.Nodes {
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 
 	nodeData := &runtime.Node{
 		Statistics: &data.Statistics{

respond/collector.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/FreifunkBremen/yanic/data"
 	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 	"github.com/FreifunkBremen/yanic/runtime"
 )
 
@@ -30,7 +30,7 @@ type Collector struct {
 }
 
 // NewCollector creates a Collector struct
-func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string , ifaces []string, port int) *Collector {
+func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string, ifaces []string, port int) *Collector {
 
 	coll := &Collector{
 		db:          db,

respond/collector_test.go

@@ -12,7 +12,7 @@ import (
 const SITE_TEST = "ffxx"
 
 func TestCollector(t *testing.T) {
-	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
 
 	collector := NewCollector(nil, nodes, []string{SITE_TEST}, []string{}, 10001)
 	collector.Start(time.Millisecond)

respond/config.go

@@ -0,0 +1,12 @@
+package respond
+
+import "github.com/FreifunkBremen/yanic/lib/duration"
+
+type Config struct {
+	Enable          bool              `toml:"enable"`
+	Synchronize     duration.Duration `toml:"synchronize"`
+	Interfaces      []string          `toml:"interfaces"`
+	Sites           []string          `toml:"sites"`
+	Port            int               `toml:"port"`
+	CollectInterval duration.Duration `toml:"collect_interval"`
+}

runtime/config.go

@@ -1,58 +0,0 @@
-package runtime
-
-import (
-	"io/ioutil"
-
-	"github.com/BurntSushi/toml"
-)
-
-//Config the config File of this daemon
-type Config struct {
-	Respondd struct {
-		Enable          bool     `toml:"enable"`
-		Synchronize     Duration `toml:"synchronize"`
-		Interfaces      []string `toml:"interfaces"`
-		Sites           []string `toml:"sites"`
-		Port            int      `toml:"port"`
-		CollectInterval Duration `toml:"collect_interval"`
-	}
-	Webserver struct {
-		Enable  bool   `toml:"enable"`
-		Bind    string `toml:"bind"`
-		Webroot string `toml:"webroot"`
-	}
-	Nodes struct {
-		StatePath    string   `toml:"state_path"`
-		SaveInterval Duration `toml:"save_interval"` // Save nodes periodically
-		OfflineAfter Duration `toml:"offline_after"` // Set node to offline if not seen within this period
-		PruneAfter   Duration `toml:"prune_after"`   // Remove nodes after n days of inactivity
-		Output       map[string]interface{}
-	}
-	Meshviewer struct {
-		Version   int    `toml:"version"`
-		NodesPath string `toml:"nodes_path"`
-		GraphPath string `toml:"graph_path"`
-	}
-	Database struct {
-		DeleteInterval Duration `toml:"delete_interval"` // Delete stats of nodes every n minutes
-		DeleteAfter    Duration `toml:"delete_after"`    // Delete stats of nodes till now-deletetill n minutes
-		Connection     map[string]interface{}
-	}
-}
-
-// ReadConfigFile reads a config model from path of a yml file
-func ReadConfigFile(path string) (config *Config, err error) {
-	config = &Config{}
-
-	file, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	err = toml.Unmarshal(file, config)
-	if err != nil {
-		return nil, err
-	}
-
-	return
-}

runtime/config_test.go

@@ -1,52 +0,0 @@
-package runtime
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestReadConfig(t *testing.T) {
-	assert := assert.New(t)
-
-	config, err := ReadConfigFile("../config_example.toml")
-	assert.NoError(err, "no error during reading")
-	assert.NotNil(config)
-
-	assert.True(config.Respondd.Enable)
-	assert.Equal([]string{"br-ffhb"}, config.Respondd.Interfaces)
-	assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
-
-	assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
-
-	assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration)
-
-	var meshviewer map[string]interface{}
-	var outputs []map[string]interface{}
-	outputs = config.Nodes.Output["meshviewer"].([]map[string]interface{})
-	assert.Len(outputs, 1, "more outputs are given")
-	meshviewer = outputs[0]
-	assert.Equal(int64(2), meshviewer["version"])
-	assert.Equal("/var/www/html/meshviewer/data/nodes.json", meshviewer["nodes_path"])
-
-	var influxdb map[string]interface{}
-	dbs := config.Database.Connection["influxdb"].([]map[string]interface{})
-	assert.Len(dbs, 1, "more influxdb are given")
-	influxdb = dbs[0]
-	assert.Equal(influxdb["database"], "ffhb")
-
-	var graphitedb map[string]interface{}
-	dbs = config.Database.Connection["graphite"].([]map[string]interface{})
-	assert.Len(dbs, 1, "more graphitedb are given")
-	graphitedb = dbs[0]
-	assert.Equal(graphitedb["address"], "localhost:2003")
-
-	_, err = ReadConfigFile("testdata/config_failed.toml")
-	assert.Error(err, "not unmarshalable")
-	assert.Contains(err.Error(), "Near line ")
-
-	_, err = ReadConfigFile("testdata/adsa.toml")
-	assert.Error(err, "not found able")
-	assert.Contains(err.Error(), "no such file or directory")
-}

runtime/node.go

@@ -4,7 +4,7 @@ import (
 	"net"
 
 	"github.com/FreifunkBremen/yanic/data"
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 )
 
 // Node struct

runtime/nodes.go

@@ -8,26 +8,26 @@ import (
 	"time"
 
 	"github.com/FreifunkBremen/yanic/data"
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 )
 
 // Nodes struct: cache DB of Node's structs
 type Nodes struct {
 	List          map[string]*Node  `json:"nodes"` // the current nodemap, indexed by node ID
 	ifaceToNodeID map[string]string // mapping from MAC address to NodeID
-	config        *Config
+	config        *NodesConfig
 	sync.RWMutex
 }
 
 // NewNodes create Nodes structs
-func NewNodes(config *Config) *Nodes {
+func NewNodes(config *NodesConfig) *Nodes {
 	nodes := &Nodes{
 		List:          make(map[string]*Node),
 		ifaceToNodeID: make(map[string]string),
 		config:        config,
 	}
 
-	if config.Nodes.StatePath != "" {
+	if config.StatePath != "" {
 		nodes.load()
 	}
 
@@ -130,7 +130,7 @@ func (nodes *Nodes) NodeLinks(node *Node) (result []Link) {
 
 // Periodically saves the cached DB to json file
 func (nodes *Nodes) worker() {
-	c := time.Tick(nodes.config.Nodes.SaveInterval.Duration)
+	c := time.Tick(nodes.config.SaveInterval.Duration)
 
 	for range c {
 		nodes.expire()
@@ -143,14 +143,14 @@ func (nodes *Nodes) expire() {
 	now := jsontime.Now()
 
 	// Nodes last seen before expireAfter will be removed
-	prunePeriod := nodes.config.Nodes.PruneAfter.Duration
+	prunePeriod := nodes.config.PruneAfter.Duration
 	if prunePeriod == 0 {
 		prunePeriod = time.Hour * 24 * 7 // our default
 	}
 	pruneAfter := now.Add(-prunePeriod)
 
 	// Nodes last seen within OfflineAfter are changed to 'offline'
-	offlineAfter := now.Add(-nodes.config.Nodes.OfflineAfter.Duration)
+	offlineAfter := now.Add(-nodes.config.OfflineAfter.Duration)
 
 	// Locking foo
 	nodes.Lock()
@@ -194,7 +194,7 @@ func (nodes *Nodes) readIfaces(nodeinfo *data.NodeInfo) {
 }
 
 func (nodes *Nodes) load() {
-	path := nodes.config.Nodes.StatePath
+	path := nodes.config.StatePath
 
 	if f, err := os.Open(path); err == nil { // transform data to legacy meshviewer
 		if err = json.NewDecoder(f).Decode(nodes); err == nil {
@@ -222,7 +222,7 @@ func (nodes *Nodes) save() {
 	defer nodes.RUnlock()
 
 	// serialize nodes
-	SaveJSON(nodes, nodes.config.Nodes.StatePath)
+	SaveJSON(nodes, nodes.config.StatePath)
 }
 
 // SaveJSON to path

runtime/nodes_config.go

@@ -0,0 +1,11 @@
+package runtime
+
+import "github.com/FreifunkBremen/yanic/lib/duration"
+
+type NodesConfig struct {
+	StatePath    string            `toml:"state_path"`
+	SaveInterval duration.Duration `toml:"save_interval"` // Save nodes periodically
+	OfflineAfter duration.Duration `toml:"offline_after"` // Set node to offline if not seen within this period
+	PruneAfter   duration.Duration `toml:"prune_after"`   // Remove nodes after n days of inactivity
+	Output       map[string]interface{}
+}

runtime/nodes_test.go

@@ -9,15 +9,15 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/FreifunkBremen/yanic/data"
-	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
 )
 
 func TestExpire(t *testing.T) {
 	assert := assert.New(t)
-	config := &Config{}
-	config.Nodes.OfflineAfter.Duration = time.Minute * 10
+	config := &NodesConfig{}
+	config.OfflineAfter.Duration = time.Minute * 10
 	// to get default (100%) path of testing
-	// config.Nodes.PruneAfter.Duration = time.Hour * 24 * 6
+	// config.PruneAfter.Duration = time.Hour * 24 * 6
 	nodes := &Nodes{
 		config:        config,
 		List:          make(map[string]*Node),
@@ -51,22 +51,22 @@ func TestExpire(t *testing.T) {
 func TestLoadAndSave(t *testing.T) {
 	assert := assert.New(t)
 
-	config := &Config{}
+	config := &NodesConfig{}
 	// not autoload without StatePath
 	NewNodes(config)
 
 	// Test unmarshalable /dev/null - autolead with StatePath
-	config.Nodes.StatePath = "/dev/null"
+	config.StatePath = "/dev/null"
 	nodes := NewNodes(config)
 	// Test unopen able
-	config.Nodes.StatePath = "/root/nodes.json"
+	config.StatePath = "/root/nodes.json"
 	nodes.load()
 	// works ;)
-	config.Nodes.StatePath = "testdata/nodes.json"
+	config.StatePath = "testdata/nodes.json"
 	nodes.load()
 
 	tmpfile, _ := ioutil.TempFile("/tmp", "nodes")
-	config.Nodes.StatePath = tmpfile.Name()
+	config.StatePath = tmpfile.Name()
 	nodes.save()
 	os.Remove(tmpfile.Name())
 
@@ -113,8 +113,8 @@ func TestUpdateNodes(t *testing.T) {
 func TestSelectNodes(t *testing.T) {
 	assert := assert.New(t)
 
-	config := &Config{}
-	config.Nodes.StatePath = "testdata/nodes.json"
+	config := &NodesConfig{}
+	config.StatePath = "testdata/nodes.json"
 
 	nodes := NewNodes(config)
 
@@ -139,7 +139,7 @@ func TestSelectNodes(t *testing.T) {
 
 func TestAddNode(t *testing.T) {
 	assert := assert.New(t)
-	nodes := NewNodes(&Config{})
+	nodes := NewNodes(&NodesConfig{})
 
 	nodes.AddNode(&Node{})
 	assert.Len(nodes.List, 0)

runtime/stats_test.go

@@ -54,7 +54,7 @@ func TestGlobalStats(t *testing.T) {
 }
 
 func createTestNodes() *Nodes {
-	nodes := NewNodes(&Config{})
+	nodes := NewNodes(&NodesConfig{})
 
 	nodeData := &Node{
 		Online: true,

runtime/testdata/config_failed.toml

@@ -1 +0,0 @@
-asdas

webserver/config.go

@@ -0,0 +1,7 @@
+package webserver
+
+type Config struct {
+	Enable  bool   `toml:"enable"`
+	Bind    string `toml:"bind"`
+	Webroot string `toml:"webroot"`
+}