Merge branch 'master' of ssh://git.freifunk-rhein-sieg.net:2222/Freifunk-Troisdorf/ubnt-freifunk-map-api

Finetuning API Calls

.drone.jsonnet

@@ -1,53 +0,0 @@
-local pipeline(os, arch) = {
-  kind: "pipeline",
-  name: os + "/" + arch,
-  platform: {
-    "os": os,
-    "arch": arch,
-  },
-  steps: [{
-      name: "compile " + os + "/" + arch,
-      image: "golang:1.20.3-alpine3.17",
-      environment: {
-        "GOOS": os,
-        "GOARCH": arch,
-        "CGO_ENABLED": "0",
-      },
-      commands: [
-        'go build -ldflags "-s -w -X main.version=${DRONE_TAG##v}" -trimpath -o release/' + os + "/" + arch + "/ubnt-freifunk-map-api .",
-        "tar -cvzf release/ubnt-freifunk-map-api_"+ os + "-" + arch + ".tar.gz -C release/" + os + "/" + arch + " ubnt-freifunk-map-api" 
-      ],
-    },
-    {
-      name: "gitea_release " + os + "/" + arch,
-      image: "plugins/gitea-release",
-      settings: {
-        api_key: { "from_secret": "gitea_api_key" },
-        base_url: "https://git.freifunk-rhein-sieg.net",
-        files: "release/*.tar.gz"
-      },
-      when: {
-        event: "tag"
-      },
-    },
-    {
-      name: "upload to gitea container registry",
-      image: "plugins/docker:latest",
-      settings: {
-        repo: "git.freifunk-rhein-sieg.net/freifunk-troisdorf/ubnt-freifunk-map-api",
-        registry: "git.freifunk-rhein-sieg.net",
-        username: { "from_secret": "docker_username" },
-        password: { "from_secret": "docker_password" },
-        tags: ["latest"],
-        auto_tag: true,
-      },
-      when: {
-        event: "tag"
-      },
-    },
-  ],
-};
-
-[
-  pipeline("linux", "amd64")
-]

example.config.json

@@ -5,14 +5,17 @@
         "APItoken": "UNMS API TOKEN",
         "devicesURL": "https://git.freifunk-rhein-sieg.net/Freifunk-Troisdorf/ubnt-freifunk-map-api/raw/branch/master/example.devices.json"
     },
-    "unifi": {
+    "unifi": [
+        {
+        "name": "Unifi Freifunk Troisdorf",
         "enabled": false,
         "displayusers": true,
         "APIUrl": "https://unifi.freifunk-troisdorf.de",
         "user": "APIuser",
         "password": "PASSWORD",
         "ucDevicesURL": "https://git.freifunk-rhein-sieg.net/Freifunk-Troisdorf/ubnt-freifunk-map-api/raw/branch/master/example.ucDevices.json"
-    },
+        }
+    ],
     "meshviewer": {
         "enabled": false,
         "files": [

main.go

@@ -9,21 +9,28 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"os"
+	"sync"
 	"time"
 
 	_ "git.nils.zone/nils/prettify"
 )
 
 const (
-	iso8601 = "2006-01-02T15:04:05-0700"
+	iso8601       = "2006-01-02T15:04:05-0700"
+	fetchInterval = 1 * time.Minute
 )
 
 // flags
+var (
+	lastFetchTime time.Time
+	cacheMutex    sync.Mutex
+	cacheNodes    []node
+	cacheLinks    []link
+)
 var configPath = flag.String("configPath", "config.json", "Path to config.json")
 var version = "development"
 var delay time.Duration = 60 * time.Second
 var conf = loadconfig(*configPath)
-var ucDev = getDevices(conf.Unifi.UCDevicesURL)
 
 func main() {
 	log.Printf("starting version %s...\n", version)
@@ -55,27 +62,35 @@ func processAPIs() error {
 	var nodes []node
 	var links []link
 
-	if conf.Unms.Enabled {
+	if conf.UISP.Enabled {
-		log.Println("Processing UNMS")
+		log.Println("Processing UISP")
-		unmsNodes, unmsLinks, err := processUNMSAPI()
+		//Process UISP RiFu Nodes
+		uispNodes, uispLinks, err := processUISPRiFu()
 		if err != nil {
 			return err
 		}
-		unmsRouters, err := processUNMSAPIRouter()
+		//Process UISP Routers (like EDGE Router)
+		uispRouters, err := processUISPRouter()
 		if err != nil {
 			return err
 		}
-		nodes = append(nodes, unmsNodes...)
+		nodes = append(nodes, uispNodes...)
-		nodes = append(nodes, unmsRouters...)
+		nodes = append(nodes, uispRouters...)
-		links = append(links, unmsLinks...)
+		links = append(links, uispLinks...)
 	}
-	if conf.Unifi.Enabled {
+	if len(conf.Unifi) > 0 {
-		log.Println("Processing Unifi")
+		log.Println("Anazahl der Unifi Server:", len(conf.Unifi))
-		ucNodes, _, err := processUcAPIs()
+		for i := range conf.Unifi {
-		if err != nil {
+			if conf.Unifi[i].Enabled {
-			return err
+				log.Println("Processing Unifi-Server: ", conf.Unifi[i].Name)
+				//Process Unifi Nodes
+				unifiNodes, _, err := processUnifiAPI(i)
+				if err != nil {
+					return err
+				}
+				nodes = append(nodes, unifiNodes...)
+			}
 		}
-		nodes = append(nodes, ucNodes...)
 	}
 	if conf.Meshviewer.Enabled {
 		log.Println("Processing Meshviewer")
@@ -85,6 +100,7 @@ func processAPIs() error {
 	}
 	if conf.Gateways.Enabled {
 		log.Println("Processing Gateways")
+		//Process Static Gateways from Json
 		gwNodes := processGateways()
 		nodes = append(nodes, gwNodes...)
 	}
@@ -101,12 +117,7 @@ func processAPIs() error {
 	if err := o.writeToFile(); err != nil {
 		log.Fatalln(err)
 	}
-	// get outages to serve as .csv
+
-	l := getUNMSLogs()
-	err := writeOutagesToCSV(l)
-	if err != nil {
-		log.Println("Error writing outages.csv")
-	}
 	// we're done here
 	log.Println("...done")
 	return nil

meshviewer.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"log"
+	"time"
 )
 
 func getMeshviewerJSON(url string) (mvDevices, error) {
@@ -129,17 +130,31 @@ func addmvDevices(d mvDevices) ([]node, []link) {
 }
 
 func getMeshviewer() ([]node, []link) {
+	cacheMutex.Lock()
+	defer cacheMutex.Unlock()
+
+	// Überprüfen, ob die Daten kürzlich aktualisiert wurden
+	if time.Since(lastFetchTime) < fetchInterval {
+		return cacheNodes, cacheLinks
+	}
+
 	var nodes []node
 	var links []link
 
 	for i := range conf.Meshviewer.Files {
 		m, err := getMeshviewerJSON(conf.Meshviewer.Files[i].URL)
 		if err != nil {
-			return nodes, links
+			return cacheNodes, cacheLinks
 		}
 		mvNodes, mvLinks := addmvDevices(m)
 		nodes = append(nodes, mvNodes...)
 		links = append(links, mvLinks...)
 	}
-	return nodes, links
+
+	// Cache aktualisieren
+	cacheNodes = nodes
+	cacheLinks = links
+	lastFetchTime = time.Now()
+
+	return cacheNodes, cacheLinks
 }

outages.go

@@ -1,37 +0,0 @@
-package main
-
-import (
-	"encoding/csv"
-	"log"
-	"os"
-)
-
-func getUNMSLogs() UNMSLogResponse {
-
-	var l UNMSLogResponse
-	log.Println("Get Outages from UNMS")
-	err := UnmsCallAPI("/outages?count=100&page=1&type=outage", &l)
-	if err != nil {
-		log.Fatalln("Error calling Outages API")
-	}
-	return l
-}
-
-func writeOutagesToCSV(l UNMSLogResponse) error {
-	csvFile, err := os.Create("output/outages.csv")
-	if err != nil {
-		return err
-	}
-	writer := csv.NewWriter(csvFile)
-
-	for _, o := range l.Items {
-		var row []string
-		row = append(row, o.StartTime.Format("02.01.2006 15:04:05"))
-		row = append(row, o.EndTime.Format("02.01.2006 15:04:05"))
-		row = append(row, o.Site.Name)
-		row = append(row, o.Device.DisplayName)
-		writer.Write(row)
-	}
-	writer.Flush()
-	return nil
-}

staticDevices.go

@@ -57,7 +57,9 @@ func processGateways() []node {
 		if err != nil {
 			log.Fatalln("Error: ", err)
 		}
-		sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		if conf.General.InfluxEnabled {
+			sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		}
 
 		//Build Nodes
 		nodes = append(nodes, node{

types.go

@@ -11,25 +11,19 @@ import (
 
 type config struct {
 	General struct {
+		InfluxEnabled      bool   `json:"influx_enabled"`
 		FreifunkInfluxPort string `json:"freifunk_influx_port"`
 		ProxmoxInfluxPort  string `json:"proxmox_influx_port"`
 		InfluxURL          string `json:"influx_url"`
 	}
-	Unms struct {
+	UISP struct {
 		Enabled    bool   `json:"enabled"`
 		UnmsAPIURL string `json:"unmsAPIUrl"`
 		APItoken   string `json:"APItoken"`
 		DevicesURL string `json:"devicesURL"`
 		RouterURL  string `json:"routerURL"`
 	} `json:"unms"`
-	Unifi struct {
+	Unifi      []UnifiServer `json:"unifi"`
-		Enabled      bool   `json:"enabled"`
-		DisplayUsers bool   `json:"displayusers"`
-		APIURL       string `json:"APIUrl"`
-		User         string `json:"user"`
-		Password     string `json:"password"`
-		UCDevicesURL string `json:"ucDevicesURL"`
-	} `json:"unifi"`
 	Meshviewer struct {
 		Enabled bool `json:"enabled"`
 		Files   []struct {
@@ -43,6 +37,16 @@ type config struct {
 	} `json:"gateways"`
 }
 
+type UnifiServer struct {
+	Name         string `json:"name"`
+	Enabled      bool   `json:"enabled"`
+	DisplayUsers bool   `json:"displayusers"`
+	APIURL       string `json:"APIUrl"`
+	User         string `json:"user"`
+	Password     string `json:"password"`
+	UCDevicesURL string `json:"ucDevicesURL"`
+}
+
 type device struct {
 	Name           string `json:"name"`
 	FQDN           string `json:"fqdn"`

unifi.go

@@ -15,14 +15,14 @@ import (
 )
 
 // Unifi Controller API processing
-func processUcAPIs() ([]node, []link, error) {
+func processUnifiAPI(s int) ([]node, []link, error) {
 	//get list of Unifi devices to display
 	var nodes []node
 	var links []link
-	d := getDevices(conf.Unifi.UCDevicesURL)
+	d := getDevices(conf.Unifi[s].UCDevicesURL)
 
 	//call Unifi Controller
-	ucAPI := UnifiNewAPI(conf.Unifi.User, conf.Unifi.Password, conf.Unifi.APIURL)
+	ucAPI := UnifiNewAPI(conf.Unifi[s].User, conf.Unifi[s].Password, conf.Unifi[s].APIURL)
 	//login
 	if err := ucAPI.ucLogin(); err != nil {
 		return nil, nil, err
@@ -49,12 +49,14 @@ func processUcAPIs() ([]node, []link, error) {
 				currentJSONDevice = jsonDevice
 			}
 		}
+
 		if isRemoteMACpublished(jsonDevice.MAC, d.Devices) {
 			//hier muss gecheckt werden ob der link valide ist
 			if checkMeshviewerLink(jsonDevice.LinkedTo) {
 				links = UnifiAddLink(jsonDevice, links)
 			}
 		}
+
 		isOnline := currentDevice.State == 1
 		var load float64
 		var mem float64
@@ -78,7 +80,7 @@ func processUcAPIs() ([]node, []link, error) {
 
 		var model = lookupModels(currentDevice.Model)
 		var clients int
-		if conf.Unifi.DisplayUsers {
+		if conf.Unifi[s].DisplayUsers {
 			clients = currentDevice.Users
 		}
 
@@ -125,43 +127,47 @@ func processUcAPIs() ([]node, []link, error) {
 			log.Fatalln("Error: ", err)
 		}
 
-		sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		if conf.General.InfluxEnabled {
+			sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		}
 
 		// INFLUX STOP
-
+		//log.Println(currentDevice.Mac)
-		nodes = append(nodes, node{
+		if currentDevice.Mac != "" {
-			Firstseen:      "0",
+			nodes = append(nodes, node{
-			Lastseen:       time.Unix(int64(currentDevice.LastSeen), 0).Format(iso8601),
+				Firstseen:      "0",
-			IsOnline:       itob(currentDevice.State),
+				Lastseen:       time.Unix(int64(currentDevice.LastSeen), 0).Format(iso8601),
-			IsGateway:      false,
+				IsOnline:       itob(currentDevice.State),
-			Clients:        clients,
+				IsGateway:      false,
-			ClientsWifi24:  0,
+				Clients:        clients,
-			ClientsWifi5:   0,
+				ClientsWifi24:  0,
-			ClientsOther:   clients,
+				ClientsWifi5:   0,
-			RootFSUsage:    0,
+				ClientsOther:   clients,
-			LoadAVG:        load / 100,
+				RootFSUsage:    0,
-			MemoryUsage:    mem / 100,
+				LoadAVG:        load / 100,
-			Uptime:         time.Now().Add(-1 * time.Second * time.Duration(currentDevice.Uptime)).Format(iso8601),
+				MemoryUsage:    mem / 100,
-			GatewayNexthop: currentJSONDevice.GatewayNexthop,
+				Uptime:         time.Now().Add(-1 * time.Second * time.Duration(currentDevice.Uptime)).Format(iso8601),
-			Gateway:        currentJSONDevice.Gateway,
+				GatewayNexthop: currentJSONDevice.GatewayNexthop,
-			Location:       &currentJSONDevice.Location,
+				Gateway:        currentJSONDevice.Gateway,
-			NodeID:         strings.ReplaceAll(currentDevice.Mac, ":", ""),
+				Location:       &currentJSONDevice.Location,
-			MAC:            currentDevice.Mac,
+				NodeID:         strings.ReplaceAll(currentDevice.Mac, ":", ""),
-			Adresses:       []string{currentDevice.IP},
+				MAC:            currentDevice.Mac,
-			Domain:         currentJSONDevice.Domain,
+				Adresses:       []string{currentDevice.IP},
-			Hostname:       "[Unifi] " + currentDevice.Name,
+				Domain:         currentJSONDevice.Domain,
-			Owner:          "Freifunk Rhein-Sieg",
+				Hostname:       "[Unifi] " + currentDevice.Name,
-			Firmware: firmware{
+				Owner:          "Freifunk Rhein-Sieg",
-				Base:    "Ubiquiti - Stock",
+				Firmware: firmware{
-				Release: currentDevice.Version,
+					Base:    "Ubiquiti - Stock",
-			},
+					Release: currentDevice.Version,
-			Autoupdater: autoupdater{
+				},
-				Enabled: false,
+				Autoupdater: autoupdater{
-				Branch:  "stable",
+					Enabled: false,
-			},
+					Branch:  "stable",
-			NProc: 1,
+				},
-			Model: model,
+				NProc: 1,
-		})
+				Model: model,
+			})
+		}
 	}
 	return nodes, links, err
 }
@@ -266,9 +272,12 @@ func UnifiAddLink(dev device, links []link) []link {
 }
 
 func findNodeID(NodeID string) bool {
-	for i := range ucDev.Devices {
+	for s := range conf.Unifi {
-		if ucDev.Devices[i].GatewayNexthop == NodeID {
+		ucDev := getDevices(conf.Unifi[s].UCDevicesURL)
-			return true
+		for i := range ucDev.Devices {
+			if ucDev.Devices[i].GatewayNexthop == NodeID {
+				return true
+			}
 		}
 	}
 	return false

unms.go

@@ -17,12 +17,12 @@ import (
 )
 
 // UNMS API processing (Richtfunk)
-func processUNMSAPI() ([]node, []link, error) {
+func processUISPRiFu() ([]node, []link, error) {
 	// Variables for runtime
 	var links []link
 	var nodes []node
 
-	d := getDevices(conf.Unms.DevicesURL)
+	d := getDevices(conf.UISP.DevicesURL)
 
 	// API CALL 1 (get Device overview)
 	log.Println("Starting UISP API Crawler for Rifu devices")
@@ -33,6 +33,7 @@ func processUNMSAPI() ([]node, []link, error) {
 	}
 
 	for i := range d.Devices {
+		time.Sleep(time.Second)
 		var dev unifiAPIResponse
 		var currentDevice device
 		for j := range u {
@@ -103,10 +104,11 @@ func processUNMSAPI() ([]node, []link, error) {
 	return nodes, links, nil
 }
 
-func processUNMSAPIRouter() ([]node, error) {
+func processUISPRouter() ([]node, error) {
+	time.Sleep(time.Second)
 	// Variables for runtime
 	var nodes []node
-	d := getDevices(conf.Unms.RouterURL)
+	d := getDevices(conf.UISP.RouterURL)
 
 	// API CALL 1, get all devices list from UNMS
 	log.Println("Get all Routers from UISP")
@@ -192,8 +194,9 @@ func processUNMSAPIRouter() ([]node, error) {
 		if err != nil {
 			log.Fatalln("Error: ", err)
 		}
-
+		if conf.General.InfluxEnabled {
-		sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+			sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		}
 		// Get info from json file (static)
 		nodes = append(nodes, node{
 			Firstseen:      dev.Overview.CreatedAt.Format(iso8601),
@@ -233,19 +236,21 @@ func processUNMSAPIRouter() ([]node, error) {
 }
 
 func UnmsCallAPI(url string, i any) error {
-	request, err := http.NewRequest(http.MethodGet, conf.Unms.UnmsAPIURL+url, nil)
+	time.Sleep(time.Second)
+	request, err := http.NewRequest(http.MethodGet, conf.UISP.UnmsAPIURL+url, nil)
 	if err != nil {
-		return errors.New(fmt.Sprint("can't set request", conf.Unms.UnmsAPIURL+url))
+		return errors.New(fmt.Sprint("can't set request", conf.UISP.UnmsAPIURL+url))
 	}
-	log.Println(conf.Unms.UnmsAPIURL + url)
+	//log.Println(conf.UISP.UnmsAPIURL + url)
-	request.Header.Set("x-auth-token", conf.Unms.APItoken)
+	request.Header.Set("x-auth-token", conf.UISP.APItoken)
 	client := &http.Client{}
 	response, err := client.Do(request)
 	if err != nil {
-		return fmt.Errorf("can't get request %s with x-auth-token %s", conf.Unms.UnmsAPIURL+url, conf.Unms.APItoken)
+		return fmt.Errorf("can't get request %s with x-auth-token %s", conf.UISP.UnmsAPIURL+url, conf.UISP.APItoken)
 	}
 	if response.StatusCode != 200 {
-		log.Fatalln("Can't call UNMS API, check token and URL. HTTP Status: ", response.StatusCode)
+		log.Println("Can't call UNMS API, check token and URL. Skipping device. HTTP Status: ", response.StatusCode)
+		return nil
 	}
 	data, err := ioutil.ReadAll(response.Body)
 	defer response.Body.Close()