Merge branch 'master' of ssh://git.freifunk-rhein-sieg.net:2222/Freifunk-Troisdorf/ubnt-freifunk-map-api

Finetuning API Calls

.drone.jsonnet

@@ -1,81 +0,0 @@
-local pipeline(os, arch) = {
-  kind: "pipeline",
-  name: os + "/" + arch,
-  platform: {
-    "os": os,
-    "arch": arch,
-  },
-  steps: [{
-      name: "compile " + os + "/" + arch,
-      image: "golang:1.15.6-alpine3.12",
-      environment: {
-        "GOOS": os,
-        "GOARCH": arch,
-        "CGO_ENABLED": "0",
-      },
-      commands: [
-        'go build -ldflags "-s -w -X main.version=${DRONE_TAG##v}" -trimpath -o release/' + os + "/" + arch + "/ubnt-freifunk-map-api .",
-        "tar -cvzf release/ubnt-freifunk-map-api_"+ os + "-" + arch + ".tar.gz -C release/" + os + "/" + arch + " ubnt-freifunk-map-api" 
-      ],
-    },
-    {
-      name: "gitea_release " + os + "/" + arch,
-      image: "plugins/gitea-release",
-      settings: {
-        api_key: { "from_secret": "gitea_api_key" },
-        base_url: "https://git.freifunk-rhein-sieg.net",
-        files: "release/*.tar.gz"
-      },
-      when: {
-        event: "tag"
-      },
-    },
-    {
-      name: "upload to docker hub " + os + "/" + arch,
-      image: "plugins/docker:" + os + "-" + arch,
-      settings: {
-        repo: "fftdf/ffmap-ubnt-api",
-        username: { "from_secret": "docker_username" },
-        password: { "from_secret": "docker_password" },
-        auto_tag: true,
-        auto_tag_suffix: os + "-" + arch
-      },
-      when: {
-        event: "tag"
-      },
-    },
-  ],
-};
-
-local manifest() = {
-  kind: "pipeline",
-  type: "docker",
-  name: "manifest",
-  depends_on: ["linux/amd64"],
-  when: {
-    event: "tag"
-  },
-
-  steps: [
-    {
-      name: "publish",
-      image: "plugins/manifest",
-      settings: {
-        auto_tag: true,
-        ignore_missing: true,
-        spec: "manifest.yml",
-        username: { "from_secret": "docker_username" },
-        password: { "from_secret": "docker_password" },
-      },
-      when: {
-        event: "tag"
-      },
-    },
-  ],
-};
-
-[
-  pipeline("linux", "amd64"),
-  // pipeline("linux", "arm64"),
-  manifest()
-]

.gitignore

@@ -14,3 +14,4 @@
 output/*
 
 ubnt-freifunk-map-api
+config.json

.woodpecker.yml

@@ -0,0 +1,25 @@
+---
+platform: linux/arm64
+
+pipeline:
+  build:
+    image: golang
+    environment:
+      - GOOS=linux
+      - GOARCH=amd64
+    commands:
+      - go build -ldflags "-s -w -X main.version=${CI_COMMIT_TAG}" -trimpath -o release/ubnt-freifunk-map-api .
+
+  docker:
+    image: woodpeckerci/plugin-docker-buildx
+    settings:
+      platforms: linux/amd64
+      registry: git.freifunk-rhein-sieg.net
+      repo: git.freifunk-rhein-sieg.net/freifunk-troisdorf/ubnt-freifunk-map-api
+      username:
+        from_secret: gitea_user
+      password:
+        from_secret: gitea_token
+      tags: ${CI_COMMIT_TAG}
+    when:
+      - branch: master

Dockerfile

@@ -1,8 +1,7 @@
 FROM alpine:3.12.3
 WORKDIR /opt/
 
-ADD ./release/*/*/ubnt-freifunk-map-api /opt/ubnt-freifunk-map-api
+ADD ./release/ubnt-freifunk-map-api /opt/ubnt-freifunk-map-api
-ADD ./devices.json /opt/devices.json
 RUN chmod +x /opt/ubnt-freifunk-map-api
 
 EXPOSE 3000

README.md

@@ -1,3 +1,104 @@
-# ubnt-freifunk-map-api
+# Freifunk Meshviewer Unifi Access Points und Richtfunkstrecken import
 
-Use the API of UNMS and Unifi Controller to display Unifi Hardware in Freifunk Map
+Dieses tool Importiert Nodes für die Freifunk Map aus den APIs UISP (Richtfunk) & Unifi (Access Points).
+Ebenfalls ist der Import statischer devices möglich. Da diese alle in unerem Proxmox cluster laufen, werden Statistikdaten aus der Proxmox InfluxDB geholt.
+
+Alle Config dateien müssen per http erreichbar sein (z.B. in einem Git)
+Für Troisdorf werden diese Dateien hier gepflegt: https://git.freifunk-rhein-sieg.net/Freifunk-Troisdorf/ubnt-api-devices
+Für die Rhein-Sieg-Map hier: https://git.freifunk-rhein-sieg.net/Freifunk-Rhein-Sieg/ubnt-api-devices
+
+## Config
+
+### Unifi Access Points (unifi_devices.json)
+
+In der Datei unifi_devices.json können die Access Points gepflegt werden, die auf der Freifunk Map erscheinen sollen.
+
+Hierzu muss die Datei im json Format erweitert werden.
+
+Für jedes Gerät muss dieser Block angelegt werden:
+```json
+{
+    "name": "ROUTERNAME",
+    "mac": "00:00:00:00:00",
+    "gateway_nexthop": "1234567890",
+    "gateway": "1234567890",
+    "linked_to": "18:e8:29:24:17:0a",
+    "domain": "unifi",
+    "location": {
+      "longitude":7.148406208,
+      "latitude":50.817093402
+},
+```
+Erklärung:
+* name:   Dient nur zur Wiedererkennung. Auf der Map erscheint der Name aus dem Unifi Controller
+* mac:    Die MAC Adresse des Access Points. Zu finden im Controller. (Statistik Übersicht in der Geräteansicht)
+* gateway_nexthop:    Die Node ID (MAC ohne :) des Freifunk Routers an den der AP angeschlossen ist.
+* gateway:    Im Normalfall die NodeID des Supernodes (zu finden in der MAP)
+* linked_to: (Optional) Die MAC Adresse des Routers an dem der AP angeschlossen ist. Normalerweise gateway_nexthop mit Doppelpunkten. Wenn nicht gesetzt wird kein Link auf der Map angezeigt.
+* domain: Die Domain in der sich der AP befindet. (tdf, inn, flu)
+
+### UISP Richtfunkstrecken
+
+In der Datei rifu_devices.json können die Richtfunkstrecken gepflegt werden, die auf der Freifunk Map erscheinen sollen.
+
+```json
+{
+  "name": "Rathaus2Bahnhof",
+  "mac": "18:E8:29:8E:C6:4D",
+  "gateway_nexthop": "18e8292f7de6",
+  "gateway": "a28cae6ff604",
+  "domain": "rifu",
+  "location": {
+    "longitude":7.148406208,
+    "latitude":50.817093402
+  }
+}
+```
+Erklärung:
+* name:   Dient nur zur Wiedererkennung. Auf der Map erscheint der Name aus dem UNMS Controller
+* mac:    Die MAC Adresse des Gerätes. Zu finden im Controller.
+* gateway_nexthop:    Die Node ID (MAC ohne :) des Freifunk Routers an dem das Gerät angeschlossen ist.
+* gateway:    Im Normalfall die NodeID des Supernodes (zu finden in der MAP)
+* domain: Die Domain in der sich der AP befindet. (tdf, inn, flu)
+
+### UISP Router
+
+In dieser datei werden die Router (meist ER-X) gepflegt. Diese Daten werden dann ebenfalls aus der UISP API Importiert.
+
+```json
+{
+        "name": "Rathaus Uplink",
+        "mac": "18:e8:29:ad:9a:34",
+        "gateway_nexthop": "18e8292f7de6",
+        "gateway": "a28cae6ff604",
+        "domain": "tdf",
+        "location": {
+          "longitude":7.149406208,
+          "latitude":50.817093402
+        }
+      },
+```
+
+### Gateways.json
+
+Hier werden Statische Geräte eingetragen die auf dem Proxmox Cluster laufen.
+
+```json
+{
+      "name": "VPN01",
+      "fqdn": "vpn01.fftdf.de",
+      "mac": "00:00:00:00:00:01",
+      "domain": "VPN1",
+      "adresses": ["5.9.220.114"]
+    },
+```
+
+### Config.json
+
+Es gibt 3 Module die Ein/Ausgeschatet werden können:
+* UNMS
+* Unifi
+* Meshviewer
+* Gateways
+
+Die Funktion Meshviewer importiert die vorhandenen meshviewer.json und manipuliert dort die Userzahlen. Sobald ein Access Point einen Node aus einer Meshviwer.json als "gateway_nexthop" eingetragen hat, werden die Clients an dem verbundenen Access Point und nicht mehr am Offloader angezeigt.

example.config.json

@@ -0,0 +1,36 @@
+{
+    "unms": {
+        "enabled": false,
+        "unmsAPIUrl": "https://uisp.freifunk-troisdorf.de/v2.1",
+        "APItoken": "UNMS API TOKEN",
+        "devicesURL": "https://git.freifunk-rhein-sieg.net/Freifunk-Troisdorf/ubnt-freifunk-map-api/raw/branch/master/example.devices.json"
+    },
+    "unifi": [
+        {
+        "name": "Unifi Freifunk Troisdorf",
+        "enabled": false,
+        "displayusers": true,
+        "APIUrl": "https://unifi.freifunk-troisdorf.de",
+        "user": "APIuser",
+        "password": "PASSWORD",
+        "ucDevicesURL": "https://git.freifunk-rhein-sieg.net/Freifunk-Troisdorf/ubnt-freifunk-map-api/raw/branch/master/example.ucDevices.json"
+        }
+    ],
+    "meshviewer": {
+        "enabled": false,
+        "files": [
+            {
+            "name": "tdf4",
+            "URL": "http://4.fftdf.de/meshviewer.json"
+            },
+            {
+            "name": "tdf5",
+            "URL": "http://5.fftdf.de/meshviewer.json"
+            },
+            {
+            "name": "tdf6",
+            "URL": "http://6.fftdf.de/meshviewer.json"
+            }
+        ]
+    }
+}

example.devices.router.json

@@ -0,0 +1,26 @@
+{
+  "devices":[
+    {
+      "name": "Rathaus2Bahnhof",
+      "mac": "18:E8:29:8E:C6:4D",
+      "gateway_nexthop": "18e8292f7de6",
+      "gateway": "a28cae6ff604",
+      "domain": "other",
+      "location": {
+        "longitude":7.148406208,
+        "latitude":50.817093402
+      }
+    },
+    {
+      "name": "Bahnhof2Rathaus",
+      "mac": "18:e8:29:dc:c3:7e",
+      "gateway_nexthop": "18e8292f7de6",
+      "gateway": "a28cae6ff604",
+      "domain": "other",
+      "location": {
+        "longitude":7.150436640,
+        "latitude":50.814456507
+      }
+    }
+  ]
+}

example.json

@@ -1,92 +0,0 @@
-{
-  "timestamp": "2021-01-01T21:16:19+0100",
-  "nodes": [
-    {
-      "firstseen": "2020-01-20T17:38:03+0000",
-      "lastseen": "2021-01-01T20:16:07+0000",
-      "is_online": true,
-      "is_gateway": false,
-      "clients": 0,
-      "clients_wifi24": 0,
-      "clients_wifi5": 0,
-      "clients_other": 0,
-      "rootfs_usage": 0,
-      "loadavg": 0.02,
-      "memory_usage": 0.66,
-      "uptime": "2020-11-22T14:02:13+0000",
-      "gateway_nexthop": "18e8292f7de6",
-      "gateway": "a28cae6ff604",
-      "location": {
-        "longitude": 7.148406208,
-        "latitude": 50.817093402
-      },
-      "node_id": "18e8298ec64d",
-      "mac": "18:e8:29:8e:c6:4d",
-      "addresses": [
-        "10.188.41.164"
-      ],
-      "domain": "rifu",
-      "hostname": "[RiFu] Port 5 - zu Bahnhof",
-      "owner": "Freifunk Rhein-Sieg",
-      "firmware": {
-        "base": "Ubiquiti - Stock",
-        "release": "8.7.1"
-      },
-      "autoupdater": {
-        "enabled": false,
-        "branch": "stable"
-      },
-      "nproc": 1,
-      "model": "NBE-5AC-Gen2"
-    },
-    {
-      "firstseen": "2020-05-05T20:08:20+0000",
-      "lastseen": "2021-01-01T20:15:52+0000",
-      "is_online": true,
-      "is_gateway": false,
-      "clients": 0,
-      "clients_wifi24": 0,
-      "clients_wifi5": 0,
-      "clients_other": 0,
-      "rootfs_usage": 0,
-      "loadavg": 0.27,
-      "memory_usage": 0.68,
-      "uptime": "2020-10-24T21:47:33+0000",
-      "gateway_nexthop": "18e8292f7de6",
-      "gateway": "a28cae6ff604",
-      "location": {
-        "longitude": 7.15043664,
-        "latitude": 50.814456507
-      },
-      "node_id": "18e829dcc37e",
-      "mac": "18:e8:29:dc:c3:7e",
-      "addresses": [
-        "10.188.47.143"
-      ],
-      "domain": "rifu",
-      "hostname": "[RiFu] Bahnhof-TDF",
-      "owner": "Freifunk Rhein-Sieg",
-      "firmware": {
-        "base": "Ubiquiti - Stock",
-        "release": "8.7.1"
-      },
-      "autoupdater": {
-        "enabled": false,
-        "branch": "stable"
-      },
-      "nproc": 1,
-      "model": "NBE-5AC-Gen2"
-    }
-  ],
-  "links": [
-    {
-      "type": "wifi",
-      "source": "18e8298ec64d",
-      "target": "18e829dcc37e",
-      "source_tq": 0.315,
-      "target_tq": 0.315,
-      "source_addr": "18:e8:29:8e:c6:4d",
-      "target_addr": "18:e8:29:dc:c3:7e"
-    }
-  ]
-}

example.ucDevices.json

@@ -0,0 +1,26 @@
+{
+    "devices":[
+      {
+        "name": "AM-01",
+        "mac": "18:e8:29:9c:90:ae",
+        "gateway_nexthop": "18e82924170a",
+        "gateway": "a28cae6ff604",
+        "linked_to": "18:e8:29:24:17:0a",
+        "location": {
+          "longitude":7.122931927,
+          "latitude":50.818885422
+        }
+      },
+      {
+        "name": "AM-02",
+        "mac": "18:e8:29:9c:96:ba",
+        "gateway_nexthop": "18e82924170a",
+        "gateway": "a28cae6ff604",
+        "linked_to": "18:e8:29:24:17:0a",
+        "location": {
+          "longitude":7.122777700,
+          "latitude":50.819022680
+        }
+      }
+    ]
+  }

go.mod

@@ -1,3 +1,17 @@
 module git.freifunk-rhein-sieg.net/Freifunk-Troisdorf/ubnt-freifunk-map-api
 
-go 1.16
+go 1.20
+
+require (
+	git.nils.zone/nils/prettify v0.0.4
+	github.com/fatih/structs v1.1.0
+	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
+)
+
+require (
+	github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect
+	github.com/fatih/color v1.9.0 // indirect
+	github.com/mattn/go-colorable v0.1.4 // indirect
+	github.com/mattn/go-isatty v0.0.11 // indirect
+	golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 // indirect
+)

go.sum

@@ -0,0 +1,20 @@
+git.nils.zone/nils/prettify v0.0.4 h1:y014Ejp0yhFAc9vu5U0nJRFxtXN6k6eWW6LiUkwn4NQ=
+git.nils.zone/nils/prettify v0.0.4/go.mod h1:q4ydEhSvXuywHe5U6uSEoEeS9f2Upz/n8z0MJylLz5c=
+github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4=
+github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w=
+github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e h1:0aewS5NTyxftZHSnFaJmWE5oCCrj4DyEXkAiMa1iZJM=
+github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
+github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c h1:qSHzRbhzK8RdXOsAdfDgO49TtqC1oZ+acxPrkfTxcCs=
+github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

influx.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+
+	client "github.com/influxdata/influxdb1-client/v2"
+)
+
+// Create InfluxDB Client
+func influxDBClient(port string) client.Client {
+	c, err := client.NewHTTPClient(client.HTTPConfig{
+		Addr: conf.General.InfluxURL + ":" + port,
+	})
+	if err != nil {
+		log.Fatalln("Error: ", err)
+	}
+	return c
+}
+
+// Get a single Datapoint from InfluxDB
+func getInfluxDataPoint(dp string, h string, p string) float64 {
+
+	//Build the Query
+	query := "SELECT last(" + dp + ") FROM system WHERE host = '" + h + "'"
+
+	c := influxDBClient(p)
+	q := client.NewQuery(query, "udp", "s")
+	response, err := c.Query(q)
+	if err != nil {
+		log.Println("Influx query error!")
+	}
+	res := 0.0
+	if len(response.Results) > 0 {
+		res, err := response.Results[0].Series[0].Values[0][1].(json.Number).Float64()
+		if err != nil {
+			log.Println("Error in type conversion")
+		}
+		return res
+	}
+	return res
+}
+
+// Send Datapoints to InfluxDB, point map and InfluxDB Port needed
+func sendInfluxBatchDataPoint(point *client.Point, influxPort string) {
+
+	// Open connection to InfluxDB
+	bp, err := client.NewBatchPoints(client.BatchPointsConfig{
+		Database:  "freifunk",
+		Precision: "s",
+	})
+	if err != nil {
+		log.Fatalln("Error: ", err)
+	}
+
+	bp.AddPoint(point)
+	c := influxDBClient(influxPort)
+	err = c.Write(bp)
+	if err != nil {
+		log.Fatal(err)
+	}
+	//
+}

main.go

@@ -2,202 +2,173 @@ package main
 
 import (
 	"encoding/json"
-	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
+	"net/http/cookiejar"
 	"os"
-	"strings"
+	"sync"
 	"time"
+
+	_ "git.nils.zone/nils/prettify"
 )
 
-// types
-
 const (
-	baseURL = "https://unifi.freifunk-troisdorf.de/v2.1"
+	iso8601       = "2006-01-02T15:04:05-0700"
-	iso8601 = "2006-01-02T15:04:05-0700"
+	fetchInterval = 1 * time.Minute
 )
 
 // flags
-var token = flag.String("token", "", "Defines the x-auth-token")
+var (
+	lastFetchTime time.Time
+	cacheMutex    sync.Mutex
+	cacheNodes    []node
+	cacheLinks    []link
+)
+var configPath = flag.String("configPath", "config.json", "Path to config.json")
 var version = "development"
 var delay time.Duration = 60 * time.Second
+var conf = loadconfig(*configPath)
 
 func main() {
 	log.Printf("starting version %s...\n", version)
 	// parse all flags
 	flag.Parse()
-	// check if token is set
+
-	if *token == "" {
+	// check if flags are set
-		log.Fatalln("Please specify an API token via the flag '-token'")
+	if *configPath == "" {
+		log.Fatalln("Please specify path to config.json flag '-configPath'")
 	}
 
 	// start API processing (runs in a loop)
-	go processAPIs()
+	go func() {
-
+		if err := processAPIs(); err != nil {
-	// start webserver
+			log.Fatalln("API processing failed, error is: ", err)
+		}
+		tick := time.Tick(delay)
+		for range tick {
+			if err := processAPIs(); err != nil {
+				log.Fatalln("API processing failed, error is: ", err)
+			}
+		}
+	}()
+	// start webserver on Port 3000
 	serveJSON()
 }
 
-func processAPIs() {
+func processAPIs() error {
-	tick := time.Tick(delay)
+	var nodes []node
-	for range tick {
+	var links []link
-		// Variables for runtime
-		var links []link
-		var nodes []node
 
-		d, err := getDevices()
+	if conf.UISP.Enabled {
+		log.Println("Processing UISP")
+		//Process UISP RiFu Nodes
+		uispNodes, uispLinks, err := processUISPRiFu()
 		if err != nil {
-			log.Fatalln(err)
+			return err
 		}
-
+		//Process UISP Routers (like EDGE Router)
-		// API CALL 1
+		uispRouters, err := processUISPRouter()
-		log.Println("calling API 1")
-		var u []unifiAPIResponse
-		err = callUnifiAPI("/devices", &u)
 		if err != nil {
-			log.Fatalln(err)
+			return err
 		}
-
+		nodes = append(nodes, uispNodes...)
-		for i := range d.Devices {
+		nodes = append(nodes, uispRouters...)
-			var dev unifiAPIResponse
+		links = append(links, uispLinks...)
-			var currentDevice device
+	}
-			for j := range u {
+	if len(conf.Unifi) > 0 {
-				if strings.ToUpper(u[j].Identification.MAC) == strings.ToUpper(d.Devices[i].MAC) {
+		log.Println("Anazahl der Unifi Server:", len(conf.Unifi))
-					dev = u[j]
+		for i := range conf.Unifi {
-					currentDevice = d.Devices[i]
+			if conf.Unifi[i].Enabled {
+				log.Println("Processing Unifi-Server: ", conf.Unifi[i].Name)
+				//Process Unifi Nodes
+				unifiNodes, _, err := processUnifiAPI(i)
+				if err != nil {
+					return err
 				}
+				nodes = append(nodes, unifiNodes...)
 			}
-
-			var isOnline bool = false
-			if dev.Overview.Status == "active" {
-				isOnline = true
-			}
-			// END OF API CALL 1
-
-			// API CALL 2
-			log.Println("calling API 2 for device", d.Devices[i].Name)
-			var details unifiAPIDetails
-			callUnifiAPI("/devices/erouters/"+dev.Identification.ID, &details)
-			// END OF API CALL 2
-
-			// API CALL 3
-			log.Println("calling API 3 for device", d.Devices[i].Name)
-			var airmaxes []unifiAPIAirmax
-			callUnifiAPI("/devices/airmaxes/"+dev.Identification.ID+"/stations", &airmaxes)
-			// check if remote mac address is part of our published network
-			for i := range airmaxes {
-				if isRemoteMACpublished(airmaxes[i].DeviceIdentification.MAC, d.Devices) == true {
-					links = addLink(dev, airmaxes[i], links)
-				}
-			}
-			// END OF API CALL 3
-
-			// Get info from json file (static)
-			nodes = append(nodes, node{
-				Firstseen:      dev.Overview.CreatedAt.Format(iso8601),
-				Lastseen:       dev.Overview.LastSeen.Format(iso8601),
-				IsOnline:       isOnline,
-				IsGateway:      false,
-				Clients:        0,
-				ClientsWifi24:  0,
-				ClientsWifi5:   0,
-				ClientsOther:   0,
-				RootFSUsage:    0,
-				LoadAVG:        details.Overview.CPU / 100,
-				MemoryUsage:    details.Overview.RAM / 100,
-				Uptime:         dev.Identification.Started.Format(iso8601),
-				GatewayNexthop: currentDevice.GatewayNexthop,
-				Gateway:        currentDevice.Gateway,
-				Location:       currentDevice.Location,
-				NodeID:         strings.ReplaceAll(dev.Identification.MAC, ":", ""),
-				MAC:            dev.Identification.MAC,
-				Adresses:       getAddresses(details.IPAddress),
-				Domain:         currentDevice.Domain,
-				Hostname:       "[RiFu] " + details.Identification.Name,
-				Owner:          "Freifunk Rhein-Sieg",
-				Firmware: firmware{
-					Base:    "Ubiquiti - Stock",
-					Release: details.Firmware.Current,
-				},
-				Autoupdater: autoupdater{
-					Enabled: false,
-					Branch:  "stable",
-				},
-				NProc: 1,
-				Model: details.Identification.Model,
-			})
 		}
-
-		// assemble final struct
-		o := output{
-			Timestamp: time.Now().Format(iso8601),
-			Nodes:     nodes,
-			Links:     links,
-		}
-
-		// create file output
-		log.Println("writing json file")
-
-		if err := o.writeToFile(); err != nil {
-			log.Fatalln(err)
-		}
-
-		// we're done here
-		log.Println("...done")
 	}
-}
+	if conf.Meshviewer.Enabled {
-
+		log.Println("Processing Meshviewer")
-func getDevices() (devices, error) {
+		mvNodes, mvLinks := getMeshviewer()
-	// get devices from JSON file
+		nodes = append(nodes, mvNodes...)
-	jsonFile, err := os.Open("devices.json")
+		links = append(links, mvLinks...)
-	if err != nil {
-		return devices{}, errors.New("can't open devices.json")
 	}
-	defer jsonFile.Close()
+	if conf.Gateways.Enabled {
-
+		log.Println("Processing Gateways")
-	// read file to bytes
+		//Process Static Gateways from Json
-	byteValue, _ := ioutil.ReadAll(jsonFile)
+		gwNodes := processGateways()
-
+		nodes = append(nodes, gwNodes...)
-	// variable for d
-	var d devices
-	// unmarshal to struct
-	json.Unmarshal(byteValue, &d)
-	return d, nil
-}
-
-func callUnifiAPI(url string, i interface{}) error {
-	request, err := http.NewRequest(http.MethodGet, baseURL+url, nil)
-	if err != nil {
-		return errors.New(fmt.Sprint("can't set request", baseURL+url))
 	}
-	request.Header.Set("x-auth-token", *token)
+	// assemble final struct
-	client := &http.Client{}
+	o := output{
-	response, err := client.Do(request)
+		Timestamp: time.Now().Format(iso8601),
-	if err != nil {
+		Nodes:     nodes,
-		return fmt.Errorf("can't get request %s with x-auth-token %s", baseURL+url, *token)
+		Links:     links,
 	}
 
-	data, err := ioutil.ReadAll(response.Body)
+	// create file output
-	defer response.Body.Close()
+	log.Println("writing json file")
-	if err != nil {
-		return fmt.Errorf("can't read response body: %+v", response.Body)
-	}
-	// try to fetch errors
-	var a apiResponse
-	json.Unmarshal(data, &a)
 
-	if a.StatusCode != 0 {
+	if err := o.writeToFile(); err != nil {
-		return fmt.Errorf("got following errorcode from API: %d %s %s", a.StatusCode, a.Error, a.Message)
+		log.Fatalln(err)
 	}
 
-	// no error occurred, unmarshal to struct
+	// we're done here
-	json.Unmarshal(data, &i)
+	log.Println("...done")
 	return nil
 }
 
+func loadconfig(file string) config {
+	var config config
+	configFile, err := os.Open(file)
+	if err != nil {
+		log.Fatalln("Failed loding Config file: ", err)
+	}
+	jsonParse := json.NewDecoder(configFile)
+	if err := jsonParse.Decode(&config); err != nil {
+		log.Fatalln(err)
+	}
+	return config
+}
+
+// int to bool converter
+func itob(i int) bool {
+	return i == 1
+}
+
+// function to get file from meshviewer
+func getFile(url string) []byte {
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Println("Error getting file from:", url)
+	}
+	data := resp.Body
+	byteValue, _ := io.ReadAll(data)
+	return byteValue
+}
+
+// get devices from devices file on webserver (config)
+func getDevices(url string) devices {
+	// get devices from JSON file
+	jsonFile := getFile(url)
+	// read file to bytes
+	// variable for d
+	var d devices
+	// unmarshal to struct
+	err := json.Unmarshal(jsonFile, &d)
+	if err != nil {
+		fmt.Println("can´t get devices file from " + url)
+		log.Fatal(err)
+	}
+	return d
+}
+
+// check for MAC Adress in current Devices
 func isRemoteMACpublished(mac string, devices []device) bool {
 	for i := range devices {
 		if devices[i].MAC == mac {
@@ -207,31 +178,6 @@ func isRemoteMACpublished(mac string, devices []device) bool {
 	return false
 }
 
-func addLink(dev unifiAPIResponse, airmaxes unifiAPIAirmax, links []link) []link {
-	for i := range links {
-		if links[i].SourceAddr == airmaxes.DeviceIdentification.MAC {
-			// link already exists
-			return links
-		}
-	}
-	links = append(links, link{
-		Type:       "wifi",
-		Source:     strings.ReplaceAll(dev.Identification.MAC, ":", ""),
-		Target:     strings.ReplaceAll(airmaxes.DeviceIdentification.MAC, ":", ""),
-		SourceTQ:   airmaxes.Statistics.LinkScore,
-		TargetTQ:   airmaxes.Statistics.LinkScore,
-		SourceAddr: dev.Identification.MAC,
-		TargetAddr: airmaxes.DeviceIdentification.MAC,
-	})
-	return links
-}
-
-func getAddresses(ip string) []string {
-	var adresses []string
-	adresses = append(adresses, strings.Split(ip, "/")[0])
-	return adresses
-}
-
 func serveJSON() {
 	fs := http.FileServer(http.Dir("./output"))
 	http.Handle("/", fs)
@@ -242,3 +188,12 @@ func serveJSON() {
 		log.Fatalln(err)
 	}
 }
+
+func httpClient() *http.Client {
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	client := &http.Client{Jar: jar}
+	return client
+}

manifest.yml

@@ -1,19 +0,0 @@
-image: fftdf/ffmap-ubnt-api:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
-{{#if build.tags}}
-tags:
-  - "latest"
-{{#each build.tags}}
-  - {{this}}
-{{/each}}
-{{/if}}
-manifests:
-  -
-    image: fftdf/ffmap-ubnt-api:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
-    platform:
-      architecture: amd64
-      os: linux
-  # -
-  #   image: fftdf/ffmap-ubnt-api:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
-  #   platform:
-  #     architecture: arm64
-  #     os: linux

meshviewer.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"time"
+)
+
+func getMeshviewerJSON(url string) (mvDevices, error) {
+	// get devices from JSON file
+	jsonFile := getFile(url)
+
+	// read file to bytes
+	// variable for d
+	var n mvDevices
+	//var l []link
+	// unmarshal to struct
+	err := json.Unmarshal(jsonFile, &n)
+	if err != nil {
+		log.Println("can´t get Meshviewer Json file from " + url)
+		log.Println(err)
+	}
+	return n, nil
+}
+
+func checkMeshviewerLink(s string) bool {
+	mvNodes, _ := getMeshviewer()
+	for i := range mvNodes {
+		if mvNodes[i].MAC == s {
+			return true
+		}
+	}
+	return false
+}
+
+func addmvDevices(d mvDevices) ([]node, []link) {
+	var nodes []node
+	var links []link
+
+	for i := range d.Nodes {
+		mvNode := d.Nodes[i]
+		if findNodeID(mvNode.NodeID) {
+			mvNode.Clients = 0
+			mvNode.ClientsWifi24 = 0
+			mvNode.ClientsWifi5 = 0
+			mvNode.ClientsOther = 0
+		}
+		if mvNode.Location.Latitude == 0 {
+			nodes = append(nodes, node{
+				Firstseen:      mvNode.Firstseen,
+				Lastseen:       mvNode.Lastseen,
+				IsOnline:       mvNode.IsOnline,
+				IsGateway:      mvNode.IsGateway,
+				Clients:        mvNode.Clients,
+				ClientsWifi24:  mvNode.ClientsWifi24,
+				ClientsWifi5:   mvNode.ClientsWifi5,
+				ClientsOther:   mvNode.ClientsOther,
+				RootFSUsage:    int(mvNode.RootfsUsage),
+				LoadAVG:        mvNode.Loadavg,
+				MemoryUsage:    mvNode.MemoryUsage,
+				Uptime:         mvNode.Uptime,
+				GatewayNexthop: mvNode.GatewayNexthop,
+				Gateway:        mvNode.Gateway,
+				NodeID:         mvNode.NodeID,
+				MAC:            mvNode.Mac,
+				Adresses:       mvNode.Addresses,
+				Domain:         mvNode.Domain,
+				Hostname:       mvNode.Hostname,
+				Owner:          mvNode.Owner,
+				Firmware: firmware{
+					Base:    mvNode.Firmware.Base,
+					Release: mvNode.Firmware.Release,
+				},
+				Autoupdater: autoupdater{
+					Enabled: mvNode.Autoupdater.Enabled,
+					Branch:  mvNode.Autoupdater.Branch,
+				},
+				NProc: mvNode.Nproc,
+				Model: mvNode.Model,
+			})
+		} else {
+			nodes = append(nodes, node{
+				Firstseen:      mvNode.Firstseen,
+				Lastseen:       mvNode.Lastseen,
+				IsOnline:       mvNode.IsOnline,
+				IsGateway:      mvNode.IsGateway,
+				Clients:        mvNode.Clients,
+				ClientsWifi24:  mvNode.ClientsWifi24,
+				ClientsWifi5:   mvNode.ClientsWifi5,
+				ClientsOther:   mvNode.ClientsOther,
+				RootFSUsage:    int(mvNode.RootfsUsage),
+				LoadAVG:        mvNode.Loadavg,
+				MemoryUsage:    mvNode.MemoryUsage,
+				Uptime:         mvNode.Uptime,
+				GatewayNexthop: mvNode.GatewayNexthop,
+				Gateway:        mvNode.Gateway,
+				Location:       &mvNode.Location,
+				NodeID:         mvNode.NodeID,
+				MAC:            mvNode.Mac,
+				Adresses:       mvNode.Addresses,
+				Domain:         mvNode.Domain,
+				Hostname:       mvNode.Hostname,
+				Owner:          mvNode.Owner,
+				Firmware: firmware{
+					Base:    mvNode.Firmware.Base,
+					Release: mvNode.Firmware.Release,
+				},
+				Autoupdater: autoupdater{
+					Enabled: mvNode.Autoupdater.Enabled,
+					Branch:  mvNode.Autoupdater.Branch,
+				},
+				NProc: mvNode.Nproc,
+				Model: mvNode.Model,
+			})
+		}
+	}
+	for i := range d.Links {
+		mvNode := d.Links[i]
+		links = append(links, link{
+			Type:       mvNode.Type,
+			Source:     mvNode.Source,
+			Target:     mvNode.Target,
+			SourceTQ:   mvNode.SourceTq,
+			TargetTQ:   mvNode.TargetTq,
+			SourceAddr: mvNode.SourceAddr,
+			TargetAddr: mvNode.TargetAddr,
+		})
+	}
+	return nodes, links
+}
+
+func getMeshviewer() ([]node, []link) {
+	cacheMutex.Lock()
+	defer cacheMutex.Unlock()
+
+	// Überprüfen, ob die Daten kürzlich aktualisiert wurden
+	if time.Since(lastFetchTime) < fetchInterval {
+		return cacheNodes, cacheLinks
+	}
+
+	var nodes []node
+	var links []link
+
+	for i := range conf.Meshviewer.Files {
+		m, err := getMeshviewerJSON(conf.Meshviewer.Files[i].URL)
+		if err != nil {
+			return cacheNodes, cacheLinks
+		}
+		mvNodes, mvLinks := addmvDevices(m)
+		nodes = append(nodes, mvNodes...)
+		links = append(links, mvLinks...)
+	}
+
+	// Cache aktualisieren
+	cacheNodes = nodes
+	cacheLinks = links
+	lastFetchTime = time.Now()
+
+	return cacheNodes, cacheLinks
+}

staticDevices.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"log"
+	"strings"
+	"time"
+
+	client "github.com/influxdata/influxdb1-client/v2"
+)
+
+func processGateways() []node {
+	d := getDevices(conf.Gateways.GatewaysURL)
+	var nodes []node
+
+	for i := range d.Devices {
+		log.Println("Processing Static Device: ", d.Devices[i].Name)
+		currentDevice := d.Devices[i]
+
+		//Collect data
+		//Calulate Memory (%)
+		mem := getInfluxDataPoint("mem", currentDevice.FQDN, conf.General.ProxmoxInfluxPort)
+		maxmem := getInfluxDataPoint("maxmem", currentDevice.FQDN, conf.General.ProxmoxInfluxPort)
+		memoryMap := mem / maxmem
+		memory := memoryMap * 100
+		// Get Network
+		rx := getInfluxDataPoint("netin", currentDevice.FQDN, conf.General.ProxmoxInfluxPort)
+		tx := getInfluxDataPoint("netout", currentDevice.FQDN, conf.General.ProxmoxInfluxPort)
+		// Get CPU (%)
+		cpuMap := getInfluxDataPoint("cpu", currentDevice.FQDN, conf.General.ProxmoxInfluxPort)
+		cpu := cpuMap * 100
+		//Uptime (seconds)
+		uptime := getInfluxDataPoint("uptime", currentDevice.FQDN, conf.General.ProxmoxInfluxPort)
+		t := time.Duration(uptime * float64(time.Second))
+		up := time.Now().Add(-t)
+
+		// fields := map[string]interface{}{}
+		fields := make(map[string]any)
+		tags := map[string]string{
+			"hostname": strings.ReplaceAll(d.Devices[i].Name, " ", "-"),
+			"nodeid":   strings.ReplaceAll(d.Devices[i].MAC, ":", ""),
+		}
+
+		//Build fields for InfluxDB
+		fields["load"] = cpu
+		fields["ram"] = int(memory)
+		fields["time.up"] = int(uptime)
+		//Network
+		fields["traffic.rx.bytes"] = int(rx)
+		fields["traffic.tx.bytes"] = int(tx)
+
+		point, err := client.NewPoint(
+			"node",
+			tags,
+			fields,
+			time.Now(),
+		)
+		if err != nil {
+			log.Fatalln("Error: ", err)
+		}
+		if conf.General.InfluxEnabled {
+			sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		}
+
+		//Build Nodes
+		nodes = append(nodes, node{
+			Firstseen:      up.Format(iso8601),
+			Lastseen:       time.Now().Format(iso8601),
+			IsOnline:       true,
+			IsGateway:      true,
+			Clients:        0,
+			ClientsWifi24:  0,
+			ClientsWifi5:   0,
+			ClientsOther:   0,
+			RootFSUsage:    0,
+			LoadAVG:        cpuMap,
+			MemoryUsage:    memoryMap,
+			Uptime:         up.Format(iso8601),
+			GatewayNexthop: "",
+			Gateway:        "",
+			NodeID:         strings.ReplaceAll(d.Devices[i].MAC, ":", ""),
+			MAC:            d.Devices[i].MAC,
+			Adresses:       d.Devices[i].Adresses,
+			Domain:         d.Devices[i].Domain,
+			Hostname:       "[Gateway] " + d.Devices[i].Name,
+			Owner:          "Freifunk Troisdorf",
+			Firmware: firmware{
+				Base:    "KVM",
+				Release: "Ubuntu 22.04",
+			},
+			Autoupdater: autoupdater{
+				Enabled: false,
+				Branch:  "stable",
+			},
+			NProc: 1,
+			Model: "KVM",
+		})
+	}
+	return nodes
+}

types.go

@@ -4,20 +4,62 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"time"
 )
 
+type config struct {
+	General struct {
+		InfluxEnabled      bool   `json:"influx_enabled"`
+		FreifunkInfluxPort string `json:"freifunk_influx_port"`
+		ProxmoxInfluxPort  string `json:"proxmox_influx_port"`
+		InfluxURL          string `json:"influx_url"`
+	}
+	UISP struct {
+		Enabled    bool   `json:"enabled"`
+		UnmsAPIURL string `json:"unmsAPIUrl"`
+		APItoken   string `json:"APItoken"`
+		DevicesURL string `json:"devicesURL"`
+		RouterURL  string `json:"routerURL"`
+	} `json:"unms"`
+	Unifi      []UnifiServer `json:"unifi"`
+	Meshviewer struct {
+		Enabled bool `json:"enabled"`
+		Files   []struct {
+			Name string `json:"name"`
+			URL  string `json:"URL"`
+		} `json:"files"`
+	} `json:"meshviewer"`
+	Gateways struct {
+		Enabled     bool   `json:"enabled"`
+		GatewaysURL string `json:"gatewaysurl"`
+	} `json:"gateways"`
+}
+
+type UnifiServer struct {
+	Name         string `json:"name"`
+	Enabled      bool   `json:"enabled"`
+	DisplayUsers bool   `json:"displayusers"`
+	APIURL       string `json:"APIUrl"`
+	User         string `json:"user"`
+	Password     string `json:"password"`
+	UCDevicesURL string `json:"ucDevicesURL"`
+}
+
 type device struct {
 	Name           string `json:"name"`
+	FQDN           string `json:"fqdn"`
 	MAC            string `json:"mac"`
 	GatewayNexthop string `json:"gateway_nexthop"`
+	LinkedTo       string `json:"linked_to"`
 	Gateway        string `json:"gateway"`
 	Domain         string `json:"domain"`
 	Location       struct {
 		Longitude float64 `json:"longitude"`
 		Latitude  float64 `json:"latitude"`
 	} `json:"location"`
+	Adresses []string `json:"adresses"`
 }
 type devices struct {
 	Devices []device `json:"devices"`
@@ -49,6 +91,29 @@ type unifiAPIDetails struct {
 		RAM float64 `json:"ram"`
 	} `json:"overview"`
 	IPAddress string `json:"ipAddress"`
+	Location  struct {
+		Latitude  float64 `json:"latitude"`
+		Longitude float64 `json:"longitude"`
+		Altitude  int     `json:"altitude"`
+	} `json:"location"`
+	Interfaces []struct {
+		Identification struct {
+			Type        string `json:"type"`
+			Name        string `json:"name"`
+			Description string `json:"description"`
+			Mac         string `json:"mac"`
+			DisplayName string `json:"displayName"`
+		} `json:"identification"`
+		Statistics struct {
+			Rxrate   int   `json:"rxrate"`
+			Txrate   int   `json:"txrate"`
+			Rxbytes  int64 `json:"rxbytes"`
+			Txbytes  int64 `json:"txbytes"`
+			PoePower int   `json:"poePower"`
+			Dropped  int   `json:"dropped"`
+			Errors   int   `json:"errors"`
+		} `json:"statistics"`
+	} `json:"interfaces"`
 }
 
 type unifiAPIAirmax struct {
@@ -74,21 +139,21 @@ type node struct {
 	Firstseen      string  `json:"firstseen"`
 	Lastseen       string  `json:"lastseen"`
 	IsOnline       bool    `json:"is_online"`
-	IsGateway      bool    `json:"is_gateway"`
+	IsGateway      bool    `json:"is_gateway,omitempty"`
 	Clients        int     `json:"clients"`
 	ClientsWifi24  int     `json:"clients_wifi24"`
 	ClientsWifi5   int     `json:"clients_wifi5"`
 	ClientsOther   int     `json:"clients_other"`
-	RootFSUsage    int     `json:"rootfs_usage"`
+	RootFSUsage    int     `json:"rootfs_usage,omitempty"`
 	LoadAVG        float64 `json:"loadavg"`
 	MemoryUsage    float64 `json:"memory_usage"`
 	Uptime         string  `json:"uptime"`
 	GatewayNexthop string  `json:"gateway_nexthop"`
 	Gateway        string  `json:"gateway"`
-	Location       struct {
+	Location       *struct {
 		Longitude float64 `json:"longitude"`
 		Latitude  float64 `json:"latitude"`
-	} `json:"location"`
+	} `json:"location,omitempty"`
 	NodeID      string      `json:"node_id"`
 	MAC         string      `json:"mac"`
 	Adresses    []string    `json:"addresses"`
@@ -137,8 +202,163 @@ func (o *output) writeToFile() error {
 	return nil
 }
 
-type apiResponse struct {
+type ucSite struct {
-	StatusCode int    `json:"statusCode"`
+	Name string `json:"desc"`
-	Error      string `json:"error"`
+	ID   string `json:"name"`
-	Message    string `json:"message"`
+}
+
+type ucDevice struct {
+	IP       string `json:"ip"`
+	Mac      string `json:"mac"`
+	Model    string `json:"model"`
+	Version  string `json:"version"`
+	Gateway  string `json:"gateway_mac"`
+	Name     string `json:"name"`
+	State    int    `json:"state"`
+	LastSeen int    `json:"last_seen"`
+	Uptime   int    `json:"uptime"`
+	Users    int    `json:"user-wlan-num_sta"`
+	Sysstats struct {
+		CPU    string `json:"cpu"`
+		Memory string `json:"mem"`
+	} `json:"system-stats"`
+}
+
+type UnifiAPIData struct {
+	user    string
+	pass    string
+	baseURL string
+	client  *http.Client
+}
+
+type mvDevices struct {
+	Nodes []struct {
+		Firstseen      string   `json:"firstseen"`
+		Lastseen       string   `json:"lastseen"`
+		IsOnline       bool     `json:"is_online"`
+		IsGateway      bool     `json:"is_gateway"`
+		Clients        int      `json:"clients"`
+		ClientsWifi24  int      `json:"clients_wifi24"`
+		ClientsWifi5   int      `json:"clients_wifi5"`
+		ClientsOther   int      `json:"clients_other"`
+		RootfsUsage    float64  `json:"rootfs_usage"`
+		Loadavg        float64  `json:"loadavg"`
+		MemoryUsage    float64  `json:"memory_usage"`
+		Uptime         string   `json:"uptime"`
+		GatewayNexthop string   `json:"gateway_nexthop,omitempty"`
+		Gateway        string   `json:"gateway,omitempty"`
+		NodeID         string   `json:"node_id"`
+		Mac            string   `json:"mac"`
+		Addresses      []string `json:"addresses"`
+		Domain         string   `json:"domain"`
+		Hostname       string   `json:"hostname"`
+		Owner          string   `json:"owner,omitempty"`
+		Location       struct {
+			Longitude float64 `json:"longitude"`
+			Latitude  float64 `json:"latitude"`
+		} `json:"location,omitempty"`
+		Firmware struct {
+			Base    string `json:"base"`
+			Release string `json:"release"`
+		} `json:"firmware"`
+		Autoupdater struct {
+			Enabled bool   `json:"enabled"`
+			Branch  string `json:"branch"`
+		} `json:"autoupdater,omitempty"`
+		Nproc int    `json:"nproc"`
+		Model string `json:"model,omitempty"`
+	} `json:"nodes"`
+	Links []struct {
+		Type       string  `json:"type"`
+		Source     string  `json:"source"`
+		Target     string  `json:"target"`
+		SourceTq   float64 `json:"source_tq"`
+		TargetTq   float64 `json:"target_tq"`
+		SourceAddr string  `json:"source_addr"`
+		TargetAddr string  `json:"target_addr"`
+	} `json:"links"`
+}
+
+// switch Unifi AP Mod IDs to Names
+func lookupModels(model string) string {
+	switch model {
+	case "BZ2", "U2S48", "U2Sv2":
+		return "Unifi AP"
+	case "BZ2LR", "U2L48", "U2Lv2":
+		return "UniFi AP-LR"
+	case "U7E", "U7Ev2":
+		return "UniFi AP-AC"
+	case "U7HD", "U7SHD":
+		return "UniFi AP-HD"
+	case "UXSDM":
+		return "UniFi AP-BaseStationXG"
+	case "UCMSH":
+		return "AP-MeshXG"
+	case "U7MP":
+		return "AP-AC-Mesh-Pro"
+	case "U7LR":
+		return "UniFi AP-AC-LR"
+	case "U7LT":
+		return "UniFi AP-AC-Lite"
+	case "U7P":
+		return "UniFi AP-Pro"
+	case "U7MSH":
+		return "UniFi AP-AC-Mesh"
+	case "U7PG2":
+		return "UniFi AP-AC-Pro"
+	default:
+		return "Unifi Gerät"
+	}
+}
+
+type UNMSLogResponse struct {
+	Items []struct {
+		ID        string    `json:"id"`
+		StartTime time.Time `json:"startTimestamp"`
+		EndTime   time.Time `json:"endTimestamp"`
+		Type      string    `json:"type"`
+
+		Site struct {
+			Name string `json:"name"`
+		} `json:"site"`
+		Device struct {
+			ModelName   string `json:"modelName"`
+			Name        string `json:"name"`
+			DisplayName string `json:"displayName"`
+		} `json:"device"`
+	} `json:"items"`
+}
+
+type XY struct {
+	X int `json:"x"`
+	Y int `json:"y"`
+}
+
+type AvgMax struct {
+	AVG []XY `json:"avg"`
+	MAX []XY `json:"max"`
+}
+
+type UNMSstatistics struct {
+	Period   int `json:"period"`
+	Interval struct {
+		Start int `json:"start"`
+		End   int `json:"end"`
+	} `json:"interval"`
+	CPU        AvgMax `json:"cpu"`
+	RAM        AvgMax `json:"ram"`
+	Errors     AvgMax `json:"errors"`
+	Interfaces []struct {
+		ID       string `json:"id"`
+		Priority int    `json:"priority"`
+		Name     string `json:"name"`
+		Receive  AvgMax `json:"receive"`
+		Transmit AvgMax `json:"transmit"`
+	} `json:"interfaces"`
+}
+
+type UNMSdhcp []struct {
+	Address  string `json:"address"`
+	Hostname string `json:"hostname"`
+	Type     string `json:"type"`
 }

unifi.go

@@ -0,0 +1,284 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	client "github.com/influxdata/influxdb1-client/v2"
+)
+
+// Unifi Controller API processing
+func processUnifiAPI(s int) ([]node, []link, error) {
+	//get list of Unifi devices to display
+	var nodes []node
+	var links []link
+	d := getDevices(conf.Unifi[s].UCDevicesURL)
+
+	//call Unifi Controller
+	ucAPI := UnifiNewAPI(conf.Unifi[s].User, conf.Unifi[s].Password, conf.Unifi[s].APIURL)
+	//login
+	if err := ucAPI.ucLogin(); err != nil {
+		return nil, nil, err
+	}
+	//get all Sites from Controller
+	sites, err := ucAPI.ucGetSites()
+	if err != nil {
+		return nil, nil, err
+	}
+	//get all devices in all sites
+	devices, err := ucAPI.ucGetDevices(sites)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	//build nodes struct
+	//mvJson, err := getMeshviewerJSON()
+	for _, jsonDevice := range d.Devices {
+		var currentDevice ucDevice
+		var currentJSONDevice device
+		for _, device := range devices {
+			if strings.EqualFold(device.Mac, jsonDevice.MAC) {
+				currentDevice = device
+				currentJSONDevice = jsonDevice
+			}
+		}
+
+		if isRemoteMACpublished(jsonDevice.MAC, d.Devices) {
+			//hier muss gecheckt werden ob der link valide ist
+			if checkMeshviewerLink(jsonDevice.LinkedTo) {
+				links = UnifiAddLink(jsonDevice, links)
+			}
+		}
+
+		isOnline := currentDevice.State == 1
+		var load float64
+		var mem float64
+		var cpu float64
+		if isOnline {
+			load, err = strconv.ParseFloat(currentDevice.Sysstats.CPU, 64)
+			cpu = load * 100
+			if err != nil {
+				log.Println("Error psrsing CPU of device ", currentDevice.Name)
+				log.Println(err)
+				load = 0
+				cpu = 0
+			}
+			mem, err = strconv.ParseFloat(currentDevice.Sysstats.Memory, 64)
+			if err != nil {
+				log.Println("Error parsing Memory of device ", currentDevice.Name)
+				log.Println(err)
+				mem = 0
+			}
+		}
+
+		var model = lookupModels(currentDevice.Model)
+		var clients int
+		if conf.Unifi[s].DisplayUsers {
+			clients = currentDevice.Users
+		}
+
+		//// INFLUX START
+		// fields := map[string]interface{}{}
+		fields := make(map[string]any)
+		tags := map[string]string{
+			"hostname": strings.ReplaceAll(currentDevice.Name, " ", "-"),
+			"nodeid":   strings.ReplaceAll(currentDevice.Mac, ":", ""),
+		}
+		// Generate fields for all network interfaces (not availible for Unifi Nodes)
+		//for eth := range details.Interfaces {
+		//	interface_name_rx := ("rate.rx" + "_" + details.Interfaces[eth].Identification.Name)
+		//	interface_name_tx := ("rate.tx" + "_" + details.Interfaces[eth].Identification.Name)
+		//	fields[interface_name_rx] = details.Interfaces[eth].Statistics.Rxrate
+		//	fields[interface_name_tx] = details.Interfaces[eth].Statistics.Txrate
+		//}
+
+		// set default values if we can't get statistics
+		fields["cpu"] = 0
+		fields["load"] = float64(0)
+		fields["ram"] = 0
+
+		if isOnline {
+			// Generate fields for all Statistics
+			//load := (float64(load) / float64(100))
+			fields["cpu"] = int(cpu)
+			fields["load"] = load
+			fields["ram"] = int(mem)
+		}
+
+		// Generate field for DHCP Leases
+		fields["clients.total"] = clients
+		fields["time.up"] = currentDevice.Uptime
+
+		// Generate Dataponts
+		point, err := client.NewPoint(
+			"node",
+			tags,
+			fields,
+			time.Now(),
+		)
+		if err != nil {
+			log.Fatalln("Error: ", err)
+		}
+
+		if conf.General.InfluxEnabled {
+			sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		}
+
+		// INFLUX STOP
+		//log.Println(currentDevice.Mac)
+		if currentDevice.Mac != "" {
+			nodes = append(nodes, node{
+				Firstseen:      "0",
+				Lastseen:       time.Unix(int64(currentDevice.LastSeen), 0).Format(iso8601),
+				IsOnline:       itob(currentDevice.State),
+				IsGateway:      false,
+				Clients:        clients,
+				ClientsWifi24:  0,
+				ClientsWifi5:   0,
+				ClientsOther:   clients,
+				RootFSUsage:    0,
+				LoadAVG:        load / 100,
+				MemoryUsage:    mem / 100,
+				Uptime:         time.Now().Add(-1 * time.Second * time.Duration(currentDevice.Uptime)).Format(iso8601),
+				GatewayNexthop: currentJSONDevice.GatewayNexthop,
+				Gateway:        currentJSONDevice.Gateway,
+				Location:       &currentJSONDevice.Location,
+				NodeID:         strings.ReplaceAll(currentDevice.Mac, ":", ""),
+				MAC:            currentDevice.Mac,
+				Adresses:       []string{currentDevice.IP},
+				Domain:         currentJSONDevice.Domain,
+				Hostname:       "[Unifi] " + currentDevice.Name,
+				Owner:          "Freifunk Rhein-Sieg",
+				Firmware: firmware{
+					Base:    "Ubiquiti - Stock",
+					Release: currentDevice.Version,
+				},
+				Autoupdater: autoupdater{
+					Enabled: false,
+					Branch:  "stable",
+				},
+				NProc: 1,
+				Model: model,
+			})
+		}
+	}
+	return nodes, links, err
+}
+
+func UnifiNewAPI(user string, pass string, baseURL string) UnifiAPIData {
+	return UnifiAPIData{
+		user:    user,
+		pass:    pass,
+		baseURL: baseURL,
+		client:  httpClient(),
+	}
+}
+
+func (u *UnifiAPIData) ucCallAPI(url string, method string, body *bytes.Buffer, output interface{}) error {
+	req, err := http.NewRequest(method, u.baseURL+url, body)
+	if err != nil {
+		return fmt.Errorf("can't set request %s", u.baseURL+url)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	response, err := u.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("can't login %s", u.baseURL+url)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != 200 {
+		return fmt.Errorf("login failed %s", u.baseURL+url)
+	}
+
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(data, &output)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (u *UnifiAPIData) ucLogin() error {
+	var loginData = []byte(`{"username":"` + u.user + `","password":"` + u.pass + `"}`)
+
+	url := "/api/login"
+	err := u.ucCallAPI(url, http.MethodPost, bytes.NewBuffer(loginData), nil)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (u *UnifiAPIData) ucGetSites() ([]ucSite, error) {
+	var d struct {
+		Data []ucSite `json:"data"`
+	}
+
+	url := "/api/self/sites"
+	err := u.ucCallAPI(url, http.MethodGet, bytes.NewBuffer([]byte{}), &d)
+	if err != nil {
+		return []ucSite{}, err
+	}
+	return d.Data, nil
+}
+
+func (u *UnifiAPIData) ucGetDevices(sites []ucSite) ([]ucDevice, error) {
+	var d struct {
+		Data []ucDevice `json:"data"`
+	}
+	var s []ucDevice
+
+	for _, site := range sites {
+		url := "/api/s/" + site.ID + "/stat/device"
+		u.ucCallAPI(url, http.MethodGet, bytes.NewBuffer([]byte{}), &d)
+		s = append(s, d.Data...)
+	}
+	return s, nil
+}
+
+func UnifiAddLink(dev device, links []link) []link {
+	for i := range links {
+		if links[i].SourceAddr == dev.MAC {
+			// link already exists
+			return links
+		}
+	}
+	if dev.LinkedTo == "" {
+		//no LinkedTo in ucDevices.json
+		return links
+	}
+	links = append(links, link{
+		Type:       "cable",
+		Source:     strings.ReplaceAll(dev.MAC, ":", ""),
+		Target:     strings.ReplaceAll(dev.GatewayNexthop, ":", ""),
+		SourceTQ:   1,
+		TargetTQ:   1,
+		SourceAddr: dev.MAC,
+		TargetAddr: dev.LinkedTo,
+	})
+	return links
+}
+
+func findNodeID(NodeID string) bool {
+	for s := range conf.Unifi {
+		ucDev := getDevices(conf.Unifi[s].UCDevicesURL)
+		for i := range ucDev.Devices {
+			if ucDev.Devices[i].GatewayNexthop == NodeID {
+				return true
+			}
+		}
+	}
+	return false
+}

unms.go

@@ -0,0 +1,287 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	_ "github.com/fatih/structs"
+
+	//_ "github.com/influxdata/influxdb1-client" // this is important because of the bug in go mod
+	client "github.com/influxdata/influxdb1-client/v2"
+)
+
+// UNMS API processing (Richtfunk)
+func processUISPRiFu() ([]node, []link, error) {
+	// Variables for runtime
+	var links []link
+	var nodes []node
+
+	d := getDevices(conf.UISP.DevicesURL)
+
+	// API CALL 1 (get Device overview)
+	log.Println("Starting UISP API Crawler for Rifu devices")
+	log.Println("Getting device overview from UNMS API")
+	var u []unifiAPIResponse
+	if err := UnmsCallAPI("/devices", &u); err != nil {
+		return nil, nil, err
+	}
+
+	for i := range d.Devices {
+		time.Sleep(time.Second)
+		var dev unifiAPIResponse
+		var currentDevice device
+		for j := range u {
+			if strings.ToUpper(u[j].Identification.MAC) == strings.ToUpper(d.Devices[i].MAC) {
+				dev = u[j]
+				currentDevice = d.Devices[i]
+			}
+		}
+		isOnline := dev.Overview.Status == "active"
+		// END OF API CALL 1
+
+		// Getting details from UISP
+		log.Println("Getting device details for: ", d.Devices[i].Name)
+		var details unifiAPIDetails
+		if err := UnmsCallAPI("/devices/erouters/"+dev.Identification.ID, &details); err != nil {
+			return nil, nil, err
+		}
+
+		// Getting details for RiFu
+		log.Println("Getting details for RiFu Link for: ", d.Devices[i].Name)
+		var airmaxes []unifiAPIAirmax
+		if err := UnmsCallAPI("/devices/airmaxes/"+dev.Identification.ID+"/stations", &airmaxes); err != nil {
+			return nil, nil, err
+		}
+		// check if remote mac address is part of our published network
+		for i := range airmaxes {
+			if isRemoteMACpublished(airmaxes[i].DeviceIdentification.MAC, d.Devices) {
+				links = UnmsAddLink(dev, airmaxes[i], links)
+			}
+		}
+		// END OF API CALL 3
+
+		// Get info from json file (static)
+		nodes = append(nodes, node{
+			Firstseen:      dev.Overview.CreatedAt.Format(iso8601),
+			Lastseen:       dev.Overview.LastSeen.Format(iso8601),
+			IsOnline:       isOnline,
+			IsGateway:      false,
+			Clients:        0,
+			ClientsWifi24:  0,
+			ClientsWifi5:   0,
+			ClientsOther:   0,
+			RootFSUsage:    0,
+			LoadAVG:        details.Overview.CPU / 100,
+			MemoryUsage:    details.Overview.RAM / 100,
+			Uptime:         dev.Identification.Started.Format(iso8601),
+			GatewayNexthop: currentDevice.GatewayNexthop,
+			Gateway:        currentDevice.Gateway,
+			Location:       &currentDevice.Location,
+			NodeID:         strings.ReplaceAll(dev.Identification.MAC, ":", ""),
+			MAC:            dev.Identification.MAC,
+			Adresses:       UnmsGetAddresses(details.IPAddress),
+			Domain:         currentDevice.Domain,
+			Hostname:       "[RiFu] " + details.Identification.Name,
+			Owner:          "Freifunk Rhein-Sieg",
+			Firmware: firmware{
+				Base:    "Ubiquiti - Stock",
+				Release: details.Firmware.Current,
+			},
+			Autoupdater: autoupdater{
+				Enabled: false,
+				Branch:  "stable",
+			},
+			NProc: 1,
+			Model: details.Identification.Model,
+		})
+	}
+	return nodes, links, nil
+}
+
+func processUISPRouter() ([]node, error) {
+	time.Sleep(time.Second)
+	// Variables for runtime
+	var nodes []node
+	d := getDevices(conf.UISP.RouterURL)
+
+	// API CALL 1, get all devices list from UNMS
+	log.Println("Get all Routers from UISP")
+	var u []unifiAPIResponse
+	if err := UnmsCallAPI("/devices", &u); err != nil {
+		return nil, err
+	}
+
+	// Get Information for devices device
+	for i := range d.Devices {
+		var dev unifiAPIResponse
+		var currentDevice device
+		for j := range u {
+			if strings.ToUpper(u[j].Identification.MAC) == strings.ToUpper(d.Devices[i].MAC) {
+				dev = u[j]
+				currentDevice = d.Devices[i]
+			}
+		}
+
+		isOnline := dev.Overview.Status == "active"
+
+		// API CALL FOR ROUTER DETAILS (Interface RX/TX)
+		log.Println("Getting details of ", d.Devices[i].Name, "from UISP API")
+		var details unifiAPIDetails
+		if err := UnmsCallAPI("/devices/erouters/"+dev.Identification.ID, &details); err != nil {
+			return nil, err
+		}
+
+		// API CALL FOR DEVICE STATISTICS (CPU, RAM)
+		log.Println("Getting statistics of ", d.Devices[i].Name, "from UISP API")
+		var statistics UNMSstatistics
+		if err := UnmsCallAPI("/devices/"+dev.Identification.ID+"/statistics?interval=hour", &statistics); err != nil {
+			return nil, err
+		}
+		// API CALL FOR DHCP LEASES
+		log.Println("Getting DHCP Leases of ", d.Devices[i].Name, "from UNMS API")
+		var dhcpleases UNMSdhcp
+		if isOnline {
+			if err := UnmsCallAPI("/devices/erouters/"+dev.Identification.ID+"/dhcp/leases", &dhcpleases); err != nil {
+				return nil, err
+			}
+		} else {
+			log.Println("Router ist offline, skipping DHCP Leases")
+		}
+
+		// fields := map[string]interface{}{}
+		fields := make(map[string]any)
+		tags := map[string]string{
+			"hostname": strings.ReplaceAll(d.Devices[i].Name, " ", "-"),
+			"nodeid":   strings.ReplaceAll(dev.Identification.MAC, ":", ""),
+		}
+		// Generate fields for all network interfaces
+		for eth := range details.Interfaces {
+			interface_name_rx := ("rate.rx" + "_" + details.Interfaces[eth].Identification.Name)
+			interface_name_tx := ("rate.tx" + "_" + details.Interfaces[eth].Identification.Name)
+			fields[interface_name_rx] = details.Interfaces[eth].Statistics.Rxrate
+			fields[interface_name_tx] = details.Interfaces[eth].Statistics.Txrate
+		}
+
+		// set default values if we can't get statistics
+		fields["cpu"] = 0
+		fields["load"] = float64(0)
+		fields["ram"] = 0
+
+		if isOnline {
+			// Generate fields for all Statistics
+			load := (float64(statistics.CPU.AVG[0].Y) / float64(100))
+			fields["cpu"] = statistics.CPU.AVG[0].Y
+			fields["load"] = load
+			fields["ram"] = statistics.RAM.AVG[0].Y
+		}
+
+		// Generate field for DHCP Leases
+		fields["clients.total"] = len(dhcpleases)
+
+		// Generate Dataponts
+		point, err := client.NewPoint(
+			"node",
+			tags,
+			fields,
+			time.Now(),
+		)
+		if err != nil {
+			log.Fatalln("Error: ", err)
+		}
+		if conf.General.InfluxEnabled {
+			sendInfluxBatchDataPoint(point, conf.General.FreifunkInfluxPort)
+		}
+		// Get info from json file (static)
+		nodes = append(nodes, node{
+			Firstseen:      dev.Overview.CreatedAt.Format(iso8601),
+			Lastseen:       dev.Overview.LastSeen.Format(iso8601),
+			IsOnline:       isOnline,
+			IsGateway:      false,
+			Clients:        0,
+			ClientsWifi24:  0,
+			ClientsWifi5:   0,
+			ClientsOther:   0,
+			RootFSUsage:    0,
+			LoadAVG:        details.Overview.CPU / 100,
+			MemoryUsage:    details.Overview.RAM / 100,
+			Uptime:         dev.Identification.Started.Format(iso8601),
+			GatewayNexthop: currentDevice.GatewayNexthop,
+			Gateway:        currentDevice.Gateway,
+			Location:       &currentDevice.Location,
+			NodeID:         strings.ReplaceAll(dev.Identification.MAC, ":", ""),
+			MAC:            dev.Identification.MAC,
+			Adresses:       UnmsGetAddresses(details.IPAddress),
+			Domain:         currentDevice.Domain,
+			Hostname:       "[VPN-Router] " + details.Identification.Name,
+			Owner:          "Freifunk Rhein-Sieg",
+			Firmware: firmware{
+				Base:    "Ubiquiti - Stock",
+				Release: details.Firmware.Current,
+			},
+			Autoupdater: autoupdater{
+				Enabled: false,
+				Branch:  "stable",
+			},
+			NProc: 1,
+			Model: details.Identification.Model,
+		})
+	}
+	return nodes, nil
+}
+
+func UnmsCallAPI(url string, i any) error {
+	time.Sleep(time.Second)
+	request, err := http.NewRequest(http.MethodGet, conf.UISP.UnmsAPIURL+url, nil)
+	if err != nil {
+		return errors.New(fmt.Sprint("can't set request", conf.UISP.UnmsAPIURL+url))
+	}
+	//log.Println(conf.UISP.UnmsAPIURL + url)
+	request.Header.Set("x-auth-token", conf.UISP.APItoken)
+	client := &http.Client{}
+	response, err := client.Do(request)
+	if err != nil {
+		return fmt.Errorf("can't get request %s with x-auth-token %s", conf.UISP.UnmsAPIURL+url, conf.UISP.APItoken)
+	}
+	if response.StatusCode != 200 {
+		log.Println("Can't call UNMS API, check token and URL. Skipping device. HTTP Status: ", response.StatusCode)
+		return nil
+	}
+	data, err := ioutil.ReadAll(response.Body)
+	defer response.Body.Close()
+	if err != nil {
+		return fmt.Errorf("can't read response body: %+v", response.Body)
+	}
+	// no error occurred, unmarshal to struct
+	return json.Unmarshal(data, &i)
+}
+
+func UnmsGetAddresses(ip string) []string {
+	var adresses []string
+	adresses = append(adresses, strings.Split(ip, "/")[0])
+	return adresses
+}
+
+func UnmsAddLink(dev unifiAPIResponse, airmaxes unifiAPIAirmax, links []link) []link {
+	for i := range links {
+		if links[i].SourceAddr == airmaxes.DeviceIdentification.MAC {
+			// link already exists
+			return links
+		}
+	}
+	links = append(links, link{
+		Type:       "wifi",
+		Source:     strings.ReplaceAll(dev.Identification.MAC, ":", ""),
+		Target:     strings.ReplaceAll(airmaxes.DeviceIdentification.MAC, ":", ""),
+		SourceTQ:   airmaxes.Statistics.LinkScore,
+		TargetTQ:   airmaxes.Statistics.LinkScore,
+		SourceAddr: dev.Identification.MAC,
+		TargetAddr: airmaxes.DeviceIdentification.MAC,
+	})
+	return links
+}