ubnt-freifunk-map-api/main.go

package main

import (
	"bytes"
	"encoding/csv"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"os"
	"strconv"
	"strings"
	"time"

	_ "git.nils.zone/nils/prettify"
)

const (
	iso8601 = "2006-01-02T15:04:05-0700"
)

// flags
var configPath = flag.String("configPath", "config.json", "Path to config.json")
var version = "development"
var delay time.Duration = 60 * time.Second
var conf = loadconfig(*configPath)
var ucDev = getDevices(conf.Unifi.UCDevicesURL)

func main() {
	log.Printf("starting version %s...\n", version)
	// parse all flags
	flag.Parse()

	// check if flags are set
	//if *configPath == "" {
	//	log.Fatalln("Please specify path to config.json flag '-configPath'")
	//}
	// start API processing (runs in a loop)
	go processAPIs()

	// start webserver on Port 3000
	serveJSON()
}

func loadconfig(file string) config {
	var config config
	configFile, err := os.Open(file)
	if err != nil {
		log.Fatalln(err)
	}
	jsonParse := json.NewDecoder(configFile)
	jsonParse.Decode(&config)
	return config
}

//int to bool converter
func itob(i int) bool {
	if i == 1 {
		return true
	}
	return false
}

//Unifi Controller API processing
func processUcAPIs() ([]node, []link) {
	//get list of Unifi devices to display
	var nodes []node
	var links []link
	d := getDevices(conf.Unifi.UCDevicesURL)

	//call Unifi Controller
	ucAPI := newAPI(conf.Unifi.User, conf.Unifi.Password, conf.Unifi.APIURL)
	//login
	ucAPI.ucLogin()
	//get all Sites from Controller
	sites, err := ucAPI.ucGetSites()
	if err != nil {
		log.Fatalln(err)
	}
	//get all devices in all sites
	devices, err := ucAPI.ucGetDevices(sites)
	if err != nil {
		log.Fatalln(err)
	}

	//build nodes struct
	for _, jsonDevice := range d.Devices {
		var currentDevice ucDevice
		var currentJSONDevice device
		for _, device := range devices {
			if strings.ToUpper(device.Mac) == strings.ToUpper(jsonDevice.MAC) {
				currentDevice = device
				currentJSONDevice = jsonDevice
			}
		}
		if isRemoteMACpublished(jsonDevice.MAC, d.Devices) == true {
			links = ucAddLink(jsonDevice, links)
		}
		load, err := strconv.ParseFloat(currentDevice.Sysstats.CPU, 64)
		if err != nil {
			fmt.Println("Error: ", currentDevice.Name)
			//log.Fatalln(err)
			load = 0
		}
		mem, err := strconv.ParseFloat(currentDevice.Sysstats.Memory, 64)
		if err != nil {
			//log.Fatalln(err)
			load = 0
		}
		var model = lookupModels(currentDevice.Model)
		var clients = currentDevice.Users
		if conf.Unifi.DisplayUsers == false {
			clients = 0
		}
		nodes = append(nodes, node{
			Firstseen:      "0",
			Lastseen:       time.Unix(int64(currentDevice.LastSeen), 0).Format(iso8601),
			IsOnline:       itob(currentDevice.State),
			IsGateway:      false,
			Clients:        clients,
			ClientsWifi24:  0,
			ClientsWifi5:   0,
			ClientsOther:   clients,
			RootFSUsage:    0,
			LoadAVG:        load / 100,
			MemoryUsage:    mem / 100,
			Uptime:         time.Now().Add(-1 * time.Second * time.Duration(currentDevice.Uptime)).Format(iso8601),
			GatewayNexthop: currentJSONDevice.GatewayNexthop,
			Gateway:        currentJSONDevice.Gateway,
			Location:       &currentJSONDevice.Location,
			NodeID:         strings.ReplaceAll(currentDevice.Mac, ":", ""),
			MAC:            currentDevice.Mac,
			Adresses:       []string{currentDevice.IP},
			Domain:         currentJSONDevice.Domain,
			Hostname:       "[Unifi] " + currentDevice.Name,
			Owner:          "Freifunk Rhein-Sieg",
			Firmware: firmware{
				Base:    "Ubiquiti - Stock",
				Release: currentDevice.Version,
			},
			Autoupdater: autoupdater{
				Enabled: false,
				Branch:  "stable",
			},
			NProc: 1,
			Model: model,
		})
	}
	return nodes, links
}

//UNMS API processing (Richtfunk)
func processUNMSAPI() ([]node, []link) {
	// Variables for runtime
	var links []link
	var nodes []node

	d := getDevices(conf.Unms.DevicesURL)

	// API CALL 1
	log.Println("calling API 1")
	var u []unifiAPIResponse
	err := callUnifiAPI("/devices", &u)
	if err != nil {
		log.Fatalln(err)
	}

	for i := range d.Devices {
		var dev unifiAPIResponse
		var currentDevice device
		for j := range u {
			if strings.ToUpper(u[j].Identification.MAC) == strings.ToUpper(d.Devices[i].MAC) {
				dev = u[j]
				currentDevice = d.Devices[i]
			}
		}

		var isOnline bool = false
		if dev.Overview.Status == "active" {
			isOnline = true
		}
		// END OF API CALL 1

		// API CALL 2
		log.Println("calling API 2 for device", d.Devices[i].Name)
		var details unifiAPIDetails
		callUnifiAPI("/devices/erouters/"+dev.Identification.ID, &details)
		// END OF API CALL 2

		// API CALL 3
		log.Println("calling API 3 for device", d.Devices[i].Name)
		var airmaxes []unifiAPIAirmax
		callUnifiAPI("/devices/airmaxes/"+dev.Identification.ID+"/stations", &airmaxes)
		// check if remote mac address is part of our published network
		for i := range airmaxes {
			if isRemoteMACpublished(airmaxes[i].DeviceIdentification.MAC, d.Devices) == true {
				links = addLink(dev, airmaxes[i], links)
			}
		}
		// END OF API CALL 3

		// Get info from json file (static)
		nodes = append(nodes, node{
			Firstseen:      dev.Overview.CreatedAt.Format(iso8601),
			Lastseen:       dev.Overview.LastSeen.Format(iso8601),
			IsOnline:       isOnline,
			IsGateway:      false,
			Clients:        0,
			ClientsWifi24:  0,
			ClientsWifi5:   0,
			ClientsOther:   0,
			RootFSUsage:    0,
			LoadAVG:        details.Overview.CPU / 100,
			MemoryUsage:    details.Overview.RAM / 100,
			Uptime:         dev.Identification.Started.Format(iso8601),
			GatewayNexthop: currentDevice.GatewayNexthop,
			Gateway:        currentDevice.Gateway,
			Location:       &currentDevice.Location,
			NodeID:         strings.ReplaceAll(dev.Identification.MAC, ":", ""),
			MAC:            dev.Identification.MAC,
			Adresses:       getAddresses(details.IPAddress),
			Domain:         currentDevice.Domain,
			Hostname:       "[RiFu] " + details.Identification.Name,
			Owner:          "Freifunk Rhein-Sieg",
			Firmware: firmware{
				Base:    "Ubiquiti - Stock",
				Release: details.Firmware.Current,
			},
			Autoupdater: autoupdater{
				Enabled: false,
				Branch:  "stable",
			},
			NProc: 1,
			Model: details.Identification.Model,
		})
	}
	return nodes, links
}

func processAPIs() {
	tick := time.Tick(delay)
	for range tick {
		var nodes []node
		var links []link

		if conf.Unms.Enabled == true {
			log.Println("Processing UNMS")
			unmsNodes, unmsLinks := processUNMSAPI()
			nodes = append(nodes, unmsNodes...)
			links = append(links, unmsLinks...)
		}
		if conf.Unifi.Enabled == true {
			log.Println("Processing Unifi")
			ucNodes, ucLinks := processUcAPIs()
			nodes = append(nodes, ucNodes...)
			links = append(links, ucLinks...)
		}
		if conf.Meshviewer.Enabled == true {
			log.Println("Processing Meshviewer")
			mvNodes, mvLinks := getMeshviewer()
			nodes = append(nodes, mvNodes...)
			links = append(links, mvLinks...)
		}
		// assemble final struct
		o := output{
			Timestamp: time.Now().Format(iso8601),
			Nodes:     nodes,
			Links:     links,
		}

		// create file output
		log.Println("writing json file")

		if err := o.writeToFile(); err != nil {
			log.Fatalln(err)
		}
		// get outages to serve as .csv
		l := getUNMSLogs()
		err := writeOutagesToCSV(l)
		if err != nil {
			log.Println("Error writing outages.csv")
		}
		// we're done here
		log.Println("...done")
	}
}
func getFile(url string) []byte {
	resp, err := http.Get(url)
	if err != nil {
		log.Println("Error getting file from:", url)
	}
	data := resp.Body
	byteValue, _ := ioutil.ReadAll(data)
	return byteValue
}

func getDevices(url string) devices {
	// get devices from JSON file
	jsonFile := getFile(url)

	// read file to bytes
	// variable for d
	var d devices
	// unmarshal to struct
	err := json.Unmarshal(jsonFile, &d)
	if err != nil {
		fmt.Println("can´t get devices file from " + url)
		log.Fatal(err)
	}
	return d
}

func callUnifiAPI(url string, i interface{}) error {
	request, err := http.NewRequest(http.MethodGet, conf.Unms.UnmsAPIURL+url, nil)
	if err != nil {
		return errors.New(fmt.Sprint("can't set request", conf.Unms.UnmsAPIURL+url))
	}
	request.Header.Set("x-auth-token", conf.Unms.APItoken)
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return fmt.Errorf("can't get request %s with x-auth-token %s", conf.Unms.UnmsAPIURL+url, conf.Unms.APItoken)
	}
	if response.StatusCode != 200 {
		log.Fatalln("Can´t call UNMS API, check token and URL. HTTP Status: ", response.StatusCode)
	}
	data, err := ioutil.ReadAll(response.Body)
	defer response.Body.Close()
	if err != nil {
		return fmt.Errorf("can't read response body: %+v", response.Body)
	}
	// no error occurred, unmarshal to struct
	json.Unmarshal(data, &i)
	return nil
}

func isRemoteMACpublished(mac string, devices []device) bool {
	for i := range devices {
		if devices[i].MAC == mac {
			return true
		}
	}
	return false
}

func addLink(dev unifiAPIResponse, airmaxes unifiAPIAirmax, links []link) []link {
	for i := range links {
		if links[i].SourceAddr == airmaxes.DeviceIdentification.MAC {
			// link already exists
			return links
		}
	}
	links = append(links, link{
		Type:       "wifi",
		Source:     strings.ReplaceAll(dev.Identification.MAC, ":", ""),
		Target:     strings.ReplaceAll(airmaxes.DeviceIdentification.MAC, ":", ""),
		SourceTQ:   airmaxes.Statistics.LinkScore,
		TargetTQ:   airmaxes.Statistics.LinkScore,
		SourceAddr: dev.Identification.MAC,
		TargetAddr: airmaxes.DeviceIdentification.MAC,
	})
	return links
}

func getAddresses(ip string) []string {
	var adresses []string
	adresses = append(adresses, strings.Split(ip, "/")[0])
	return adresses
}

func serveJSON() {
	fs := http.FileServer(http.Dir("./output"))
	http.Handle("/", fs)

	log.Println("Listening on :3000...")
	err := http.ListenAndServe(":3000", nil)
	if err != nil {
		log.Fatalln(err)
	}
}

func httpClient() *http.Client {
	jar, err := cookiejar.New(nil)
	if err != nil {
		log.Fatal(err)
	}
	client := &http.Client{Jar: jar}
	return client
}

func newAPI(user string, pass string, baseURL string) ucAPIData {
	return ucAPIData{
		user:    user,
		pass:    pass,
		baseURL: baseURL,
		client:  httpClient(),
	}
}

func (u *ucAPIData) ucCallAPI(url string, method string, body *bytes.Buffer, output interface{}) error {
	req, err := http.NewRequest(method, u.baseURL+url, body)
	if err != nil {
		return fmt.Errorf("can't set request %s", u.baseURL+url)
	}

	req.Header.Set("Content-Type", "application/json")
	response, err := u.client.Do(req)
	if err != nil {
		return fmt.Errorf("can't login %s", u.baseURL+url)
	}
	defer response.Body.Close()
	if response.StatusCode != 200 {
		return fmt.Errorf("Login failed %s", u.baseURL+url)
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(data, &output)
	if err != nil {
		return err
	}

	return nil
}

func (u *ucAPIData) ucLogin() error {
	var loginData = []byte(`{"username":"` + u.user + `","password":"` + u.pass + `"}`)

	url := "/api/login"
	err := u.ucCallAPI(url, http.MethodPost, bytes.NewBuffer(loginData), nil)
	if err != nil {
		return err
	}
	return nil
}

func (u *ucAPIData) ucGetSites() ([]ucSite, error) {
	var d struct {
		Data []ucSite `json:"data"`
	}

	url := "/api/self/sites"
	err := u.ucCallAPI(url, http.MethodGet, bytes.NewBuffer([]byte{}), &d)
	if err != nil {
		return []ucSite{}, err
	}
	return d.Data, nil
}

func (u *ucAPIData) ucGetDevices(sites []ucSite) ([]ucDevice, error) {
	var d struct {
		Data []ucDevice `json:"data"`
	}
	var s []ucDevice

	for _, site := range sites {
		url := "/api/s/" + site.ID + "/stat/device"
		u.ucCallAPI(url, http.MethodGet, bytes.NewBuffer([]byte{}), &d)
		s = append(s, d.Data...)
	}
	return s, nil
}

func ucAddLink(dev device, links []link) []link {
	for i := range links {
		if links[i].SourceAddr == dev.MAC {
			// link already exists
			return links
		}
	}
	if dev.LinkedTo == "" {
		//no LinkedTo in ucDevices.json
		return links
	}
	links = append(links, link{
		Type:       "cable",
		Source:     strings.ReplaceAll(dev.MAC, ":", ""),
		Target:     strings.ReplaceAll(dev.GatewayNexthop, ":", ""),
		SourceTQ:   1,
		TargetTQ:   1,
		SourceAddr: dev.MAC,
		TargetAddr: dev.LinkedTo,
	})
	return links
}

func getMeshviewerJSON(url string) (mvDevices, error) {
	// get devices from JSON file
	jsonFile := getFile(url)

	// read file to bytes
	// variable for d
	var n mvDevices
	//var l []link
	// unmarshal to struct
	err := json.Unmarshal(jsonFile, &n)
	if err != nil {
		fmt.Println("can´t get Meshviewer Json file from " + url)
		log.Println(err)
	}
	return n, nil
}
func findNodeID(NodeID string) bool {
	for i := range ucDev.Devices {
		if ucDev.Devices[i].GatewayNexthop == NodeID {
			return true
		}
	}
	return false
}
func addmvDevices(d mvDevices) ([]node, []link) {
	var nodes []node
	var links []link

	for i := range d.Nodes {
		mvNode := d.Nodes[i]
		if findNodeID(mvNode.NodeID) == true {
			mvNode.Clients = 0
			mvNode.ClientsWifi24 = 0
			mvNode.ClientsWifi5 = 0
			mvNode.ClientsOther = 0
		}
		if mvNode.Location.Latitude == 0 {
			nodes = append(nodes, node{
				Firstseen:      mvNode.Firstseen,
				Lastseen:       mvNode.Lastseen,
				IsOnline:       mvNode.IsOnline,
				IsGateway:      mvNode.IsGateway,
				Clients:        mvNode.Clients,
				ClientsWifi24:  mvNode.ClientsWifi24,
				ClientsWifi5:   mvNode.ClientsWifi5,
				ClientsOther:   mvNode.ClientsOther,
				RootFSUsage:    int(mvNode.RootfsUsage),
				LoadAVG:        mvNode.Loadavg,
				MemoryUsage:    mvNode.MemoryUsage,
				Uptime:         mvNode.Uptime,
				GatewayNexthop: mvNode.GatewayNexthop,
				Gateway:        mvNode.Gateway,
				NodeID:         mvNode.NodeID,
				MAC:            mvNode.Mac,
				Adresses:       mvNode.Addresses,
				Domain:         mvNode.Domain,
				Hostname:       mvNode.Hostname,
				Owner:          mvNode.Owner,
				Firmware: firmware{
					Base:    mvNode.Firmware.Base,
					Release: mvNode.Firmware.Release,
				},
				Autoupdater: autoupdater{
					Enabled: mvNode.Autoupdater.Enabled,
					Branch:  mvNode.Autoupdater.Branch,
				},
				NProc: mvNode.Nproc,
				Model: mvNode.Model,
			})
		} else {
			nodes = append(nodes, node{
				Firstseen:      mvNode.Firstseen,
				Lastseen:       mvNode.Lastseen,
				IsOnline:       mvNode.IsOnline,
				IsGateway:      mvNode.IsGateway,
				Clients:        mvNode.Clients,
				ClientsWifi24:  mvNode.ClientsWifi24,
				ClientsWifi5:   mvNode.ClientsWifi5,
				ClientsOther:   mvNode.ClientsOther,
				RootFSUsage:    int(mvNode.RootfsUsage),
				LoadAVG:        mvNode.Loadavg,
				MemoryUsage:    mvNode.MemoryUsage,
				Uptime:         mvNode.Uptime,
				GatewayNexthop: mvNode.GatewayNexthop,
				Gateway:        mvNode.Gateway,
				Location:       &mvNode.Location,
				NodeID:         mvNode.NodeID,
				MAC:            mvNode.Mac,
				Adresses:       mvNode.Addresses,
				Domain:         mvNode.Domain,
				Hostname:       mvNode.Hostname,
				Owner:          mvNode.Owner,
				Firmware: firmware{
					Base:    mvNode.Firmware.Base,
					Release: mvNode.Firmware.Release,
				},
				Autoupdater: autoupdater{
					Enabled: mvNode.Autoupdater.Enabled,
					Branch:  mvNode.Autoupdater.Branch,
				},
				NProc: mvNode.Nproc,
				Model: mvNode.Model,
			})
		}
	}
	for i := range d.Links {
		mvNode := d.Links[i]
		links = append(links, link{
			Type:       mvNode.Type,
			Source:     mvNode.Source,
			Target:     mvNode.Target,
			SourceTQ:   mvNode.SourceTq,
			TargetTQ:   mvNode.TargetTq,
			SourceAddr: mvNode.SourceAddr,
			TargetAddr: mvNode.TargetAddr,
		})
	}
	return nodes, links
}

func getMeshviewer() ([]node, []link) {
	var nodes []node
	var links []link

	for i := range conf.Meshviewer.Files {
		log.Println("Hole Meshviewer JSON von: ", conf.Meshviewer.Files[i].URL)
		m, err := getMeshviewerJSON(conf.Meshviewer.Files[i].URL)
		if err != nil {
			return nodes, links
		}
		mvNodes, mvLinks := addmvDevices(m)
		nodes = append(nodes, mvNodes...)
		links = append(links, mvLinks...)
	}
	return nodes, links
}

func getUNMSLogs() UNMSLogResponse {

	var l UNMSLogResponse
	log.Println("Get Outages from UNMS")
	err := callUnifiAPI("/outages?count=100&page=1&type=outage", &l)
	if err != nil {
		log.Fatalln("Error calling Outages API")
	}
	return l
}

func writeOutagesToCSV(l UNMSLogResponse) error {
	csvFile, err := os.Create("output/outages.csv")
	if err != nil {
		return err
	}
	writer := csv.NewWriter(csvFile)

	for _, o := range l.Items {
		var row []string
		row = append(row, o.StartTime.Format("02.01.2006 15:04:05"))
		row = append(row, o.EndTime.Format("02.01.2006 15:04:05"))
		row = append(row, o.Site.Name)
		row = append(row, o.Device.DisplayName)
		writer.Write(row)
	}
	writer.Flush()
	return nil
}