Add shared state network nodes

packages/lime-system/files/usr/lib/lua/lime/utils.lua

@@ -182,6 +182,10 @@ function utils.slugify(s)
 	return s
 end
 
+function utils.hostname()
+    return io.input("/proc/sys/kernel/hostname"):read("*line")
+end
+
 function utils.sanitize_hostname(hostname)
 	hostname = hostname:gsub(' ', '-')
 	hostname = hostname:gsub('[^-a-zA-Z0-9]', '')
@@ -273,6 +277,10 @@ function utils.getBoardAsTable(board_path)
 	return json.parse(fs.readfile(board_path))
 end
 
+function utils.current_board()
+    return utils.read_file("/tmp/sysinfo/board_name"):gsub("\n","")
+end
+
 function utils.printJson(obj)
     print(json.stringify(obj))
 end

packages/shared-state-network_nodes/Makefile

@@ -0,0 +1,22 @@
+#
+# Copyright (C) 2020 German Ferrero <germanferrero@altermundi.net>
+#
+# This is free software, licensed under the GNU General Public License v3.
+#
+
+include ../../libremesh.mk
+
+define Package/$(PKG_NAME)
+  SECTION:=utils
+  CATEGORY:=Utilities
+  TITLE:=$(PKG_NAME) provides shared state database of all times network nodes
+  MAINTAINER:=Germán Ferrero <germanferrero@altermundi.net>
+  DEPENDS:=+shared-state +shared-state-nodes_and_links +lime-system
+  PKGARCH:=all
+endef
+
+define Package/$(PKG_NAME)/description
+	This $(PKG_NAME) provides shared state database of all times network nodes.
+endef
+
+$(eval $(call BuildPackage,$(PKG_NAME)))

packages/shared-state-network_nodes/files/etc/shared-state/publishers/shared-state-publish_network_nodes

@@ -0,0 +1 @@
+../../../usr/bin/shared-state-publish_network_nodes

packages/shared-state-network_nodes/files/etc/uci-defaults/shared-state-network_nodes-cron

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+unique_append()
+{
+	grep -qF "$1" "$2" || echo "$1" >> "$2"
+}
+
+unique_append \
+	'*/5 * * * * ((sleep $(($RANDOM % 120)); shared-state-multiwriter sync network_nodes &> /dev/null)&)'\
+	/etc/crontabs/root

packages/shared-state-network_nodes/files/usr/bin/network-nodes

@@ -0,0 +1,5 @@
+#!/usr/bin/lua
+
+local network_nodes = require('network-nodes')
+
+io.stdout:write(network_nodes.as_human_readable_table())

packages/shared-state-network_nodes/files/usr/bin/shared-state-publish_network_nodes

@@ -0,0 +1,21 @@
+#!/usr/bin/lua
+
+--! LibreMesh
+--! Copyright (C) 2021  Germán Ferrero <germanferrero@altermundi.net>
+--!
+--! This program is free software: you can redistribute it and/or modify
+--! it under the terms of the GNU Affero General Public License as
+--! published by the Free Software Foundation, either version 3 of the
+--! License, or (at your option) any later version.
+--!
+--! This program is distributed in the hope that it will be useful,
+--! but WITHOUT ANY WARRANTY; without even the implied warranty of
+--! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+--! GNU Affero General Public License for more details.
+--!
+--! You should have received a copy of the GNU Affero General Public License
+--! along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+local network_nodes = require('network-nodes')
+
+network_nodes.publish()

packages/shared-state-network_nodes/files/usr/lib/lua/network-nodes.lua

@@ -0,0 +1,101 @@
+local shared_state = require("shared-state")
+local utils = require("lime.utils")
+local config = require("lime.config")
+
+local network_nodes = {}
+
+function network_nodes._node(hostname, member, fw_version, board, ipv4, ipv6)
+    return {hostname=hostname, member=member, fw_version=fw_version, board=board, ipv4=ipv4, ipv6=ipv6}
+end
+
+function network_nodes._serialize_for_network_nodes(node)
+    return {hostname=node.hostname, member=node.member, fw_version=node.fw_version, board=node.board,
+            ipv4=node.ipv4, ipv6=node.ipv6}
+end
+
+function network_nodes._deserialize_from_network_nodes(data)
+    return network_nodes._node(data.hostname, data.member, data.fw_version, data.board, data.ipv4, data.ipv6)
+end
+
+function network_nodes._nodes_from_db(db)
+    local nodes = {}
+    for hostname, value in pairs(db:get()) do
+        nodes[hostname] = network_nodes._deserialize_from_network_nodes(value.data)
+    end
+    return nodes
+end
+
+function network_nodes._create_node()
+    local uci = config.get_uci_cursor()
+
+    local hostname = utils.hostname()
+    local fw_version = utils.release_info()['DISTRIB_RELEASE']
+    local board = utils.current_board()
+    local member = true
+    local ipv4 = uci:get("network", "lan", "ipaddr")
+    local ipv6 = uci:get("network", "lan", "ip6addr")
+    if ipv6 then ipv6 = ipv6:gsub("/.*$", "") end -- remove the netmask info
+    local node = network_nodes._node(hostname, member, fw_version, board, ipv4, ipv6)
+    node.status = "recently_reachable"
+
+    return node
+end
+
+--! Public API
+
+function network_nodes.get_nodes()
+    local network_nodes_db = shared_state.SharedStateMultiWriter:new("network_nodes")
+    local node_and_links_db = shared_state.SharedState:new("nodes_and_links")
+
+    local nodes = {}
+    -- augment the node information from the network_nodes and the 'nodes_and_links' dbs
+    for hostname, node in pairs(network_nodes._nodes_from_db(network_nodes_db)) do
+        if node_and_links_db:get()[hostname] then
+            node.status = "recently_reachable"
+        elseif node.member then
+            node.status = "unreachable"
+        else
+            node.status = "gone"
+        end
+        nodes[hostname] = node
+    end
+    return nodes
+end
+
+function network_nodes.as_human_readable_table()
+    local nodes = network_nodes.get_nodes()
+    local tmpl = "%-26s %-16s %-30s %-20s %-30s %-40s\n"
+    local out = string.format(tmpl, "hostname", "ipv4", "ipv6", "status", "board", "fw_version")
+    for _, node in pairs(nodes) do
+        if node.member then
+            out = out .. string.format(tmpl, node.hostname, node.ipv4, node.ipv6, node.status,
+                                       node.board, node.fw_version)
+        end
+    end
+    return out
+end
+
+function network_nodes.publish()
+    local node = network_nodes._create_node()
+    local data = {
+        [node.hostname] = network_nodes._serialize_for_network_nodes(node)
+    }
+    network_nodes_db = shared_state.SharedStateMultiWriter:new("network_nodes")
+    network_nodes_db:insert(data)
+end
+
+function network_nodes.mark_nodes_as_gone(hostnames)
+    local network_nodes_db = shared_state.SharedStateMultiWriter:new("network_nodes")
+    local nodes = network_nodes._nodes_from_db(network_nodes_db)
+    local data = {}
+    for _, hostname in pairs(hostnames or {}) do
+        local node = nodes[hostname]
+        if node then
+            node.member = false
+            data[hostname] = network_nodes._serialize_for_network_nodes(node)
+        end
+    end
+    network_nodes_db:insert(data)
+end
+
+return network_nodes

packages/shared-state-network_nodes/files/usr/libexec/rpcd/network-nodes

@@ -0,0 +1,51 @@
+#!/usr/bin/env lua
+--[[
+  Copyright (C) 2013-2020 LibreMesh.org
+  This is free software, licensed under the GNU AFFERO GENERAL PUBLIC LICENSE Version 3
+
+  Copyright 2021 German Ferrero <germanferrero@altermundi.net>
+]]--
+
+local ubus = require "ubus"
+local json = require 'luci.jsonc'
+local utils = require 'lime.utils'
+local network_nodes = require 'network-nodes'
+
+local conn = ubus.connect()
+if not conn then
+    error("Failed to connect to ubus")
+end
+
+local function get_nodes(msg)
+    local nodes = network_nodes.get_nodes()
+    utils.printJson({ status = "ok", nodes = nodes })
+end
+
+local function mark_nodes_as_gone(msg)
+    if not msg.hostnames then
+      utils.printJson({ status = "error", message = "missing attribute hostnames" })
+      return
+    end
+
+    network_nodes.mark_nodes_as_gone(msg.hostnames)
+    utils.printJson({ status = "ok" })
+end
+
+
+local methods = {
+    get_nodes = { no_params = 0},
+    mark_nodes_as_gone = { hostnames = 'value' }
+}
+
+if arg[1] == 'list' then
+    utils.printJson(methods)
+end
+
+if arg[1] == 'call' then
+    local msg = utils.rpcd_readline() or '{}'
+    msg = json.parse(msg)
+    if      arg[2] == 'get_nodes' then get_nodes(msg)
+    elseif  arg[2] == 'mark_nodes_as_gone' then mark_nodes_as_gone(msg)
+    else utils.printJson({ error = "Method not found" })
+    end
+end

packages/shared-state-network_nodes/files/usr/share/rpcd/acl.d/network_nodes.json

@@ -0,0 +1,18 @@
+{
+    "lime-app": {
+        "description": "lime-app public access",
+        "read": {
+            "ubus": {
+                    "network-nodes": [ "get_nodes" ]
+            }
+        }
+    },
+    "root": {
+        "description": "root privileged access",
+        "write": {
+            "ubus": {
+                    "network-nodes": [ "mark_nodes_as_gone" ]
+            }
+        }
+    }
+}

packages/shared-state-network_nodes/tests/test_network_nodes.lua

@@ -0,0 +1,106 @@
+local utils = require('lime.utils')
+local test_utils = require('tests.utils')
+local shared_state = require('shared-state')
+local network_nodes = require('network-nodes')
+
+local uci = nil
+
+describe('Tests network_nodes #network_nodes', function ()
+    before_each('', function()
+        test_dir = test_utils.setup_test_dir()
+        shared_state.PERSISTENT_DATA_DIR = test_dir
+        shared_state.DATA_DIR = test_dir
+        uci = test_utils.setup_test_uci()
+    end)
+
+    after_each('', function()
+        test_utils.teardown_test_dir()
+        test_utils.teardown_test_uci(uci)
+    end)
+
+    it('test node creation and serialization', function()
+        stub(utils, "release_info", function () return {DISTRIB_RELEASE='2021.1'} end)
+        stub(utils, "current_board", function () return 'devboard' end)
+        uci:set('network', 'lan', 'interface')
+        uci:set('network', 'lan', 'ipaddr', '10.5.0.5')
+        uci:set('network', 'lan', 'ip6addr', 'fd0d:fe46:8ce8::ab:cd00/64')
+        uci:commit('network')
+        local node = network_nodes._create_node()
+        assert.are.equal('devboard', node.board)
+        assert.are.equal('2021.1', node.fw_version)
+        assert.are.equal('recently_reachable', node.status)
+        assert.are.equal('10.5.0.5', node.ipv4)
+        assert.are.equal('fd0d:fe46:8ce8::ab:cd00', node.ipv6)
+
+        local node = network_nodes._node("node1", true, "2021.1", "librerouter-v1")
+        assert.are.same(node, network_nodes._deserialize_from_network_nodes(network_nodes._serialize_for_network_nodes(node)))
+
+    end)
+
+    it('test get_nodes return expected format #get_nodes', function()
+        uci:set('network', 'lan', 'interface')
+        uci:set('network', 'lan', 'ipaddr', '10.5.0.5')
+        uci:set('network', 'lan', 'ip6addr', 'fd0d:fe46:8ce8::ab:cd00/64')
+        uci:commit('network')
+        local node1 = network_nodes._node("node1", true, "2021.1", "librerouter-v1", "10.24.3.97",
+                                         "fd0d:fe46:8ce8::ab:cd00")
+        local node2 = network_nodes._node("node2", true, "2020.3", "librerouter-v1", "10.24.3.98",
+                                         "fd0d:fe46:8ce8::ab:cd01")
+        local node3 = network_nodes._node("node3", true, "2020.1", "tplink-wdr3600", "10.24.3.98",
+                                         "fd0d:fe46:8ce8::ab:cd02")
+        local network_nodes_db = shared_state.SharedStateMultiWriter:new('network_nodes')
+        local data = {
+            ["node1"] = network_nodes._serialize_for_network_nodes(node1),
+            ["node2"] = network_nodes._serialize_for_network_nodes(node2),
+            ["node3"] = network_nodes._serialize_for_network_nodes(node3),
+        }
+        network_nodes_db:insert(data)
+
+        local nodes_and_links_db = shared_state.SharedState:new('nodes_and_links')
+        nodes_and_links_db:get()
+        nodes_and_links_db:insert({node1={foo='bar'}})
+
+        assert.are.same({node1=node1, node2=node2, node3=node3}, network_nodes._nodes_from_db(network_nodes_db))
+
+        local nodes = network_nodes.get_nodes()
+        assert.is.equal("recently_reachable", nodes["node1"].status)
+        assert.is.equal("unreachable", nodes["node2"].status)
+        assert.is.equal("librerouter-v1", nodes["node2"].board)
+
+        local csv = network_nodes.as_human_readable_table() -- ok just some excercise...
+    end)
+
+    it('test mark_nodes_as_gone marks nodes as gone', function ()
+        local node1 = network_nodes._node("node1", true, "2021.1", "librerouter-v1")
+        local node2 = network_nodes._node("node2", true, "2020.3", "librerouter-v1")
+        local node3 = network_nodes._node("node3", true, "2020.1", "tplink-wdr3600")
+        local network_nodes_db = shared_state.SharedStateMultiWriter:new('network_nodes')
+        local data = {
+            ["node1"] = network_nodes._serialize_for_network_nodes(node1),
+            ["node2"] = network_nodes._serialize_for_network_nodes(node2),
+            ["node3"] = network_nodes._serialize_for_network_nodes(node3),
+        }
+        network_nodes_db:insert(data)
+
+        network_nodes.mark_nodes_as_gone({
+            'node1', 'node2'
+        })
+
+        local nodes = network_nodes.get_nodes()
+        assert.is.equal("gone", nodes["node1"].status)
+        assert.is.equal("gone", nodes["node2"].status)
+        assert.is.equal("unreachable", nodes["node3"].status)
+        assert.is.equal("2020.1", nodes["node3"].fw_version)
+    end)
+
+    it('test publish', function ()
+        local hostname = 'mydevpc'
+        stub(utils, "hostname", function () return hostname end)
+        stub(utils, "release_info", function () return {RELEASE='2021.1'} end)
+        stub(utils, "current_board", function () return 'devboard' end)
+        network_nodes.publish()
+        local nodes = network_nodes.get_nodes()
+        assert.is.equal("devboard", nodes[hostname].board)
+    end)
+
+end)

packages/shared-state/tests/test_shared_state.lua

@@ -129,7 +129,6 @@ describe('Shared State MultiWriter tests #sharedstatemultiwriter', function()
 
     it('test insert new data', function()
         local ss = shared_state.SharedStateMultiWriter:new('foo')
-        local sharedStateB = shared_state.SharedStateMultiWriter:new('B')
         stub(shared_state, "_getFortune", function () return 100 end)
         ss:insert({ bar = '123'})
         local db = ss:get()

packages/ubus-lime-location/files/usr/libexec/rpcd/lime-location

@@ -49,7 +49,7 @@ local function set_location(msg)
 end
 
 local function nodes_and_links()
-  local hostname = io.input("/proc/sys/kernel/hostname"):read("*line")
+  local hostname = utils.hostname()
   local result = {}
   result[hostname] = location.nodes_and_links()
   utils.printJson(result)

packages/ubus-lime-location/tests/test_lime_location.lua

@@ -91,7 +91,7 @@ describe('ubus-lime-utils tests #ubuslimelocation', function()
 
         stub(network, "get_own_macs", function () return own_macs end)
         local response  = rpcd_call(ubus_lime_loc, {'call', 'nodes_and_links'}, '')
-        local hostname = io.input("/proc/sys/kernel/hostname"):read("*line")
+        local hostname = utils.hostname()
         assert.is.equal(hostname, response[hostname].hostname)
 
         assert.are.same(own_macs, response[hostname].macs)