Network-metrics (#1994)

* Add a couple quick functions for converting a uint64_t network ID/node ID into std::string

* Network metrics

node/Metrics.cpp

@@ -166,6 +166,16 @@ namespace ZeroTier {
         prometheus::simpleapi::counter_metric_t tcp_recv
         { "zt_tcp_data_recv", "number of bytes ZeroTier has received via TCP" };
 
+        // Network Metrics
+        prometheus::simpleapi::gauge_metric_t network_num_joined
+        { "zt_num_networks", "number of networks this instance is joined to" };
+        prometheus::simpleapi::gauge_family_t network_num_multicast_groups
+        { "zt_network_multcast_groups_subscribed", "number of multicast groups networks are subscribed to" };
+        prometheus::simpleapi::counter_family_t network_incoming_packets
+        { "zt_network_incoming_packets", "number of incoming packets per network" };
+        prometheus::simpleapi::counter_family_t network_outgoing_packets
+        { "zt_network_outgoing_packets", "number of outgoing packets per network" };
+
         // General Controller Metrics
         prometheus::simpleapi::gauge_metric_t   network_count
         {"controller_network_count", "number of networks the controller is serving"};

node/Metrics.hpp

@@ -101,6 +101,12 @@ namespace ZeroTier {
         extern prometheus::simpleapi::counter_metric_t tcp_send;
         extern prometheus::simpleapi::counter_metric_t tcp_recv;
 
+        // Network Metrics
+        extern prometheus::simpleapi::gauge_metric_t network_num_joined;
+        extern prometheus::simpleapi::gauge_family_t network_num_multicast_groups;
+        extern prometheus::simpleapi::counter_family_t network_incoming_packets;
+        extern prometheus::simpleapi::counter_family_t network_outgoing_packets;
+
         // General Controller Metrics
         extern prometheus::simpleapi::gauge_metric_t   network_count;
         extern prometheus::simpleapi::gauge_metric_t   member_count;

node/Network.cpp

@@ -32,6 +32,7 @@
 #include "Node.hpp"
 #include "Peer.hpp"
 #include "Trace.hpp"
+#include "Metrics.hpp"
 
 #include <set>
 
@@ -559,13 +560,19 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 	RR(renv),
 	_uPtr(uptr),
 	_id(nwid),
+	_nwidStr(OSUtils::networkIDStr(nwid)),
 	_lastAnnouncedMulticastGroupsUpstream(0),
 	_mac(renv->identity.address(),nwid),
 	_portInitialized(false),
 	_lastConfigUpdate(0),
 	_destroyed(false),
 	_netconfFailure(NETCONF_FAILURE_NONE),
-	_portError(0)
+	_portError(0),
+	_num_multicast_groups{Metrics::network_num_multicast_groups.Add({{"network_id", _nwidStr}})},
+	_incoming_packets_accpeted{Metrics::network_incoming_packets.Add({{"network_id", _nwidStr},{"accepted","yes"}})},
+	_incoming_packets_dropped{Metrics::network_incoming_packets.Add({{"network_id", _nwidStr},{"accepted","no"}})},
+	_outgoing_packets_accepted{Metrics::network_outgoing_packets.Add({{"network_id", _nwidStr},{"accepted","yes"}})},
+	_outgoing_packets_dropped{Metrics::network_outgoing_packets.Add({{"network_id", _nwidStr},{"accepted","no"}})}
 {
 	for(int i=0;i<ZT_NETWORK_MAX_INCOMING_UPDATES;++i) {
 		_incomingConfigChunks[i].ts = 0;
@@ -609,13 +616,15 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 		_portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
 		_portInitialized = true;
 	}
+
+	Metrics::network_num_joined++;
 }
 
 Network::~Network()
 {
 	ZT_VirtualNetworkConfig ctmp;
 	_externalConfig(&ctmp);
-
+	Metrics::network_num_joined--;
 	if (_destroyed) {
 		// This is done in Node::leave() so we can pass tPtr properly
 		//RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
@@ -705,6 +714,7 @@ bool Network::filterOutgoingPacket(
 	}
 
 	if (accept) {
+		_outgoing_packets_accepted++;
 		if ((!noTee)&&(cc)) {
 			Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(_id);
@@ -739,6 +749,7 @@ bool Network::filterOutgoingPacket(
 			return true;
 		}
 	} else {
+		_outgoing_packets_dropped++;
 		if (_config.remoteTraceTarget) {
 			RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0);
 		}
@@ -826,6 +837,7 @@ int Network::filterIncomingPacket(
 	}
 
 	if (accept) {
+		_incoming_packets_accpeted++;
 		if (cc) {
 			Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(_id);
@@ -854,6 +866,8 @@ int Network::filterIncomingPacket(
 			}
 			return 0; // DROP locally, since we redirected
 		}
+	} else {
+		_incoming_packets_dropped++;
 	}
 
 	if (_config.remoteTraceTarget) {
@@ -879,6 +893,7 @@ void Network::multicastSubscribe(void *tPtr,const MulticastGroup &mg)
 	if (!std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) {
 		_myMulticastGroups.insert(std::upper_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg),mg);
 		_sendUpdatesToMembers(tPtr,&mg);
+		_num_multicast_groups++;
 	}
 }
 
@@ -888,6 +903,7 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg)
 	std::vector<MulticastGroup>::iterator i(std::lower_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg));
 	if ( (i != _myMulticastGroups.end()) && (*i == mg) ) {
 		_myMulticastGroups.erase(i);
+		_num_multicast_groups--;
 	}
 }
 

node/Network.hpp

@@ -37,6 +37,7 @@
 #include "Membership.hpp"
 #include "NetworkConfig.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Metrics.hpp"
 
 #define ZT_NETWORK_MAX_INCOMING_UPDATES 3
 #define ZT_NETWORK_MAX_UPDATE_CHUNKS ((ZT_NETWORKCONFIG_DICT_CAPACITY / 1024) + 1)
@@ -439,6 +440,7 @@ private:
 	const RuntimeEnvironment *const RR;
 	void *_uPtr;
 	const uint64_t _id;
+	std::string _nwidStr;
 	uint64_t _lastAnnouncedMulticastGroupsUpstream;
 	MAC _mac; // local MAC address
 	bool _portInitialized;
@@ -479,6 +481,12 @@ private:
 	Mutex _lock;
 
 	AtomicCounter __refCount;
+
+	prometheus::simpleapi::gauge_metric_t _num_multicast_groups;
+	prometheus::simpleapi::counter_metric_t _incoming_packets_accpeted;
+	prometheus::simpleapi::counter_metric_t _incoming_packets_dropped;
+	prometheus::simpleapi::counter_metric_t _outgoing_packets_accepted;
+	prometheus::simpleapi::counter_metric_t _outgoing_packets_dropped;
 };
 
 }	// namespace ZeroTier

osdep/OSUtils.cpp

@@ -17,6 +17,7 @@
 #include <stdarg.h>
 #include <sys/stat.h>
 #include <stdlib.h>
+#include <inttypes.h>
 
 #include "../node/Constants.hpp"
 #include "../node/Utils.hpp"
@@ -66,6 +67,18 @@ unsigned int OSUtils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...)
 	return (unsigned int)n;
 }
 
+std::string OSUtils::networkIDStr(const uint64_t nwid) {
+	char tmp[32] = {};
+	ztsnprintf(tmp, sizeof(tmp), "%.16" PRIx64, nwid);
+	return std::string(tmp);
+}
+
+std::string OSUtils::nodeIDStr(const uint64_t nid) {
+	char tmp[32] = {};
+	ztsnprintf(tmp, sizeof(tmp), "%.10" PRIx64, nid);
+	return std::string(tmp);
+}
+
 #ifdef __UNIX_LIKE__
 bool OSUtils::redirectUnixOutputs(const char *stdoutPath,const char *stderrPath)
 	throw()

osdep/OSUtils.hpp

@@ -68,6 +68,22 @@ public:
 	 */
 	static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...);
 
+	/**
+	 * Converts a uint64_t network ID into a string
+	 * 
+	 * @param nwid network ID
+	 * @throws std::length_error buf[] too short (buf[] will still be left null-terminated)
+	 */
+	static std::string networkIDStr(const uint64_t nwid);
+
+	/**
+	 * Converts a uint64_t node ID into a string
+	 * 
+	 * @param nid node ID
+	 * @throws std::length_error buf[] too short (buf[] will still be left null-terminated)
+	 */
+	static std::string nodeIDStr(const uint64_t nid);
+
 #ifdef __UNIX_LIKE__
 	/**
 	 * Close STDOUT_FILENO and STDERR_FILENO and replace them with output to given path