Backport guts of 1.8 to 1.6 tree so we can point release without waiting for UI quirks to be fixed.

controller/ConnectionPool.hpp

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c)2021 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2025-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_CONNECTION_POOL_H_
+#define ZT_CONNECTION_POOL_H_
+
+
+#ifndef _DEBUG
+	#define _DEBUG(x)
+#endif
+
+#include <deque>
+#include <set>
+#include <memory>
+#include <mutex>
+#include <exception>
+#include <string>
+
+namespace ZeroTier {
+
+struct ConnectionUnavailable : std::exception { 
+    char const* what() const throw() {
+        return "Unable to allocate connection";
+    }; 
+};
+
+
+class Connection {
+public:
+    virtual ~Connection() {};
+};
+
+class ConnectionFactory {
+public:
+    virtual ~ConnectionFactory() {};
+    virtual std::shared_ptr<Connection> create()=0;
+};
+
+struct ConnectionPoolStats {
+    size_t pool_size;
+    size_t borrowed_size;
+};
+
+template<class T>
+class ConnectionPool {
+public:
+    ConnectionPool(size_t max_pool_size, size_t min_pool_size, std::shared_ptr<ConnectionFactory> factory)
+        : m_maxPoolSize(max_pool_size)
+        , m_minPoolSize(min_pool_size)
+        , m_factory(factory)
+    {
+        while(m_pool.size() < m_minPoolSize){
+            m_pool.push_back(m_factory->create());
+        }
+    };
+
+    ConnectionPoolStats get_stats() {
+        std::unique_lock<std::mutex> lock(m_poolMutex);
+
+        ConnectionPoolStats stats;
+        stats.pool_size = m_pool.size();
+        stats.borrowed_size = m_borrowed.size();			
+
+        return stats;
+    };
+
+    ~ConnectionPool() {
+    };
+
+    /**
+     * Borrow
+     *
+     * Borrow a connection for temporary use
+     *
+     * When done, either (a) call unborrow() to return it, or (b) (if it's bad) just let it go out of scope.  This will cause it to automatically be replaced.
+     * @retval a shared_ptr to the connection object
+     */
+    std::shared_ptr<T> borrow() {
+        std::unique_lock<std::mutex> l(m_poolMutex);
+        
+        while((m_pool.size() + m_borrowed.size()) < m_minPoolSize) {
+            std::shared_ptr<Connection> conn = m_factory->create();
+            m_pool.push_back(conn);
+        }
+
+        if(m_pool.size()==0){
+            
+            if ((m_pool.size() + m_borrowed.size()) <= m_maxPoolSize) {
+                try {
+                    std::shared_ptr<Connection> conn = m_factory->create();
+                    m_borrowed.insert(conn);
+                    return std::static_pointer_cast<T>(conn);
+                } catch (std::exception &e) {
+                    throw ConnectionUnavailable();
+                }
+            } else {
+                for(auto it = m_borrowed.begin(); it != m_borrowed.end(); ++it){
+                    if((*it).unique()) {
+                        // This connection has been abandoned! Destroy it and create a new connection
+                        try {
+                            // If we are able to create a new connection, return it
+                            _DEBUG("Creating new connection to replace discarded connection");
+                            std::shared_ptr<Connection> conn = m_factory->create();
+                            m_borrowed.erase(it);
+                            m_borrowed.insert(conn);
+                            return std::static_pointer_cast<T>(conn);
+                        } catch(std::exception& e) {
+                            // Error creating a replacement connection
+                            throw ConnectionUnavailable();
+                        }
+                    }
+                }
+                // Nothing available
+                throw ConnectionUnavailable();
+            }
+        }
+
+        // Take one off the front
+        std::shared_ptr<Connection> conn = m_pool.front();
+        m_pool.pop_front();
+        // Add it to the borrowed list
+        m_borrowed.insert(conn);
+        return std::static_pointer_cast<T>(conn);
+    };
+
+    /**
+     * Unborrow a connection
+     *
+     * Only call this if you are returning a working connection.  If the connection was bad, just let it go out of scope (so the connection manager can replace it).
+     * @param the connection
+     */
+    void unborrow(std::shared_ptr<T> conn) {
+        // Lock
+        std::unique_lock<std::mutex> lock(m_poolMutex);
+        m_borrowed.erase(conn);
+        if ((m_pool.size() + m_borrowed.size()) < m_maxPoolSize) {
+            m_pool.push_back(conn);
+        }
+    };
+protected:
+    size_t m_maxPoolSize;
+    size_t m_minPoolSize;
+    std::shared_ptr<ConnectionFactory> m_factory;
+    std::deque<std::shared_ptr<Connection> > m_pool;
+    std::set<std::shared_ptr<Connection> > m_borrowed;
+    std::mutex m_poolMutex;
+};
+
+}
+
+#endif

controller/DB.cpp

@@ -49,6 +49,9 @@ void DB::initNetwork(nlohmann::json &network)
 		}};
 	}
 	if (!network.count("dns")) network["dns"] = nlohmann::json::array();
+	if (!network.count("ssoEnabled")) network["ssoEnabled"] = false;
+	if (!network.count("clientId")) network["clientId"] = "";
+	if (!network.count("authorizationEndpoint")) network["authorizationEndpoint"] = "";
 
 	network["objtype"] = "network";
 }
@@ -56,6 +59,7 @@ void DB::initNetwork(nlohmann::json &network)
 void DB::initMember(nlohmann::json &member)
 {
 	if (!member.count("authorized")) member["authorized"] = false;
+	if (!member.count("ssoExempt")) member["ssoExempt"] = false;
 	if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array();
 	if (!member.count("activeBridge")) member["activeBridge"] = false;
 	if (!member.count("tags")) member["tags"] = nlohmann::json::array();
@@ -67,6 +71,7 @@ void DB::initMember(nlohmann::json &member)
 	if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL;
 	if (!member.count("lastAuthorizedCredentialType")) member["lastAuthorizedCredentialType"] = nlohmann::json();
 	if (!member.count("lastAuthorizedCredential")) member["lastAuthorizedCredential"] = nlohmann::json();
+	if (!member.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = 0LL;
 	if (!member.count("vMajor")) member["vMajor"] = -1;
 	if (!member.count("vMinor")) member["vMinor"] = -1;
 	if (!member.count("vRev")) member["vRev"] = -1;
@@ -92,6 +97,8 @@ void DB::cleanMember(nlohmann::json &member)
 	member.erase("recentLog");
 	member.erase("lastModified");
 	member.erase("lastRequestMetaData");
+	member.erase("authenticationURL"); // computed
+	member.erase("authenticationClientID"); // computed
 }
 
 DB::DB() {}
@@ -174,8 +181,9 @@ bool DB::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohma
 	{
 		std::lock_guard<std::mutex> l2(nw->lock);
 		network = nw->config;
-		for(auto m=nw->members.begin();m!=nw->members.end();++m)
+		for(auto m=nw->members.begin();m!=nw->members.end();++m) {
 			members.push_back(m->second);
+		}
 	}
 	return true;
 }
@@ -188,6 +196,14 @@ void DB::networks(std::set<uint64_t> &networks)
 		networks.insert(n->first);
 }
 
+void DB::networkMemberSSOHasExpired(uint64_t nwid, int64_t now) {
+	std::lock_guard<std::mutex> l(_networks_l);
+	auto nw = _networks.find(nwid);
+	if (nw != _networks.end()) {
+		nw->second->mostRecentDeauthTime = now;
+	}
+}
+
 void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners)
 {
 	uint64_t memberId = 0;

controller/DB.hpp

@@ -31,9 +31,12 @@
 #include <atomic>
 #include <mutex>
 #include <set>
+#include <map>
 
 #include "../ext/json/json.hpp"
 
+#define ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE 25000
+
 namespace ZeroTier
 {
 
@@ -101,12 +104,13 @@ public:
 	}
 
 	virtual bool save(nlohmann::json &record,bool notifyListeners) = 0;
-
 	virtual void eraseNetwork(const uint64_t networkId) = 0;
 	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0;
-
 	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0;
 
+	virtual std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL) { return ""; }
+	virtual void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts);
+
 	inline void addListener(DB::ChangeListener *const listener)
 	{
 		std::lock_guard<std::mutex> l(_changeListeners_l);
@@ -148,8 +152,8 @@ protected:
 		std::mutex lock;
 	};
 
-	void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners);
-	void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners);
+	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners);
+	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners);
 	void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info);
 
 	std::vector<DB::ChangeListener *> _changeListeners;

controller/DBMirrorSet.cpp

@@ -36,7 +36,7 @@ DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener) :
 			}
 
 			for(auto db=dbs.begin();db!=dbs.end();++db) {
-				(*db)->each([this,&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) {
+				(*db)->each([&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) {
 					try {
 						if (network.is_object()) {
 							if (memberId == 0) {
@@ -125,6 +125,26 @@ bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,std::vect
 	return false;
 }
 
+std::string DBMirrorSet::getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL) 
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) { 
+		std::string url = (*d)->getSSOAuthURL(member, redirectURL);
+		if (!url.empty()) {
+			return url;
+		}
+	}
+	return "";
+}
+
+void DBMirrorSet::networkMemberSSOHasExpired(uint64_t nwid, int64_t ts)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) { 
+		(*d)->networkMemberSSOHasExpired(nwid, ts);
+	}
+}
+
 void DBMirrorSet::networks(std::set<uint64_t> &networks)
 {
 	std::lock_guard<std::mutex> l(_dbs_l);
@@ -228,4 +248,47 @@ void DBMirrorSet::onNetworkMemberDeauthorize(const void *db,uint64_t networkId,u
 	_listener->onNetworkMemberDeauthorize(this,networkId,memberId);
 }
 
+void DBMirrorSet::membersExpiring(std::set< std::pair<uint64_t, uint64_t> > &soon, std::set< std::pair<uint64_t, uint64_t> > &expired)
+{
+	std::unique_lock<std::mutex> l(_membersExpiringSoon_l);
+	int64_t now = OSUtils::now();
+	for(auto next=_membersExpiringSoon.begin();next!=_membersExpiringSoon.end();) {
+		if (next->first > now) {
+			const uint64_t nwid = next->second.first;
+			const uint64_t memberId = next->second.second;
+			nlohmann::json network, member;
+			if (this->get(nwid, network, memberId, member)) {
+				try {
+					const bool authorized = member["authorized"];
+					const bool ssoExempt = member["ssoExempt"];
+					const int64_t authenticationExpiryTime = member["authenticationExpiryTime"];
+					if ((authenticationExpiryTime == next->first)&&(authorized)&&(!ssoExempt)) {
+						if ((authenticationExpiryTime - now) > ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE) {
+							// Stop when we get to entries too far in the future.
+							break;
+						} else {
+							const bool ssoEnabled = network["ssoEnabled"];
+							if (ssoEnabled)
+								soon.insert(std::pair<uint64_t, uint64_t>(nwid, memberId));
+						}
+					} else {
+						// Obsolete entry, no longer authorized, or SSO exempt.
+					}
+				} catch ( ... ) {
+					// Invalid member object, erase.
+				}
+			} else {
+				// Not found.
+			}
+		}
+		_membersExpiringSoon.erase(next++);
+	}
+}
+
+void DBMirrorSet::memberWillExpire(int64_t expTime, uint64_t nwid, uint64_t memberId)
+{
+	std::unique_lock<std::mutex> l(_membersExpiringSoon_l);
+	_membersExpiringSoon.insert(std::pair< int64_t, std::pair< uint64_t, uint64_t > >(expTime, std::pair< uint64_t, uint64_t >(nwid, memberId)));
+}
+
 } // namespace ZeroTier

controller/DBMirrorSet.hpp

@@ -51,6 +51,9 @@ public:
 	virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member);
 	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);
 
+	std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL);
+	void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts);
+
 	inline void addDB(const std::shared_ptr<DB> &db)
 	{
 		db->addListener(this);
@@ -58,12 +61,17 @@ public:
 		_dbs.push_back(db);
 	}
 
+	void membersExpiring(std::set< std::pair<uint64_t, uint64_t> > &soon, std::set< std::pair<uint64_t, uint64_t> > &expired);
+	void memberWillExpire(int64_t expTime, uint64_t nwid, uint64_t memberId);
+
 private:
 	DB::ChangeListener *const _listener;
 	std::atomic_bool _running;
 	std::thread _syncCheckerThread;
 	std::vector< std::shared_ptr< DB > > _dbs;
 	mutable std::mutex _dbs_l;
+	std::set< std::pair< int64_t, std::pair<uint64_t, uint64_t> > > _membersExpiringSoon;
+	mutable std::mutex _membersExpiringSoon_l;
 };
 
 } // namespace ZeroTier

controller/EmbeddedNetworkController.cpp

@@ -28,6 +28,9 @@
 #include <map>
 #include <thread>
 #include <memory>
+#include <iomanip>
+#include <sstream>
+#include <cctype>
 
 #include "../include/ZeroTierOne.h"
 #include "../version.h"
@@ -60,6 +63,29 @@ namespace ZeroTier {
 
 namespace {
 
+std::string url_encode(const std::string &value) {
+    std::ostringstream escaped;
+    escaped.fill('0');
+    escaped << std::hex;
+
+    for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
+        std::string::value_type c = (*i);
+
+        // Keep alphanumeric and other accepted characters intact
+        if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
+            escaped << c;
+            continue;
+        }
+
+        // Any other characters are percent-encoded
+        escaped << std::uppercase;
+        escaped << '%' << std::setw(2) << int((unsigned char) c);
+        escaped << std::nouppercase;
+    }
+
+    return escaped.str();
+}
+
 static json _renderRule(ZT_VirtualNetworkRule &rule)
 {
 	char tmp[128];
@@ -476,6 +502,10 @@ EmbeddedNetworkController::~EmbeddedNetworkController()
 		t->join();
 }
 
+void EmbeddedNetworkController::setSSORedirectURL(const std::string &url) {
+	_ssoRedirectURL = url_encode(url);
+}
+
 void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 {
 	char tmp[64];
@@ -688,8 +718,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 					DB::initMember(member);
 
 					try {
-						if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"],false);
-						if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"],false);
+						if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"], false);
+						if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"], false);
+						if (b.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = (uint64_t)OSUtils::jsonInt(b["authenticationExpiryTime"], 0ULL);
+						if (b.count("authenticationURL")) member["authenticationURL"] = OSUtils::jsonString(b["authenticationURL"], "");
 
 						if (b.count("remoteTraceTarget")) {
 							const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],""));
@@ -1248,7 +1280,7 @@ void EmbeddedNetworkController::_request(
 	Utils::hex(nwid,nwids);
 	_db.get(nwid,network,identity.address().toInt(),member,ns);
 	if ((!network.is_object())||(network.empty())) {
-		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND);
+		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND, nullptr, 0);
 		return;
 	}
 	const bool newMember = ((!member.is_object())||(member.empty()));
@@ -1262,11 +1294,11 @@ void EmbeddedNetworkController::_request(
 			// known member.
 			try {
 				if (Identity(haveIdStr.c_str()) != identity) {
-					_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
+					_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
 					return;
 				}
 			} catch ( ... ) {
-				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
+				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
 				return;
 			}
 		} else {
@@ -1323,6 +1355,38 @@ void EmbeddedNetworkController::_request(
 		member["lastAuthorizedCredential"] = autoAuthCredential;
 	}
 
+	// Should we check SSO Stuff?
+	// If network is configured with SSO, and the member is not marked exempt: yes
+	// Otherwise no, we use standard auth logic.
+	bool networkSSOEnabled = OSUtils::jsonBool(network["ssoEnabled"], false);
+	bool memberSSOExempt = OSUtils::jsonBool(member["ssoExempt"], false);
+	std::string authenticationURL;
+	if (networkSSOEnabled && !memberSSOExempt) {
+		authenticationURL = _db.getSSOAuthURL(member, _ssoRedirectURL);
+		std::string memberId = member["id"];
+		//fprintf(stderr, "ssoEnabled && !ssoExempt %s-%s\n", nwids, memberId.c_str());
+		uint64_t authenticationExpiryTime = (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0);
+		//fprintf(stderr, "authExpiryTime: %lld\n", authenticationExpiryTime);
+		if (authenticationExpiryTime < now) {
+			if (!authenticationURL.empty()) {
+				_db.networkMemberSSOHasExpired(nwid, now);
+				onNetworkMemberDeauthorize(&_db, nwid, identity.address().toInt());
+
+				Dictionary<3072> authInfo;
+				authInfo.add("aU", authenticationURL.c_str());
+				//fprintf(stderr, "sending auth URL: %s\n", authenticationURL.c_str());
+
+				DB::cleanMember(member);
+				_db.save(member,true);
+
+				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
+				return;
+			}
+		} else if (authorized) {
+			_db.memberWillExpire(authenticationExpiryTime, nwid, identity.address().toInt());
+		}
+	}
+
 	if (authorized) {
 		// Update version info and meta-data if authorized and if this is a genuine request
 		if (requestPacketId) {
@@ -1347,17 +1411,18 @@ void EmbeddedNetworkController::_request(
 				ms.lastRequestMetaData = metaData;
 				ms.identity = identity;
 			}
-		}
+		}		
 	} else {
+		
 		// If they are not authorized, STOP!
 		DB::cleanMember(member);
 		_db.save(member,true);
-		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
+		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
 		return;
 	}
 
 	// -------------------------------------------------------------------------
-	// If we made it this far, they are authorized.
+	// If we made it this far, they are authorized (and authenticated).
 	// -------------------------------------------------------------------------
 
 	int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA;
@@ -1386,7 +1451,11 @@ void EmbeddedNetworkController::_request(
 	nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU);
 	nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL);
 
-	
+	nc->ssoEnabled = OSUtils::jsonBool(network["ssoEnabled"], false);
+	nc->authenticationExpiryTime = OSUtils::jsonInt(member["authenticationExpiryTime"], 0LL);
+	if (!authenticationURL.empty())
+		Utils::scopy(nc->authenticationURL, sizeof(nc->authenticationURL), authenticationURL.c_str());
+
 	std::string rtt(OSUtils::jsonString(member["remoteTraceTarget"],""));
 	if (rtt.length() == 10) {
 		nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str()));
@@ -1415,6 +1484,8 @@ void EmbeddedNetworkController::_request(
 	json &memberTags = member["tags"];
 	json &dns = network["dns"];
 
+	//fprintf(stderr, "IP Assignment Pools for Network %s: %s\n", nwids, OSUtils::jsonDump(ipAssignmentPools, 2).c_str());
+
 	if (metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,0) <= 0) {
 		// Old versions with no rules engine support get an allow everything rule.
 		// Since rules are enforced bidirectionally, newer versions *will* still
@@ -1730,11 +1801,11 @@ void EmbeddedNetworkController::_request(
 		nc->certificateOfOwnershipCount = 1;
 	}
 
-	CertificateOfMembership com(now,credentialtmd,nwid,identity.address());
+	CertificateOfMembership com(now,credentialtmd,nwid,identity);
 	if (com.sign(_signingId)) {
 		nc->com = com;
 	} else {
-		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR);
+		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0);
 		return;
 	}
 
@@ -1753,17 +1824,44 @@ void EmbeddedNetworkController::_startThreads()
 		_threads.emplace_back([this]() {
 			for(;;) {
 				_RQEntry *qe = (_RQEntry *)0;
-				if (!_queue.get(qe))
+				auto timedWaitResult = _queue.get(qe, 1000);
+				if (timedWaitResult == BlockingQueue<_RQEntry *>::STOP) {
 					break;
-				try {
+				} else if (timedWaitResult == BlockingQueue<_RQEntry *>::OK) {
 					if (qe) {
-						_request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData);
+						try {
+							_request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData);
+						} catch (std::exception &e) {
+							fprintf(stderr,"ERROR: exception in controller request handling thread: %s" ZT_EOL_S,e.what());
+						} catch ( ... ) {
+							fprintf(stderr,"ERROR: exception in controller request handling thread: unknown exception" ZT_EOL_S);
+						}
 						delete qe;
 					}
-				} catch (std::exception &e) {
-					fprintf(stderr,"ERROR: exception in controller request handling thread: %s" ZT_EOL_S,e.what());
-				} catch ( ... ) {
-					fprintf(stderr,"ERROR: exception in controller request handling thread: unknown exception" ZT_EOL_S);
+				}
+
+				std::set< std::pair<uint64_t, uint64_t> > soon;
+				std::set< std::pair<uint64_t, uint64_t> > expired;
+				_db.membersExpiring(soon, expired);
+
+				for(auto s=soon.begin();s!=soon.end();++s) {
+					Identity identity;
+					Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> lastMetaData;
+					{
+						std::unique_lock<std::mutex> ll(_memberStatus_l);
+						auto ms = _memberStatus.find(_MemberStatusKey(s->first, s->second));
+						if (ms != _memberStatus.end()) {
+							lastMetaData = ms->second.lastRequestMetaData;
+							identity = ms->second.identity;
+						}
+					}
+					if (identity) {
+						request(s->first,InetAddress(),0,identity,lastMetaData);
+					}
+				}
+
+				for(auto e=expired.begin();e!=expired.end();++e) {
+					onNetworkMemberDeauthorize(nullptr, e->first, e->second);
 				}
 			}
 		});

controller/EmbeddedNetworkController.hpp

@@ -57,6 +57,8 @@ public:
 
 	virtual void init(const Identity &signingId,Sender *sender);
 
+	void setSSORedirectURL(const std::string &url);
+
 	virtual void request(
 		uint64_t nwid,
 		const InetAddress &fromAddr,
@@ -151,6 +153,7 @@ private:
 	std::mutex _memberStatus_l;
 
 	RedisConfig *_rc;
+	std::string _ssoRedirectURL;
 };
 
 } // namespace ZeroTier

controller/FileDB.cpp

@@ -140,8 +140,7 @@ void FileDB::eraseNetwork(const uint64_t networkId)
 void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
 {
 	nlohmann::json network,member,nullJson;
-	get(networkId,network);
-	get(memberId,member);
+	get(networkId,network,memberId,member);
 	char p[4096];
 	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member" ZT_PATH_SEPARATOR_S "%.10llx.json",_networksPath.c_str(),networkId,memberId);
 	OSUtils::rm(p);

controller/PostgreSQL.hpp

@@ -20,6 +20,9 @@
 
 #define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
 
+#include "ConnectionPool.hpp"
+#include <pqxx/pqxx>
+
 #include <memory>
 #include <redis++/redis++.h>
 
@@ -31,14 +34,69 @@ namespace ZeroTier {
 
 struct RedisConfig;
 
+
+class PostgresConnection : public Connection {
+public:
+	virtual ~PostgresConnection() {
+	}
+
+	std::shared_ptr<pqxx::connection> c;
+	int a;
+};
+
+
+class PostgresConnFactory : public ConnectionFactory {
+public:
+	PostgresConnFactory(std::string &connString) 
+		: m_connString(connString)
+	{
+	}
+
+	virtual std::shared_ptr<Connection> create() {
+		auto c = std::shared_ptr<PostgresConnection>(new PostgresConnection());
+		c->c = std::make_shared<pqxx::connection>(m_connString);
+		return std::static_pointer_cast<Connection>(c);
+	}
+private:
+	std::string m_connString;
+};
+
+class PostgreSQL;
+
+class MemberNotificationReceiver : public pqxx::notification_receiver {
+public: 
+	MemberNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	virtual ~MemberNotificationReceiver() {
+		fprintf(stderr, "MemberNotificationReceiver destroyed\n");
+	}
+
+	virtual void operator() (const std::string &payload, int backendPid);
+private:
+	PostgreSQL *_psql;
+};
+
+class NetworkNotificationReceiver : public pqxx::notification_receiver {
+public:
+	NetworkNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	virtual ~NetworkNotificationReceiver() {
+		fprintf(stderr, "NetworkNotificationReceiver destroyed\n");
+	};
+
+	virtual void operator() (const std::string &payload, int packend_pid);
+private:
+	PostgreSQL *_psql;
+};
+
 /**
  * A controller database driver that talks to PostgreSQL
  *
  * This is for use with ZeroTier Central.  Others are free to build and use it
- * but be aware taht we might change it at any time.
+ * but be aware that we might change it at any time.
  */
 class PostgreSQL : public DB
 {
+	friend class MemberNotificationReceiver;
+	friend class NetworkNotificationReceiver;
 public:
 	PostgreSQL(const Identity &myId, const char *path, int listenPort, RedisConfig *rc);
 	virtual ~PostgreSQL();
@@ -49,21 +107,29 @@ public:
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
+	virtual std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL);
 
 protected:
 	struct _PairHasher
 	{
 		inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); }
 	};
+	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners) {
+		DB::_memberChanged(old, memberConfig, notifyListeners);
+	}
+
+	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners) {
+		DB::_networkChanged(old, networkConfig, notifyListeners);
+	}
 
 private:
-	void initializeNetworks(PGconn *conn);
-	void initializeMembers(PGconn *conn);
+	void initializeNetworks();
+	void initializeMembers();
 	void heartbeat();
 	void membersDbWatcher();
-	void _membersWatcher_Postgres(PGconn *conn);
+	void _membersWatcher_Postgres();
 	void networksDbWatcher();
-	void _networksWatcher_Postgres(PGconn *conn);
+	void _networksWatcher_Postgres();
 
 	void _membersWatcher_Redis();
 	void _networksWatcher_Redis();
@@ -80,7 +146,7 @@ private:
 		NO_OVERRIDE = 1
 	};
 
-	PGconn * getPgConn( OverrideMode m = ALLOW_PGBOUNCER_OVERRIDE );
+	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
 
 	const Identity _myId;
 	const Address _myAddress;
@@ -103,6 +169,7 @@ private:
 	mutable volatile bool _waitNoticePrinted;
 
 	int _listenPort;
+	uint8_t _ssoPsk[48];
 
 	RedisConfig *_rc;
 	std::shared_ptr<sw::redis::Redis> _redis;

include/ZeroTierOne.h

@@ -420,157 +420,6 @@ enum ZT_ResultCode
  */
 #define ZT_ResultCode_isFatal(x) ((((int)(x)) >= 100)&&(((int)(x)) < 1000))
 
-
-/**
- *  Multipath bonding policy
- */
-enum ZT_MultipathBondingPolicy
-{
-	/**
-	 * Normal operation. No fault tolerance, no load balancing
-	 */
-	ZT_BONDING_POLICY_NONE = 0,
-
-	/**
-	 * Sends traffic out on only one path at a time. Configurable immediate
-	 * fail-over.
-	 */
-	ZT_BONDING_POLICY_ACTIVE_BACKUP = 1,
-
-	/**
-	 * Sends traffic out on all paths
-	 */
-	ZT_BONDING_POLICY_BROADCAST = 2,
-
-	/**
-	 * Stripes packets across all paths
-	 */
-	ZT_BONDING_POLICY_BALANCE_RR = 3,
-
-	/**
-	 * Packets destined for specific peers will always be sent over the same
-	 * path.
-	 */
-	ZT_BONDING_POLICY_BALANCE_XOR = 4,
-
-	/**
-	 * Balances flows among all paths according to path performance
-	 */
-	ZT_BONDING_POLICY_BALANCE_AWARE = 5
-};
-
-/**
- * Multipath active re-selection policy (linkSelectMethod)
- */
-enum ZT_MultipathLinkSelectMethod
-{
-	/**
-	 * Primary link regains status as active link whenever it comes back up
-	 * (default when links are explicitly specified)
-	 */
-	ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS = 0,
-
-	/**
-	 * Primary link regains status as active link when it comes back up and
-	 * (if) it is better than the currently-active link.
-	 */
-	ZT_MULTIPATH_RESELECTION_POLICY_BETTER = 1,
-
-	/**
-	 * Primary link regains status as active link only if the currently-active
-	 * link fails.
-	 */
-	ZT_MULTIPATH_RESELECTION_POLICY_FAILURE = 2,
-
-	/**
-	 * The primary link can change if a superior path is detected.
-	 * (default if user provides no fail-over guidance)
-	 */
-	ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE = 3
-};
-
-/**
- * Mode of multipath link interface
- */
-enum ZT_MultipathLinkMode
-{
-	ZT_MULTIPATH_SLAVE_MODE_PRIMARY = 0,
-	ZT_MULTIPATH_SLAVE_MODE_SPARE = 1
-};
-
-/**
- * Strategy for path monitoring
- */
-enum ZT_MultipathMonitorStrategy
-{
-	/**
-	 * Use bonding policy's default strategy
-	 */
-	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DEFAULT = 0,
-
-	/**
-	 * Does not actively send probes to judge aliveness, will rely
-	 * on conventional traffic and summary statistics.
-	 */
-	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE = 1,
-
-	/**
-	 * Sends probes at a constant rate to judge aliveness.
-	 */
-	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_ACTIVE = 2,
-
-	/**
-	 * Sends probes at varying rates which correlate to native
-	 * traffic loads to judge aliveness.
-	 */
-	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC = 3
-};
-
-/**
- * Strategy for re-balancing protocol flows
- */
-enum ZT_MultipathFlowRebalanceStrategy
-{
-	/**
-	 * Flows will only be re-balanced among links during
-	 * assignment or failover. This minimizes the possibility
-	 * of sequence reordering and is thus the default setting.
-	 */
-	ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_PASSIVE = 0,
-
-	/**
-	 * Flows that are active may be re-assigned to a new more
-	 * suitable link if it can be done without disrupting the flow.
-	 * This setting can sometimes cause sequence re-ordering.
-	 */
-	ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_OPPORTUNISTIC = 0,
-
-	/**
-	 * Flows will be continuously re-assigned the most suitable link
-	 * in order to maximize "balance". This can often cause sequence
-	 * reordering and is thus only reccomended for protocols like UDP.
-	 */
-	ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_AGGRESSIVE = 2
-};
-
-/**
- * Indices for the path quality weight vector
- */
-enum ZT_MultipathQualityWeightIndex
-{
-	ZT_QOS_LAT_IDX,
-	ZT_QOS_LTM_IDX,
-	ZT_QOS_PDV_IDX,
-	ZT_QOS_PLR_IDX,
-	ZT_QOS_PER_IDX,
-	ZT_QOS_THR_IDX,
-	ZT_QOS_THM_IDX,
-	ZT_QOS_THV_IDX,
-	ZT_QOS_AGE_IDX,
-	ZT_QOS_SCP_IDX,
-	ZT_QOS_WEIGHT_SIZE
-};
-
 /**
  * Status codes sent to status update callback when things happen
  */
@@ -820,7 +669,12 @@ enum ZT_VirtualNetworkStatus
 	/**
 	 * ZeroTier core version too old
 	 */
-	ZT_NETWORK_STATUS_CLIENT_TOO_OLD = 5
+	ZT_NETWORK_STATUS_CLIENT_TOO_OLD = 5,
+
+	/**
+	 * External authentication is required (e.g. SSO)
+	 */
+	ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED = 6
 };
 
 /**
@@ -1339,6 +1193,21 @@ typedef struct
 	 * Network specific DNS configuration
 	 */
 	ZT_VirtualNetworkDNS dns;
+
+	/**
+	 * sso enabled
+	 */
+	bool ssoEnabled;
+
+	/**
+	 * If the status us AUTHENTICATION_REQUIRED, this may contain a URL for authentication.
+	 */
+	char authenticationURL[2048];
+
+	/**
+	 * Time that current authentication expires. only valid if ssoEnabled is true
+	 */
+	uint64_t authenticationExpiryTime;
 } ZT_VirtualNetworkConfig;
 
 /**

node/BondController.cpp

@@ -1,212 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2025-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#include "../osdep/OSUtils.hpp"
-
-#include "Constants.hpp"
-#include "BondController.hpp"
-#include "Peer.hpp"
-
-namespace ZeroTier {
-
-int BondController::_minReqPathMonitorInterval;
-uint8_t BondController::_defaultBondingPolicy;
-
-BondController::BondController(const RuntimeEnvironment *renv) :
-	RR(renv)
-{
-	bondStartTime = RR->node->now();
-	_defaultBondingPolicy = ZT_BONDING_POLICY_NONE;
-}
-
-bool BondController::linkAllowed(std::string &policyAlias, SharedPtr<Link> link)
-{
-	bool foundInDefinitions = false;
-	if (_linkDefinitions.count(policyAlias)) {
-		auto it = _linkDefinitions[policyAlias].begin();
-		while (it != _linkDefinitions[policyAlias].end()) {
-			if (link->ifname() == (*it)->ifname()) {
-				foundInDefinitions = true;
-				break;
-			}
-			++it;
-		}
-	}
-	return _linkDefinitions[policyAlias].empty() || foundInDefinitions;
-}
-
-void BondController::addCustomLink(std::string& policyAlias, SharedPtr<Link> link)
-{
-	Mutex::Lock _l(_links_m);
-	_linkDefinitions[policyAlias].push_back(link);
-	auto search = _interfaceToLinkMap[policyAlias].find(link->ifname());
-	if (search == _interfaceToLinkMap[policyAlias].end()) {
-		link->setAsUserSpecified(true);
-		_interfaceToLinkMap[policyAlias].insert(std::pair<std::string, SharedPtr<Link>>(link->ifname(), link));
-	}
-}
-
-bool BondController::addCustomPolicy(const SharedPtr<Bond>& newBond)
-{
-	Mutex::Lock _l(_bonds_m);
-	if (!_bondPolicyTemplates.count(newBond->policyAlias())) {
-		_bondPolicyTemplates[newBond->policyAlias()] = newBond;
-		return true;
-	}
-	return false;
-}
-
-bool BondController::assignBondingPolicyToPeer(int64_t identity, const std::string& policyAlias)
-{
-	Mutex::Lock _l(_bonds_m);
-	if (!_policyTemplateAssignments.count(identity)) {
-		_policyTemplateAssignments[identity] = policyAlias;
-		return true;
-	}
-	return false;
-}
-
-SharedPtr<Bond> BondController::getBondByPeerId(int64_t identity)
-{
-	Mutex::Lock _l(_bonds_m);
-	return _bonds.count(identity) ? _bonds[identity] : SharedPtr<Bond>();
-}
-
-SharedPtr<Bond> BondController::createTransportTriggeredBond(const RuntimeEnvironment *renv, const SharedPtr<Peer>& peer)
-{
-	Mutex::Lock _l(_bonds_m);
-	int64_t identity = peer->identity().address().toInt();
-	Bond *bond = nullptr;
-	char traceMsg[128];
-	if (!_bonds.count(identity)) {
-		std::string policyAlias;
-		if (!_policyTemplateAssignments.count(identity)) {
-			if (_defaultBondingPolicy) {
-				sprintf(traceMsg, "%s (bond) Creating new default %s bond to peer %llx",
-					OSUtils::humanReadableTimestamp().c_str(), getPolicyStrByCode(_defaultBondingPolicy).c_str(), identity); RR->t->bondStateMessage(NULL, traceMsg);
-				bond = new Bond(renv, _defaultBondingPolicy, peer);
-			}
-			if (!_defaultBondingPolicy && _defaultBondingPolicyStr.length()) {
-				sprintf(traceMsg, "%s (bond) Creating new default custom %s bond to peer %llx",
-					OSUtils::humanReadableTimestamp().c_str(), _defaultBondingPolicyStr.c_str(), identity);
-				RR->t->bondStateMessage(NULL, traceMsg);
-				bond = new Bond(renv, _bondPolicyTemplates[_defaultBondingPolicyStr].ptr(), peer);
-			}
-		}
-		else {
-			if (!_bondPolicyTemplates[_policyTemplateAssignments[identity]]) {
-				sprintf(traceMsg, "%s (bond) Creating new bond. Assignment for peer %llx was specified as %s but the bond definition was not found. Using default %s",
-					OSUtils::humanReadableTimestamp().c_str(), identity, _policyTemplateAssignments[identity].c_str(), getPolicyStrByCode(_defaultBondingPolicy).c_str());
-				RR->t->bondStateMessage(NULL, traceMsg);
-				bond = new Bond(renv, _defaultBondingPolicy, peer);
-			}
-			else {
-				sprintf(traceMsg, "%s (bond) Creating new default bond %s to peer %llx",
-					OSUtils::humanReadableTimestamp().c_str(), _defaultBondingPolicyStr.c_str(), identity);
-				RR->t->bondStateMessage(NULL, traceMsg);
-				bond = new Bond(renv, _bondPolicyTemplates[_policyTemplateAssignments[identity]].ptr(), peer);
-			}
-		}
-	}
-	if (bond) {
-		_bonds[identity] = bond;
-		/**
-		 * Determine if user has specified anything that could affect the bonding policy's decisions
-		 */
-		if (_interfaceToLinkMap.count(bond->policyAlias())) {
-			std::map<std::string, SharedPtr<Link> >::iterator it = _interfaceToLinkMap[bond->policyAlias()].begin();
-			while (it != _interfaceToLinkMap[bond->policyAlias()].end()) {
-				if (it->second->isUserSpecified()) {
-					bond->_userHasSpecifiedLinks = true;
-				}
-				if (it->second->isUserSpecified() && it->second->primary()) {
-					bond->_userHasSpecifiedPrimaryLink = true;
-				}
-				if (it->second->isUserSpecified() && it->second->userHasSpecifiedFailoverInstructions()) {
-					bond->_userHasSpecifiedFailoverInstructions = true;
-				}
-				if (it->second->isUserSpecified() && (it->second->speed() > 0)) {
-					bond->_userHasSpecifiedLinkSpeeds = true;
-				}
-				++it;
-			}
-		}
-		return bond;
-	}
-	return SharedPtr<Bond>();
-}
-
-SharedPtr<Link> BondController::getLinkBySocket(const std::string& policyAlias, uint64_t localSocket)
-{
-	Mutex::Lock _l(_links_m);
-	char ifname[16];
-	_phy->getIfName((PhySocket *) ((uintptr_t)localSocket), ifname, 16);
-	std::string ifnameStr(ifname);
-	auto search = _interfaceToLinkMap[policyAlias].find(ifnameStr);
-	if (search == _interfaceToLinkMap[policyAlias].end()) {
-		SharedPtr<Link> s = new Link(ifnameStr, 0, 0, 0, 0, 0, true, ZT_MULTIPATH_SLAVE_MODE_SPARE, "", 0.0);
-		_interfaceToLinkMap[policyAlias].insert(std::pair<std::string,SharedPtr<Link> >(ifnameStr, s));
-		return s;
-	}
-	else {
-		return search->second;
-	}
-}
-
-SharedPtr<Link> BondController::getLinkByName(const std::string& policyAlias, const std::string& ifname)
-{
-	Mutex::Lock _l(_links_m);
-	auto search = _interfaceToLinkMap[policyAlias].find(ifname);
-	if (search != _interfaceToLinkMap[policyAlias].end()) {
-		return search->second;
-	}
-	return SharedPtr<Link>();
-}
-
-bool BondController::allowedToBind(const std::string& ifname)
-{
-	return true;
-	/*
-	if (!_defaultBondingPolicy) {
-		return true; // no restrictions
-	}
-	Mutex::Lock _l(_links_m);
-	if (_interfaceToLinkMap.empty()) {
-		return true; // no restrictions
-	}
-	std::map<std::string, std::map<std::string, SharedPtr<Link> > >::iterator policyItr = _interfaceToLinkMap.begin();
-	while (policyItr != _interfaceToLinkMap.end()) {
-		std::map<std::string, SharedPtr<Link> >::iterator linkItr = policyItr->second.begin();
-		while (linkItr != policyItr->second.end()) {
-			if (linkItr->first == ifname) {
-				return true;
-			}
-			++linkItr;
-		}
-		++policyItr;
-	}
-	return false;
-	*/
-}
-
-void BondController::processBackgroundTasks(void *tPtr, const int64_t now)
-{
-	Mutex::Lock _l(_bonds_m);
-	std::map<int64_t,SharedPtr<Bond> >::iterator bondItr = _bonds.begin();
-	while (bondItr != _bonds.end()) {
-		bondItr->second->processBackgroundTasks(tPtr, now);
-		++bondItr;
-	}
-}
-
-} // namespace ZeroTier

node/BondController.hpp

@@ -1,239 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2025-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#ifndef ZT_BONDCONTROLLER_HPP
-#define ZT_BONDCONTROLLER_HPP
-
-#include <map>
-#include <vector>
-
-#include "SharedPtr.hpp"
-#include "../osdep/Phy.hpp"
-#include "../osdep/Link.hpp"
-
-namespace ZeroTier {
-
-class RuntimeEnvironment;
-class Bond;
-class Peer;
-
-class BondController
-{
-	friend class Bond;
-
-public:
-
-	BondController(const RuntimeEnvironment *renv);
-
-	/**
-	 * @return Whether this link is permitted to become a member of a bond.
-	 */
-	bool linkAllowed(std::string &policyAlias, SharedPtr<Link> link);
-
-	/**
-	 * @return The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements.
-	 */
-	int minReqPathMonitorInterval() { return _minReqPathMonitorInterval; }
-
-	/**
-	 * @param minReqPathMonitorInterval The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements.
-	 */
-	static void setMinReqPathMonitorInterval(int minReqPathMonitorInterval) { _minReqPathMonitorInterval = minReqPathMonitorInterval; }
-
-	/**
-	 * @return Whether the bonding layer is currently set up to be used.
-	 */
-	bool inUse() { return !_bondPolicyTemplates.empty() || _defaultBondingPolicy; }
-
-	/**
-	 * @param basePolicyName Bonding policy name (See ZeroTierOne.h)
-	 * @return The bonding policy code for a given human-readable bonding policy name
-	 */
-	static int getPolicyCodeByStr(const std::string& basePolicyName)
-	{
-		if (basePolicyName == "active-backup") { return 1; }
-		if (basePolicyName == "broadcast") { return 2; }
-		if (basePolicyName == "balance-rr") { return 3; }
-		if (basePolicyName == "balance-xor") { return 4; }
-		if (basePolicyName == "balance-aware") { return 5; }
-		return 0; // "none"
-	}
-
-	/**
-	 * @param policy Bonding policy code (See ZeroTierOne.h)
-	 * @return The human-readable name for the given bonding policy code
-	 */
-	static std::string getPolicyStrByCode(int policy)
-	{
-		if (policy == 1) { return "active-backup"; }
-		if (policy == 2) { return "broadcast"; }
-		if (policy == 3) { return "balance-rr"; }
-		if (policy == 4) { return "balance-xor"; }
-		if (policy == 5) { return "balance-aware"; }
-		return "none";
-	}
-
-	/**
-	 * Sets the default bonding policy for new or undefined bonds.
-	 *
-	 * @param bp Bonding policy
-	 */
-	void setBondingLayerDefaultPolicy(uint8_t bp) { _defaultBondingPolicy = bp; }
-
-	/**
-	 * Sets the default (custom) bonding policy for new or undefined bonds.
-	 *
-	 * @param alias Human-readable string alias for bonding policy
-	 */
-	void setBondingLayerDefaultPolicyStr(std::string alias) { _defaultBondingPolicyStr = alias; }
-
-	/**
-	 * @return The default bonding policy
-	 */
-	static int defaultBondingPolicy() { return _defaultBondingPolicy; }
-
-	/**
-	 * Add a user-defined link to a given bonding policy.
-	 *
-	 * @param policyAlias User-defined custom name for variant of bonding policy
-	 * @param link Pointer to new link definition
-	 */
-	void addCustomLink(std::string& policyAlias, SharedPtr<Link> link);
-
-	/**
-	 * Add a user-defined bonding policy that is based on one of the standard types.
-	 *
-	 * @param newBond Pointer to custom Bond object
-	 * @return Whether a uniquely-named custom policy was successfully added
-	 */
-	bool addCustomPolicy(const SharedPtr<Bond>& newBond);
-
-	/**
-	 * Assigns a specific bonding policy
-	 *
-	 * @param identity
-	 * @param policyAlias
-	 * @return
-	 */
-	bool assignBondingPolicyToPeer(int64_t identity, const std::string& policyAlias);
-
-	/**
-	 * Get pointer to bond by a given peer ID
-	 *
-	 * @param peer Remote peer ID
-	 * @return A pointer to the Bond
-	 */
-	SharedPtr<Bond> getBondByPeerId(int64_t identity);
-
-	/**
-	 * Add a new bond to the bond controller.
-	 *
-	 * @param renv Runtime environment
-	 * @param peer Remote peer that this bond services
-	 * @return A pointer to the newly created Bond
-	 */
-	SharedPtr<Bond> createTransportTriggeredBond(const RuntimeEnvironment *renv, const SharedPtr<Peer>& peer);
-
-	/**
-	 * Periodically perform maintenance tasks for the bonding layer.
-	 *
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param now Current time
-	 */
-	void processBackgroundTasks(void *tPtr, int64_t now);
-
-	/**
-	 * Gets a reference to a physical link definition given a policy alias and a local socket.
-	 *
-	 * @param policyAlias Policy in use
-	 * @param localSocket Local source socket
-	 * @return Physical link definition
-	 */
-	SharedPtr<Link> getLinkBySocket(const std::string& policyAlias, uint64_t localSocket);
-
-	/**
-	 * Gets a reference to a physical link definition given its human-readable system name.
-	 *
-	 * @param policyAlias Policy in use
-	 * @param ifname Alphanumeric human-readable name
-	 * @return Physical link definition
-	 */
-	SharedPtr<Link> getLinkByName(const std::string& policyAlias, const std::string& ifname);
-
-	/**
-	 * @param ifname Name of interface that we want to know if we can bind to
-	 */
-	bool allowedToBind(const std::string& ifname);
-
-	uint64_t getBondStartTime() { return bondStartTime; }
-
-private:
-
-	Phy<BondController *> *_phy;
-	const RuntimeEnvironment *RR;
-
-	Mutex _bonds_m;
-	Mutex _links_m;
-
-	/**
-	 * The last time that the bond controller updated the set of bonds.
-	 */
-	uint64_t _lastBackgroundBondControlTaskCheck;
-
-	/**
-	 * The minimum monitoring interval among all paths in this bond.
-	 */
-	static int _minReqPathMonitorInterval;
-
-	/**
-	 * The default bonding policy used for new bonds unless otherwise specified.
-	 */
-	static uint8_t _defaultBondingPolicy;
-
-	/**
-	 * The default bonding policy used for new bonds unless otherwise specified.
-	 */
-	std::string _defaultBondingPolicyStr;
-
-	/**
-	 * All currently active bonds.
-	 */
-	std::map<int64_t,SharedPtr<Bond> > _bonds;
-
-	/**
-	 * Map of peers to custom bonding policies
-	 */
-	std::map<int64_t,std::string> _policyTemplateAssignments;
-
-	/**
-	 * User-defined bonding policies (can be assigned to a peer)
-	 */
-	std::map<std::string,SharedPtr<Bond> > _bondPolicyTemplates;
-
-	/**
-	 * Set of links defined for a given bonding policy
-	 */
-	std::map<std::string,std::vector<SharedPtr<Link> > > _linkDefinitions;
-
-	/**
-	 * Set of link objects mapped to their physical interfaces
-	 */
-	std::map<std::string, std::map<std::string, SharedPtr<Link> > > _interfaceToLinkMap;
-
-	// TODO: Remove
-	uint64_t bondStartTime;
-};
-
-} // namespace ZeroTier
-
-#endif

node/CertificateOfMembership.cpp

@@ -20,165 +20,67 @@
 
 namespace ZeroTier {
 
-void CertificateOfMembership::setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta)
+CertificateOfMembership::CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Identity &issuedTo)
 {
-	_signedBy.zero();
-
-	for(unsigned int i=0;i<_qualifierCount;++i) {
-		if (_qualifiers[i].id == id) {
-			_qualifiers[i].value = value;
-			_qualifiers[i].maxDelta = maxDelta;
-			return;
-		}
-	}
-
-	if (_qualifierCount < ZT_NETWORK_COM_MAX_QUALIFIERS) {
-		_qualifiers[_qualifierCount].id = id;
-		_qualifiers[_qualifierCount].value = value;
-		_qualifiers[_qualifierCount].maxDelta = maxDelta;
-		++_qualifierCount;
-		std::sort(&(_qualifiers[0]),&(_qualifiers[_qualifierCount]));
-	}
-}
-
-#ifdef ZT_SUPPORT_OLD_STYLE_NETCONF
-
-std::string CertificateOfMembership::toString() const
-{
-	char tmp[ZT_NETWORK_COM_MAX_QUALIFIERS * 32];
-	std::string s;
-
-	s.append("1:"); // COM_UINT64_ED25519
-
-	uint64_t *const buf = new uint64_t[_qualifierCount * 3];
-	try {
-		unsigned int ptr = 0;
-		for(unsigned int i=0;i<_qualifierCount;++i) {
-			buf[ptr++] = Utils::hton(_qualifiers[i].id);
-			buf[ptr++] = Utils::hton(_qualifiers[i].value);
-			buf[ptr++] = Utils::hton(_qualifiers[i].maxDelta);
-		}
-		s.append(Utils::hex(buf,ptr * sizeof(uint64_t),tmp));
-		delete [] buf;
-	} catch ( ... ) {
-		delete [] buf;
-		throw;
-	}
-
-	s.push_back(':');
-
-	s.append(_signedBy.toString(tmp));
-
-	if (_signedBy) {
-		s.push_back(':');
-		s.append(Utils::hex(_signature.data,ZT_C25519_SIGNATURE_LEN,tmp));
+	_qualifiers[0].id = COM_RESERVED_ID_TIMESTAMP;
+	_qualifiers[0].value = timestamp;
+	_qualifiers[0].maxDelta = timestampMaxDelta;
+	_qualifiers[1].id = COM_RESERVED_ID_NETWORK_ID;
+	_qualifiers[1].value = nwid;
+	_qualifiers[1].maxDelta = 0;
+	_qualifiers[2].id = COM_RESERVED_ID_ISSUED_TO;
+	_qualifiers[2].value = issuedTo.address().toInt();
+	_qualifiers[2].maxDelta = 0xffffffffffffffffULL;
+
+	// Include hash of full identity public key in COM for hardening purposes. Pack it in
+	// using the original COM format. Format may be revised in the future to make this cleaner.
+	uint64_t idHash[6];
+	issuedTo.publicKeyHash(idHash);
+	for(unsigned long i=0;i<4;++i) {
+		_qualifiers[i + 3].id = (uint64_t)(i + 3);
+		_qualifiers[i + 3].value = Utils::ntoh(idHash[i]);
+		_qualifiers[i + 3].maxDelta = 0xffffffffffffffffULL;
 	}
 
-	return s;
-}
-
-void CertificateOfMembership::fromString(const char *s)
-{
-	_qualifierCount = 0;
-	_signedBy.zero();
+	_qualifierCount = 7;
 	memset(_signature.data,0,ZT_C25519_SIGNATURE_LEN);
-
-	if (!*s)
-		return;
-
-	unsigned int colonAt = 0;
-	while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
-
-	if (!((colonAt == 1)&&(s[0] == '1'))) // COM_UINT64_ED25519?
-		return;
-
-	s += colonAt + 1;
-	colonAt = 0;
-	while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
-
-	if (colonAt) {
-		const unsigned int buflen = colonAt / 2;
-		char *const buf = new char[buflen];
-		unsigned int bufactual = Utils::unhex(s,colonAt,buf,buflen);
-		char *bufptr = buf;
-		try {
-			while (bufactual >= 24) {
-				if (_qualifierCount < ZT_NETWORK_COM_MAX_QUALIFIERS) {
-					_qualifiers[_qualifierCount].id = Utils::ntoh(*((uint64_t *)bufptr)); bufptr += 8;
-					_qualifiers[_qualifierCount].value = Utils::ntoh(*((uint64_t *)bufptr)); bufptr += 8;
-					_qualifiers[_qualifierCount].maxDelta = Utils::ntoh(*((uint64_t *)bufptr)); bufptr += 8;
-					++_qualifierCount;
-				} else {
-					bufptr += 24;
-				}
-				bufactual -= 24;
-			}
-		} catch ( ... ) {}
-		delete [] buf;
-	}
-
-	if (s[colonAt]) {
-		s += colonAt + 1;
-		colonAt = 0;
-		while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
-
-		if (colonAt) {
-			char addrbuf[ZT_ADDRESS_LENGTH];
-			if (Utils::unhex(s,colonAt,addrbuf,sizeof(addrbuf)) == ZT_ADDRESS_LENGTH)
-				_signedBy.setTo(addrbuf,ZT_ADDRESS_LENGTH);
-
-			if ((_signedBy)&&(s[colonAt])) {
-				s += colonAt + 1;
-				colonAt = 0;
-				while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
-				if (colonAt) {
-					if (Utils::unhex(s,colonAt,_signature.data,ZT_C25519_SIGNATURE_LEN) != ZT_C25519_SIGNATURE_LEN)
-						_signedBy.zero();
-				} else {
-					_signedBy.zero();
-				}
-			} else {
-				_signedBy.zero();
-			}
-		}
-	}
-
-	std::sort(&(_qualifiers[0]),&(_qualifiers[_qualifierCount]));
 }
 
-#endif // ZT_SUPPORT_OLD_STYLE_NETCONF
-
-bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other) const
+bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other, const Identity &otherIdentity) const
 {
-	unsigned int myidx = 0;
-	unsigned int otheridx = 0;
-
 	if ((_qualifierCount == 0)||(other._qualifierCount == 0))
 		return false;
 
-	while (myidx < _qualifierCount) {
-		// Fail if we're at the end of other, since this means the field is
-		// missing.
-		if (otheridx >= other._qualifierCount)
+	std::map< uint64_t, uint64_t > otherFields;
+	for(unsigned int i=0;i<other._qualifierCount;++i)
+		otherFields[other._qualifiers[i].id] = other._qualifiers[i].value;
+
+	bool fullIdentityVerification = false;
+	for(unsigned int i=0;i<_qualifierCount;++i) {
+		const uint64_t qid = _qualifiers[i].id;
+		if ((qid >= 3)&&(qid <= 6))
+			fullIdentityVerification = true;
+		std::map< uint64_t, uint64_t >::iterator otherQ(otherFields.find(qid));
+		if (otherQ == otherFields.end())
 			return false;
+		const uint64_t a = _qualifiers[i].value;
+		const uint64_t b = otherQ->second;
+		if (((a >= b) ? (a - b) : (b - a)) > _qualifiers[i].maxDelta)
+			return false;
+	}
 
-		// Seek to corresponding tuple in other, ignoring tuples that
-		// we may not have. If we run off the end of other, the tuple is
-		// missing. This works because tuples are sorted by ID.
-		while (other._qualifiers[otheridx].id != _qualifiers[myidx].id) {
-			++otheridx;
-			if (otheridx >= other._qualifierCount)
+	// If this COM has a full hash of its identity, assume the other must have this as well.
+	// Otherwise we are on a controller that does not incorporate these.
+	if (fullIdentityVerification) {
+		uint64_t idHash[6];
+		otherIdentity.publicKeyHash(idHash);
+		for(unsigned long i=0;i<4;++i) {
+			std::map< uint64_t, uint64_t >::iterator otherQ(otherFields.find((uint64_t)(i + 3)));
+			if (otherQ == otherFields.end())
+				return false;
+			if (otherQ->second != Utils::ntoh(idHash[i]))
 				return false;
 		}
-
-		// Compare to determine if the absolute value of the difference
-		// between these two parameters is within our maxDelta.
-		const uint64_t a = _qualifiers[myidx].value;
-		const uint64_t b = other._qualifiers[myidx].value;
-		if (((a >= b) ? (a - b) : (b - a)) > _qualifiers[myidx].maxDelta)
-			return false;
-
-		++myidx;
 	}
 
 	return true;

node/CertificateOfMembership.hpp

@@ -94,6 +94,8 @@ public:
 		 * ZeroTier address to whom certificate was issued
 		 */
 		COM_RESERVED_ID_ISSUED_TO = 2
+
+		// IDs 3-6 reserved for full hash of identity to which this COM was issued.
 	};
 
 	/**
@@ -110,20 +112,7 @@ public:
 	 * @param nwid Network ID
 	 * @param issuedTo Certificate recipient
 	 */
-	CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Address &issuedTo)
-	{
-		_qualifiers[0].id = COM_RESERVED_ID_TIMESTAMP;
-		_qualifiers[0].value = timestamp;
-		_qualifiers[0].maxDelta = timestampMaxDelta;
-		_qualifiers[1].id = COM_RESERVED_ID_NETWORK_ID;
-		_qualifiers[1].value = nwid;
-		_qualifiers[1].maxDelta = 0;
-		_qualifiers[2].id = COM_RESERVED_ID_ISSUED_TO;
-		_qualifiers[2].value = issuedTo.toInt();
-		_qualifiers[2].maxDelta = 0xffffffffffffffffULL;
-		_qualifierCount = 3;
-		memset(_signature.data,0,ZT_C25519_SIGNATURE_LEN);
-	}
+	CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Identity &issuedTo);
 
 	/**
 	 * Create from binary-serialized COM in buffer
@@ -183,36 +172,6 @@ public:
 		return 0ULL;
 	}
 
-	/**
-	 * Add or update a qualifier in this certificate
-	 *
-	 * Any signature is invalidated and signedBy is set to null.
-	 *
-	 * @param id Qualifier ID
-	 * @param value Qualifier value
-	 * @param maxDelta Qualifier maximum allowed difference (absolute value of difference)
-	 */
-	void setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta);
-	inline void setQualifier(ReservedId id,uint64_t value,uint64_t maxDelta) { setQualifier((uint64_t)id,value,maxDelta); }
-
-#ifdef ZT_SUPPORT_OLD_STYLE_NETCONF
-	/**
-	 * @return String-serialized representation of this certificate
-	 */
-	std::string toString() const;
-
-	/**
-	 * Set this certificate equal to the hex-serialized string
-	 *
-	 * Invalid strings will result in invalid or undefined certificate
-	 * contents. These will subsequently fail validation and comparison.
-	 * Empty strings will result in an empty certificate.
-	 *
-	 * @param s String to deserialize
-	 */
-	void fromString(const char *s);
-#endif // ZT_SUPPORT_OLD_STYLE_NETCONF
-
 	/**
 	 * Compare two certificates for parameter agreement
 	 *
@@ -224,9 +183,10 @@ public:
 	 * tuples present in this cert but not in other result in 'false'.
 	 *
 	 * @param other Cert to compare with
+	 * @param otherIdentity Identity of other node
 	 * @return True if certs agree and 'other' may be communicated with
 	 */
-	bool agreesWith(const CertificateOfMembership &other) const;
+	bool agreesWith(const CertificateOfMembership &other, const Identity &otherIdentity) const;
 
 	/**
 	 * Sign this certificate

node/Constants.hpp

@@ -361,7 +361,7 @@
 /**
  * Maximum number of outgoing packets we monitor for QoS information
  */
-#define ZT_QOS_MAX_OUTSTANDING_RECORDS (1024*16)
+#define ZT_QOS_MAX_OUTSTANDING_RECORDS (1024 * 16)
 
 /**
  * Interval used for rate-limiting the computation of path quality estimates.
@@ -403,117 +403,11 @@
 
 /**
  * All unspecified traffic is put in this bucket. Anything in a bucket with a
- * smaller value is deprioritized. Anything in a bucket with a higher value is
+ * smaller value is de-prioritized. Anything in a bucket with a higher value is
  prioritized over other traffic.
  */
 #define ZT_AQM_DEFAULT_BUCKET 0
 
-/**
- * How often we emit a one-liner bond summary for each peer
- */
-#define ZT_MULTIPATH_BOND_STATUS_INTERVAL 60000
-
-/**
- * How long before we consider a path to be dead in the general sense. This is
- * used while searching for default or alternative paths to try in the absence
- * of direct guidance from the user or a selection policy.
- */
-#define ZT_MULTIPATH_DEFAULT_FAILOVER_INTERVAL 10000
-
-/**
- * How often flows are evaluated
- */
-#define ZT_MULTIPATH_FLOW_CHECK_INTERVAL 10000
-
-/**
- * How long before we consider a flow to be dead and remove it from the
- * policy's list.
- */
-#define ZT_MULTIPATH_FLOW_EXPIRATION_INTERVAL (60000 * 5)
-
-/**
- * How often a flow's statistical counters are reset
- */
-#define ZT_FLOW_STATS_RESET_INTERVAL ZT_MULTIPATH_FLOW_EXPIRATION_INTERVAL
-
-/**
- * Maximum number of flows allowed before we start forcibly forgetting old ones
- */
-#define ZT_FLOW_MAX_COUNT (1024*64)
-
-/**
- * How often flows are rebalanced across link (if at all)
- */
-#define ZT_FLOW_MIN_REBALANCE_INTERVAL 5000
-
-/**
- * How often flows are rebalanced across link (if at all)
- */
-#define ZT_FLOW_REBALANCE_INTERVAL 5000
-
-/**
- * A defensive timer to prevent path quality metrics from being
- * processed too often.
- */
-#define ZT_BOND_BACKGROUND_TASK_MIN_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
-
-/**
- * How often a bonding policy's background tasks are processed,
- * some need more frequent attention than others.
- */
-#define ZT_MULTIPATH_ACTIVE_BACKUP_CHECK_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
-
-/**
- * Minimum amount of time (since a previous transition) before the active-backup bonding
- * policy is allowed to transition to a different link. Only valid for active-backup.
- */
-#define ZT_MULTIPATH_MIN_ACTIVE_BACKUP_AUTOFLOP_INTERVAL 10000
-
-/**
- * How often a peer checks that incoming (and outgoing) traffic on a bonded link is
- * appropriately paired.
- */
-#define ZT_PATH_NEGOTIATION_CHECK_INTERVAL 15000
-
-/**
- * Time horizon for path negotiation paths cutoff
- */
-#define ZT_PATH_NEGOTIATION_CUTOFF_TIME 60000
-
-/**
- * Maximum number of path negotiations within cutoff time
- *
- * This limits response to PATH_NEGOTIATION to CUTOFF_LIMIT responses
- * per CUTOFF_TIME milliseconds per peer to prevent this from being
- * useful for DOS amplification attacks.
- */
-#define ZT_PATH_NEGOTIATION_CUTOFF_LIMIT 8
-
-/**
- * How many times a peer will attempt to petition another peer to synchronize its
- * traffic to the same path before giving up and surrendering to the other peer's preference.
- */
-#define ZT_PATH_NEGOTIATION_TRY_COUNT 3
-
-/**
- * How much greater the quality of a path should be before an
- * optimization procedure triggers a switch.
- */
-#define ZT_MULTIPATH_ACTIVE_BACKUP_OPTIMIZE_MIN_THRESHOLD 0.10
-
-/**
- * Artificially inflates the failover score for paths which meet
- * certain non-performance-related policy ranking criteria.
- */
-#define ZT_MULTIPATH_FAILOVER_HANDICAP_PREFERRED 500
-#define ZT_MULTIPATH_FAILOVER_HANDICAP_PRIMARY 1000
-#define ZT_MULTIPATH_FAILOVER_HANDICAP_NEGOTIATED 5000
-
-/**
- * An indicator that no flow is to be associated with the given packet
- */
-#define ZT_QOS_NO_FLOW -1
-
 /**
  * Timeout for overall peer activity (measured from last receive)
  */
@@ -604,8 +498,8 @@
 #define ZT_ACK_CUTOFF_LIMIT 128
 #define ZT_ACK_DRAINAGE_DIVISOR (1000 / ZT_ACK_CUTOFF_LIMIT)
 
-#define ZT_MULTIPATH_DEFAULT_REFRCTORY_PERIOD 8000
-#define ZT_MULTIPATH_MAX_REFRACTORY_PERIOD 600000
+#define ZT_BOND_DEFAULT_REFRCTORY_PERIOD 8000
+#define ZT_BOND_MAX_REFRACTORY_PERIOD 600000
 
 /**
  * Maximum number of direct path pushes within cutoff time
@@ -641,6 +535,92 @@
  */
 #define ZT_PEER_GENERAL_RATE_LIMIT 1000
 
+
+/**
+ * Minimum allowed amount of time between flow/path optimizations (anti-flapping)
+ */
+#define ZT_BOND_OPTIMIZE_INTERVAL 15000
+
+/**
+ * Maximum number of flows allowed before we start forcibly forgetting old ones
+ */
+#define ZT_FLOW_MAX_COUNT (1024 * 64)
+
+/**
+ * How often we emit a bond summary for each bond
+ */
+#define ZT_BOND_STATUS_INTERVAL 30000
+
+/**
+ * How long before we consider a path to be dead in the general sense. This is
+ * used while searching for default or alternative paths to try in the absence
+ * of direct guidance from the user or a selection policy.
+ */
+#define ZT_BOND_FAILOVER_DEFAULT_INTERVAL 5000
+
+/**
+ * Anything below this value gets into thrashing territory since we divide
+ * this value by ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL to send ECHOs often.
+ */
+#define ZT_BOND_FAILOVER_MIN_INTERVAL 250
+
+/**
+ * How many times per failover interval that an ECHO is sent. This should be
+ * at least 2. Anything more then 4 starts to increase overhead significantly.
+ */
+#define ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL 4
+
+/**
+ * A defensive timer to prevent path quality metrics from being
+ * processed too often.
+ */
+#define ZT_BOND_BACKGROUND_TASK_MIN_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
+
+/**
+ * How often a bonding policy's background tasks are processed,
+ * some need more frequent attention than others.
+ */
+#define ZT_BOND_ACTIVE_BACKUP_CHECK_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
+
+/**
+ * Time horizon for path negotiation paths cutoff
+ */
+#define ZT_PATH_NEGOTIATION_CUTOFF_TIME 60000
+
+/**
+ * Maximum number of path negotiations within cutoff time
+ *
+ * This limits response to PATH_NEGOTIATION to CUTOFF_LIMIT responses
+ * per CUTOFF_TIME milliseconds per peer to prevent this from being
+ * useful for DOS amplification attacks.
+ */
+#define ZT_PATH_NEGOTIATION_CUTOFF_LIMIT 8
+
+/**
+ * How many times a peer will attempt to petition another peer to synchronize its
+ * traffic to the same path before giving up and surrendering to the other peer's preference.
+ */
+#define ZT_PATH_NEGOTIATION_TRY_COUNT 3
+
+/**
+ * How much greater the quality of a path should be before an
+ * optimization procedure triggers a switch.
+ */
+#define ZT_BOND_ACTIVE_BACKUP_OPTIMIZE_MIN_THRESHOLD 0.10
+
+/**
+ * Artificially inflates the failover score for paths which meet
+ * certain non-performance-related policy ranking criteria.
+ */
+#define ZT_BOND_FAILOVER_HANDICAP_PREFERRED  500
+#define ZT_BOND_FAILOVER_HANDICAP_PRIMARY    1000
+#define ZT_BOND_FAILOVER_HANDICAP_NEGOTIATED 5000
+
+/**
+ * An indicator that no flow is to be associated with the given packet
+ */
+#define ZT_QOS_NO_FLOW -1
+
 /**
  * Don't do expensive identity validation more often than this
  *
@@ -666,11 +646,6 @@
  */
 #define ZT_TRUST_EXPIRATION 600000
 
-/**
- * Enable support for older network configurations from older (pre-1.1.6) controllers
- */
-#define ZT_SUPPORT_OLD_STYLE_NETCONF 1
-
 /**
  * Desired buffer size for UDP sockets (used in service and osdep but defined here)
  */

node/Flow.hpp

@@ -1,124 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2025-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#ifndef ZT_FLOW_HPP
-#define ZT_FLOW_HPP
-
-#include "Path.hpp"
-#include "SharedPtr.hpp"
-
-namespace ZeroTier {
-
-/**
- * A protocol flow that is identified by the origin and destination port.
- */
-struct Flow
-{
-	/**
-	 * @param flowId Given flow ID
-	 * @param now Current time
-	 */
-	Flow(int32_t flowId, int64_t now) :
-		_flowId(flowId),
-		_bytesInPerUnitTime(0),
-		_bytesOutPerUnitTime(0),
-		_lastActivity(now),
-		_lastPathReassignment(0),
-		_assignedPath(SharedPtr<Path>())
-	{}
-
-	/**
-	 * Reset flow statistics
-	 */
-	void resetByteCounts()
-	{
-		_bytesInPerUnitTime = 0;
-		_bytesOutPerUnitTime = 0;
-	}
-
-	/**
-	 * @return The Flow's ID
-	 */
-	int32_t id() { return _flowId; }
-
-	/**
-	 * @return Number of incoming bytes processed on this flow per unit time
-	 */
-	int64_t bytesInPerUnitTime() { return _bytesInPerUnitTime; }
-
-	/**
-	 * Record number of incoming bytes on this flow
-	 *
-	 * @param bytes Number of incoming bytes
-	 */
-	void recordIncomingBytes(uint64_t bytes) { _bytesInPerUnitTime += bytes; }
-
-	/**
-	 * @return Number of outgoing bytes processed on this flow per unit time
-	 */
-	int64_t bytesOutPerUnitTime() { return _bytesOutPerUnitTime; }
-
-	/**
-	 * Record number of outgoing bytes on this flow
-	 *
-	 * @param bytes
-	 */
-	void recordOutgoingBytes(uint64_t bytes) { _bytesOutPerUnitTime += bytes; }
-
-	/**
-	 * @return The total number of bytes processed on this flow
-	 */
-	uint64_t totalBytes() { return _bytesInPerUnitTime + _bytesOutPerUnitTime; }
-
-	/**
-	 * How long since a packet was sent or received in this flow
-	 *
-	 * @param now Current time
-	 * @return The age of the flow in terms of last recorded activity
-	 */
-	int64_t age(int64_t now) { return now - _lastActivity; }
-
-	/**
-	 * Record that traffic was processed on this flow at the given time.
-	 *
-	 * @param now Current time
-	 */
-	void updateActivity(int64_t now) { _lastActivity = now; }
-
-	/**
-	 * @return Path assigned to this flow
-	 */
-	SharedPtr<Path> assignedPath() { return _assignedPath; }
-
-	/**
-	 * @param path Assigned path over which this flow should be handled
-	 */
-	void assignPath(const SharedPtr<Path> &path, int64_t now) {
-		_assignedPath = path;
-		_lastPathReassignment = now;
-	}
-
-	AtomicCounter __refCount;
-
-	int32_t _flowId;
-	uint64_t _bytesInPerUnitTime;
-	uint64_t _bytesOutPerUnitTime;
-	int64_t _lastActivity;
-	int64_t _lastPathReassignment;
-	SharedPtr<Path> _assignedPath;
-	SharedPtr<Path> _previouslyAssignedPath;
-};
-
-} // namespace ZeroTier
-
-#endif

node/Identity.hpp

@@ -109,6 +109,18 @@ public:
 	 */
 	inline bool hasPrivate() const { return (_privateKey != (C25519::Private *)0); }
 
+	/**
+	 * Compute a SHA384 hash of this identity's address and public key(s).
+	 * 
+	 * @param sha384buf Buffer with 48 bytes of space to receive hash
+	 */
+	inline void publicKeyHash(void *sha384buf) const
+	{
+		uint8_t address[ZT_ADDRESS_LENGTH];
+		_address.copyTo(address, ZT_ADDRESS_LENGTH);
+		SHA384(sha384buf, address, ZT_ADDRESS_LENGTH, _publicKey.data, ZT_C25519_PUBLIC_KEY_LEN);
+	}
+
 	/**
 	 * Compute the SHA512 hash of our private key (if we have one)
 	 *

node/IncomingPacket.cpp

@@ -88,7 +88,6 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr,int32_t f
 					peer->received(tPtr,_path,hops(),packetId(),payloadLength(),v,0,Packet::VERB_NOP,false,0,ZT_QOS_NO_FLOW);
 					break;
 				case Packet::VERB_HELLO:                      r = _doHELLO(RR,tPtr,true); break;
-				case Packet::VERB_ACK:                        r = _doACK(RR,tPtr,peer); break;
 				case Packet::VERB_QOS_MEASUREMENT:            r = _doQOS_MEASUREMENT(RR,tPtr,peer); break;
 				case Packet::VERB_ERROR:                      r = _doERROR(RR,tPtr,peer); break;
 				case Packet::VERB_OK:                         r = _doOK(RR,tPtr,peer); break;
@@ -191,6 +190,29 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 			}
 		}	break;
 
+		case Packet::ERROR_NETWORK_AUTHENTICATION_REQUIRED: {
+			const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+			if ((network)&&(network->controller() == peer->address())) {
+				bool noUrl = true;
+				int s = (int)size() - (ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8);
+				if (s > 2) {
+					const uint16_t errorDataSize = at<uint16_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8);
+					s -= 2;
+					if (s >= (int)errorDataSize) {
+						Dictionary<3072> authInfo(((const char *)this->data()) + (ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 10), errorDataSize);
+						char authenticationURL[2048];
+						if (authInfo.get("aU", authenticationURL, sizeof(authenticationURL)) > 0) {
+							authenticationURL[sizeof(authenticationURL) - 1] = 0; // ensure always zero terminated
+							network->setAuthenticationRequired(authenticationURL);
+							noUrl = false;
+						}
+					}
+				}
+				if (noUrl)
+					network->setAuthenticationRequired("");
+			}
+		}	break;
+
 		default: break;
 	}
 
@@ -199,35 +221,12 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 	return true;
 }
 
-bool IncomingPacket::_doACK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
-{
-	SharedPtr<Bond> bond = peer->bond();
-	if (!bond || !bond->rateGateACK(RR->node->now())) {
-		return true;
-	}
-	/* Dissect incoming ACK packet. From this we can estimate current throughput of the path, establish known
-	 * maximums and detect packet loss. */
-	int32_t ackedBytes;
-	if (payloadLength() != sizeof(ackedBytes)) {
-		return true; // ignore
-	}
-	memcpy(&ackedBytes, payload(), sizeof(ackedBytes));
-	if (bond) {
-		bond->receivedAck(_path, RR->node->now(), Utils::ntoh(ackedBytes));
-	}
-	return true;
-}
-
 bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
 {
 	SharedPtr<Bond> bond = peer->bond();
-	/* TODO: Fix rate gate issue
-	if (!bond || !bond->rateGateQoS(RR->node->now())) {
+	if (!bond || !bond->rateGateQoS(RR->node->now(), _path)) {
 		return true;
 	}
-	*/
-	/* Dissect incoming QoS packet. From this we can compute latency values and their variance.
-	 * The latency variance is used as a measure of "jitter". */
 	if (payloadLength() > ZT_QOS_MAX_PACKET_SIZE || payloadLength() < ZT_QOS_MIN_PACKET_SIZE) {
 		return true; // ignore
 	}
@@ -1306,7 +1305,7 @@ bool IncomingPacket::_doPATH_NEGOTIATION_REQUEST(const RuntimeEnvironment *RR,vo
 {
 	uint64_t now = RR->node->now();
 	SharedPtr<Bond> bond = peer->bond();
-	if (!bond || !bond->rateGatePathNegotiation(now)) {
+	if (!bond || !bond->rateGatePathNegotiation(now, _path)) {
 		return true;
 	}
 	if (payloadLength() != sizeof(int16_t)) {

node/IncomingPacket.hpp

@@ -112,7 +112,6 @@ private:
 	// been authenticated, decrypted, decompressed, and classified.
 	bool _doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated);
-	bool _doACK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doQOS_MEASUREMENT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);

node/Membership.hpp

@@ -91,13 +91,14 @@ public:
 	 * Check whether the peer represented by this Membership should be allowed on this network at all
 	 *
 	 * @param nconf Our network config
+	 * @param otherNodeIdentity Identity of remote node
 	 * @return True if this peer is allowed on this network at all
 	 */
-	inline bool isAllowedOnNetwork(const NetworkConfig &nconf) const
+	inline bool isAllowedOnNetwork(const NetworkConfig &thisNodeNetworkConfig, const Identity &otherNodeIdentity) const
 	{
-		if (nconf.isPublic()) return true;
+		if (thisNodeNetworkConfig.isPublic()) return true;
 		if (_com.timestamp() <= _comRevocationThreshold) return false;
-		return nconf.com.agreesWith(_com);
+		return thisNodeNetworkConfig.com.agreesWith(_com, otherNodeIdentity);
 	}
 
 	inline bool recentlyAssociated(const int64_t now) const

node/Network.cpp

@@ -1022,6 +1022,7 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD
 		}
 
 		_portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
+		_authenticationURL = nconf.authenticationURL;
 
 		if (saveToDisk) {
 			Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *const d = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
@@ -1226,7 +1227,7 @@ bool Network::gate(void *tPtr,const SharedPtr<Peer> &peer)
 	try {
 		if (_config) {
 			Membership *m = _memberships.get(peer->address());
-			if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) {
+			if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config, peer->identity()))) ) {
 				if (!m)
 					m = &(_membership(peer->address()));
 				if (m->multicastLikeGate(now)) {
@@ -1379,6 +1380,8 @@ ZT_VirtualNetworkStatus Network::_status() const
 			return ZT_NETWORK_STATUS_NOT_FOUND;
 		case NETCONF_FAILURE_NONE:
 			return ((_config) ? ZT_NETWORK_STATUS_OK : ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION);
+		case NETCONF_FAILURE_AUTHENTICATION_REQUIRED:
+			return ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED;
 		default:
 			return ZT_NETWORK_STATUS_PORT_ERROR;
 	}
@@ -1429,6 +1432,10 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	}
 
 	memcpy(&ec->dns, &_config.dns, sizeof(ZT_VirtualNetworkDNS));
+
+	Utils::scopy(ec->authenticationURL, sizeof(ec->authenticationURL), _authenticationURL.c_str());
+	ec->authenticationExpiryTime = _config.authenticationExpiryTime;
+	ec->ssoEnabled = _config.ssoEnabled;
 }
 
 void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup)
@@ -1480,8 +1487,11 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu
 		Membership *m = (Membership *)0;
 		Hashtable<Address,Membership>::Iterator i(_memberships);
 		while (i.next(a,m)) {
-			if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config)) && (!std::binary_search(alwaysAnnounceTo.begin(),alwaysAnnounceTo.end(),*a)) )
-				_announceMulticastGroupsTo(tPtr,*a,groups);
+			const Identity remoteIdentity(RR->topology->getIdentity(tPtr, *a));
+			if (remoteIdentity) {
+				if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config, remoteIdentity)) && (!std::binary_search(alwaysAnnounceTo.begin(),alwaysAnnounceTo.end(),*a)) )
+					_announceMulticastGroupsTo(tPtr,*a,groups);
+			}
 		}
 	}
 }

node/Network.hpp

@@ -220,6 +220,17 @@ public:
 		_netconfFailure = NETCONF_FAILURE_NOT_FOUND;
 	}
 
+	/**
+	 * Set netconf failure to 'authentication required' possibly with an authorization URL
+	 */
+	inline void setAuthenticationRequired(const char *url)
+	{
+		Mutex::Lock _l(_lock);
+		_netconfFailure = NETCONF_FAILURE_AUTHENTICATION_REQUIRED;
+		_authenticationURL = (url) ? url : "";
+		_config.ssoEnabled = true;
+	}	
+
 	/**
 	 * Causes this network to request an updated configuration from its master node now
 	 *
@@ -435,9 +446,11 @@ private:
 		NETCONF_FAILURE_NONE,
 		NETCONF_FAILURE_ACCESS_DENIED,
 		NETCONF_FAILURE_NOT_FOUND,
-		NETCONF_FAILURE_INIT_FAILED
+		NETCONF_FAILURE_INIT_FAILED,
+		NETCONF_FAILURE_AUTHENTICATION_REQUIRED
 	} _netconfFailure;
 	int _portError; // return value from port config callback
+	std::string _authenticationURL;
 
 	Hashtable<Address,Membership> _memberships;
 

node/NetworkConfig.cpp

@@ -182,6 +182,14 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 			if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_DNS,*tmp)) return false;
 		}
 
+		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_SSO_ENABLED, this->ssoEnabled)) return false;
+		if (this->ssoEnabled) {
+			if (this->authenticationURL[0]) {
+				if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL, this->authenticationURL)) return false;
+			}
+			if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_EXPIRY_TIME, this->authenticationExpiryTime)) return false;
+		}
+
 		delete tmp;
 	} catch ( ... ) {
 		delete tmp;
@@ -365,6 +373,20 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 				unsigned int p = 0;
 				DNS::deserializeDNS(*tmp, p, &dns);
 			}
+
+
+			this->ssoEnabled = d.getB(ZT_NETWORKCONFIG_DICT_KEY_SSO_ENABLED, false);
+			if (this->ssoEnabled) {
+				if (d.get(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL, this->authenticationURL, (unsigned int)sizeof(this->authenticationURL)) > 0) {
+					this->authenticationURL[sizeof(this->authenticationURL) - 1] = 0; // ensure null terminated
+				} else {
+					this->authenticationURL[0] = 0;
+				}
+				this->authenticationExpiryTime = d.getI(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_EXPIRY_TIME, 0);
+			} else {
+				this->authenticationURL[0] = 0;
+				this->authenticationExpiryTime = 0;
+			}
 		}
 
 		//printf("~~~\n%s\n~~~\n",d.data());

node/NetworkConfig.hpp

@@ -94,7 +94,7 @@
 namespace ZeroTier {
 
 // Dictionary capacity needed for max size network config
-#define ZT_NETWORKCONFIG_DICT_CAPACITY (1024 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
+#define ZT_NETWORKCONFIG_DICT_CAPACITY (4096 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
 
 // Dictionary capacity needed for max size network meta-data
 #define ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY 1024
@@ -178,6 +178,12 @@ namespace ZeroTier {
 #define ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP "COO"
 // dns (binary blobs)
 #define ZT_NETWORKCONFIG_DICT_KEY_DNS "DNS"
+// sso enabld
+#define ZT_NETWORKCONFIG_DICT_KEY_SSO_ENABLED "ssoe"
+// authentication URL
+#define ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL "aurl"
+// authentication expiry
+#define ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_EXPIRY_TIME "aexpt"
 
 // Legacy fields -- these are obsoleted but are included when older clients query
 
@@ -233,7 +239,10 @@ public:
 		tags(),
 		certificatesOfOwnership(),
 		type(ZT_NETWORK_TYPE_PRIVATE),
-		dnsCount(0)
+		dnsCount(0),
+		ssoEnabled(false),
+		authenticationURL(),
+		authenticationExpiryTime(0)
 	{
 		name[0] = 0;
 		memset(specialists, 0, sizeof(uint64_t)*ZT_MAX_NETWORK_SPECIALISTS);
@@ -604,6 +613,21 @@ public:
 	 * ZT pushed DNS configuration
 	 */
 	ZT_VirtualNetworkDNS dns;
+
+	/**
+	 * SSO enabled flag.
+	 */
+	bool ssoEnabled;
+
+	/**
+	 * Authentication URL if authentication is required
+	 */
+	char authenticationURL[2048];
+
+	/**
+	 * Time current authentication expires or 0 if external authentication is disabled
+	 */
+	uint64_t authenticationExpiryTime;
 };
 
 } // namespace ZeroTier

node/NetworkController.hpp

@@ -38,7 +38,8 @@ public:
 		NC_ERROR_NONE = 0,
 		NC_ERROR_OBJECT_NOT_FOUND = 1,
 		NC_ERROR_ACCESS_DENIED = 2,
-		NC_ERROR_INTERNAL_SERVER_ERROR = 3
+		NC_ERROR_INTERNAL_SERVER_ERROR = 3,
+		NC_ERROR_AUTHENTICATION_REQUIRED = 4
 	};
 
 	/**
@@ -69,12 +70,17 @@ public:
 		/**
 		 * Send a network configuration request error
 		 *
+		 * If errorData/errorDataSize are provided they must point to a valid serialized
+		 * Dictionary containing error data. They can be null/zero if not specified.
+		 * 
 		 * @param nwid Network ID
 		 * @param requestPacketId Request packet ID or 0 if none
 		 * @param destination Destination peer Address
 		 * @param errorCode Error code
+		 * @param errorData Data associated with error or NULL if none
+		 * @param errorDataSize Size of errorData in bytes
 		 */
-		virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) = 0;
+		virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode, const void *errorData, unsigned int errorDataSize) = 0;
 	};
 
 	NetworkController() {}

node/Node.cpp

@@ -103,7 +103,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		const unsigned long mcs = sizeof(Multicaster) + (((sizeof(Multicaster) & 0xf) != 0) ? (16 - (sizeof(Multicaster) & 0xf)) : 0);
 		const unsigned long topologys = sizeof(Topology) + (((sizeof(Topology) & 0xf) != 0) ? (16 - (sizeof(Topology) & 0xf)) : 0);
 		const unsigned long sas = sizeof(SelfAwareness) + (((sizeof(SelfAwareness) & 0xf) != 0) ? (16 - (sizeof(SelfAwareness) & 0xf)) : 0);
-		const unsigned long bc = sizeof(BondController) + (((sizeof(BondController) & 0xf) != 0) ? (16 - (sizeof(BondController) & 0xf)) : 0);
+		const unsigned long bc = sizeof(Bond) + (((sizeof(Bond) & 0xf) != 0) ? (16 - (sizeof(Bond) & 0xf)) : 0);
 
 		m = reinterpret_cast<char *>(::malloc(16 + ts + sws + mcs + topologys + sas + bc));
 		if (!m)
@@ -121,14 +121,14 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		m += topologys;
 		RR->sa = new (m) SelfAwareness(RR);
 		m += sas;
-		RR->bc = new (m) BondController(RR);
+		RR->bc = new (m) Bond(RR);
 	} catch ( ... ) {
 		if (RR->sa) RR->sa->~SelfAwareness();
 		if (RR->topology) RR->topology->~Topology();
 		if (RR->mc) RR->mc->~Multicaster();
 		if (RR->sw) RR->sw->~Switch();
 		if (RR->t) RR->t->~Trace();
-		if (RR->bc) RR->bc->~BondController();
+		if (RR->bc) RR->bc->~Bond();
 		::free(m);
 		throw;
 	}
@@ -147,7 +147,7 @@ Node::~Node()
 	if (RR->mc) RR->mc->~Multicaster();
 	if (RR->sw) RR->sw->~Switch();
 	if (RR->t) RR->t->~Trace();
-	if (RR->bc) RR->bc->~BondController();
+	if (RR->bc) RR->bc->~Bond();
 	::free(RR->rtmem);
 }
 
@@ -252,18 +252,14 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 	_now = now;
 	Mutex::Lock bl(_backgroundTasksLock);
 
-
-	unsigned long bondCheckInterval = ZT_CORE_TIMER_TASK_GRANULARITY;
+	// Process background bond tasks
+	unsigned long bondCheckInterval = ZT_PING_CHECK_INVERVAL;
 	if (RR->bc->inUse()) {
-		// Gratuitously ping active peers so that QoS metrics have enough data to work with (if active path monitoring is enabled)
-		bondCheckInterval = std::min(std::max(RR->bc->minReqPathMonitorInterval(), ZT_CORE_TIMER_TASK_GRANULARITY), ZT_PING_CHECK_INVERVAL);
-		if ((now - _lastGratuitousPingCheck) >= bondCheckInterval) {
-			Hashtable< Address,std::vector<InetAddress> > alwaysContact;
-			_PingPeersThatNeedPing pfunc(RR,tptr,alwaysContact,now);
-			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
+		bondCheckInterval = std::max(RR->bc->minReqMonitorInterval(), ZT_CORE_TIMER_TASK_GRANULARITY);
+		if ((now - _lastGratuitousPingCheck) >= ZT_CORE_TIMER_TASK_GRANULARITY) {
 			_lastGratuitousPingCheck = now;
+			RR->bc->processBackgroundTasks(tptr, now);
 		}
-		RR->bc->processBackgroundTasks(tptr, now);
 	}
 
 	unsigned long timeUntilNextPingCheck = ZT_PING_CHECK_INVERVAL;
@@ -512,7 +508,7 @@ ZT_PeerList *Node::peers() const
 		}
 		if (pi->second->bond()) {
 			p->isBonded = pi->second->bond();
-			p->bondingPolicy = pi->second->bond()->getPolicy();
+			p->bondingPolicy = pi->second->bond()->policy();
 			p->isHealthy = pi->second->bond()->isHealthy();
 			p->numAliveLinks = pi->second->bond()->getNumAliveLinks();
 			p->numTotalLinks = pi->second->bond()->getNumTotalLinks();
@@ -731,7 +727,7 @@ void Node::ncSendRevocation(const Address &destination,const Revocation &rev)
 	}
 }
 
-void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode)
+void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode, const void *errorData, unsigned int errorDataSize)
 {
 	if (destination == RR->identity.address()) {
 		SharedPtr<Network> n(network(nwid));
@@ -744,6 +740,9 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 			case NetworkController::NC_ERROR_ACCESS_DENIED:
 				n->setAccessDenied();
 				break;
+			case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED: {
+			}
+				break;
 
 			default: break;
 		}
@@ -760,8 +759,18 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 			case NetworkController::NC_ERROR_ACCESS_DENIED:
 				outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 				break;
+			case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED:
+				outp.append((unsigned char)Packet::ERROR_NETWORK_AUTHENTICATION_REQUIRED);
+				break;
 		}
+
 		outp.append(nwid);
+
+		if ((errorData)&&(errorDataSize > 0)&&(errorDataSize <= 0xffff)) {
+			outp.append((uint16_t)errorDataSize);
+			outp.append(errorData, errorDataSize);
+		}
+
 		RR->sw->send((void *)0,outp,true);
 	} // else we can't send an ERROR() in response to nothing, so discard
 }

node/Node.hpp

@@ -34,7 +34,7 @@
 #include "Salsa20.hpp"
 #include "NetworkController.hpp"
 #include "Hashtable.hpp"
-#include "BondController.hpp"
+#include "Bond.hpp"
 
 // Bit mask for "expecting reply" hash
 #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255
@@ -187,7 +187,7 @@ public:
 
 	inline const Identity &identity() const { return _RR.identity; }
 
-	inline BondController *bondController() const { return _RR.bc; }
+	inline Bond *bondController() const { return _RR.bc; }
 
 	/**
 	 * Register that we are expecting a reply to a packet ID
@@ -245,7 +245,7 @@ public:
 
 	virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig);
 	virtual void ncSendRevocation(const Address &destination,const Revocation &rev);
-	virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode);
+	virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode, const void *errorData, unsigned int errorDataSize);
 
 	inline const Address &remoteTraceTarget() const { return _remoteTraceTarget; }
 	inline Trace::Level remoteTraceLevel() const { return _remoteTraceLevel; }

node/Packet.hpp

@@ -792,6 +792,12 @@ public:
 		 *
 		 * ERROR response payload:
 		 *   <[8] 64-bit network ID>
+     *   <[2] 16-bit length of error-related data (optional)>
+     *   <[...] error-related data (optional)>
+     * 
+     * Error related data is a Dictionary containing things like a URL
+     * for authentication or a human-readable error message, and is
+     * optional and may be absent or empty.
 		 */
 		VERB_NETWORK_CONFIG_REQUEST = 0x0b,
 
@@ -1076,7 +1082,10 @@ public:
 		ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */
 
 		/* Multicasts to this group are not wanted */
-		ERROR_UNWANTED_MULTICAST = 0x08
+		ERROR_UNWANTED_MULTICAST = 0x08,
+
+    /* Network requires external or 2FA authentication (e.g. SSO). */
+    ERROR_NETWORK_AUTHENTICATION_REQUIRED = 0x09
 	};
 
 	template<unsigned int C2>

node/Path.hpp

@@ -29,8 +29,6 @@
 #include "Packet.hpp"
 #include "RingBuffer.hpp"
 
-#include "../osdep/Link.hpp"
-
 /**
  * Maximum return value of preferenceRank()
  */
@@ -88,46 +86,7 @@ public:
 		_localSocket(-1),
 		_latency(0xffff),
 		_addr(),
-		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_lastAckReceived(0),
-		_lastAckSent(0),
-		_lastQoSMeasurement(0),
-		_lastThroughputEstimation(0),
-		_lastRefractoryUpdate(0),
-		_lastAliveToggle(0),
-		_lastEligibilityState(false),
-		_lastTrialBegin(0),
-		_refractoryPeriod(0),
-		_monitorInterval(0),
-		_upDelay(0),
-		_downDelay(0),
-		_ipvPref(0),
-		_mode(0),
-		_onlyPathOnLink(false),
-		_enabled(false),
-		_bonded(false),
-		_negotiated(false),
-		_deprecated(false),
-		_shouldReallocateFlows(false),
-		_assignedFlowCount(0),
-		_latencyMean(0),
-		_latencyVariance(0),
-		_packetLossRatio(0),
-		_packetErrorRatio(0),
-		_throughputMean(0),
-		_throughputMax(0),
-		_throughputVariance(0),
-		_allocation(0),
-		_byteLoad(0),
-		_relativeByteLoad(0),
-		_affinity(0),
-		_failoverScore(0),
-		_unackedBytes(0),
-		_packetsReceivedSinceLastAck(0),
-		_packetsReceivedSinceLastQoS(0),
-		_bytesAckedSinceLastThroughputEstimation(0),
-		_packetsIn(0),
-		_packetsOut(0)
+		_ipScope(InetAddress::IP_SCOPE_NONE)
 		{}
 
 	Path(const int64_t localSocket,const InetAddress &addr) :
@@ -137,46 +96,7 @@ public:
 		_localSocket(localSocket),
 		_latency(0xffff),
 		_addr(addr),
-		_ipScope(addr.ipScope()),
-		_lastAckReceived(0),
-		_lastAckSent(0),
-		_lastQoSMeasurement(0),
-		_lastThroughputEstimation(0),
-		_lastRefractoryUpdate(0),
-		_lastAliveToggle(0),
-		_lastEligibilityState(false),
-		_lastTrialBegin(0),
-		_refractoryPeriod(0),
-		_monitorInterval(0),
-		_upDelay(0),
-		_downDelay(0),
-		_ipvPref(0),
-		_mode(0),
-		_onlyPathOnLink(false),
-		_enabled(false),
-		_bonded(false),
-		_negotiated(false),
-		_deprecated(false),
-		_shouldReallocateFlows(false),
-		_assignedFlowCount(0),
-		_latencyMean(0),
-		_latencyVariance(0),
-		_packetLossRatio(0),
-		_packetErrorRatio(0),
-		_throughputMean(0),
-		_throughputMax(0),
-		_throughputVariance(0),
-		_allocation(0),
-		_byteLoad(0),
-		_relativeByteLoad(0),
-		_affinity(0),
-		_failoverScore(0),
-		_unackedBytes(0),
-		_packetsReceivedSinceLastAck(0),
-		_packetsReceivedSinceLastQoS(0),
-		_bytesAckedSinceLastThroughputEstimation(0),
-		_packetsIn(0),
-		_packetsOut(0)
+		_ipScope(addr.ipScope())
 	{}
 
 	/**
@@ -186,9 +106,6 @@ public:
 	 */
 	inline void received(const uint64_t t)
 	{
-		if (!alive(t,_bonded)) {
-			_lastAliveToggle = _lastIn;
-		}
 		_lastIn = t;
 	}
 
@@ -317,21 +234,11 @@ public:
 		return (((age < (ZT_PATH_HEARTBEAT_PERIOD + 5000)) ? l : (l + 0xffff + age)) * (long)((ZT_INETADDRESS_MAX_SCOPE - _ipScope) + 1));
 	}
 
-	/**
-	 * @param bonded Whether this path is part of a bond.
-	 */
-	inline void setBonded(bool bonded) { _bonded = bonded; }
-
-	/**
-	 * @return True if this path is currently part of a bond.
-	 */
-	inline bool bonded() { return _bonded; }
-
 	/**
 	 * @return True if this path is alive (receiving heartbeats)
 	 */
-	inline bool alive(const int64_t now, bool bondingEnabled = false) const {
-		return (bondingEnabled && _monitorInterval) ? ((now - _lastIn) < (_monitorInterval * 3)) : ((now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000));
+	inline bool alive(const int64_t now) const {
+		return (now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000);
 	}
 
 	/**
@@ -339,11 +246,6 @@ public:
 	 */
 	inline bool needsHeartbeat(const int64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); }
 
-	/**
-	 * @return True if this path needs a heartbeat in accordance to the user-specified path monitor frequency
-	 */
-	inline bool needsGratuitousHeartbeat(const int64_t now) { return allowed() && (_monitorInterval > 0) && ((now - _lastOut) >= _monitorInterval); }
-
 	/**
 	 * @return Last time we sent something
 	 */
@@ -364,134 +266,7 @@ public:
 	 */
 	inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }
 
-	/**
-	 * @return Time since last VERB_ACK was received
-	 */
-	inline int64_t ackAge(int64_t now) { return _lastAckReceived ? now - _lastAckReceived : 0; }
-
-	/**
-	 * Set or update a refractory period for the path.
-	 *
-	 * @param punishment How much a path should be punished
-	 * @param pathFailure Whether this call is the result of a recent path failure
-	 */
-	inline void adjustRefractoryPeriod(int64_t now, uint32_t punishment, bool pathFailure) {
-		if (pathFailure) {
-			unsigned int suggestedRefractoryPeriod = _refractoryPeriod ? punishment + (_refractoryPeriod * 2) : punishment;
-			_refractoryPeriod = std::min(suggestedRefractoryPeriod, (unsigned int)ZT_MULTIPATH_MAX_REFRACTORY_PERIOD);
-			_lastRefractoryUpdate = 0;
-		} else {
-			uint32_t drainRefractory = 0;
-			if (_lastRefractoryUpdate) {
-				drainRefractory = (now - _lastRefractoryUpdate);
-			} else {
-				drainRefractory = (now - _lastAliveToggle);
-			}
-			_lastRefractoryUpdate = now;
-			if (_refractoryPeriod > drainRefractory) {
-				_refractoryPeriod -= drainRefractory;
-			} else {
-				_refractoryPeriod = 0;
-				_lastRefractoryUpdate = 0;
-			}
-		}
-	}
-
-	/**
-	 * Determine the current state of eligibility of the path.
-	 *
-	 * @param includeRefractoryPeriod Whether current punishment should be taken into consideration
-	 * @return True if this path can be used in a bond at the current time
-	 */
-	inline bool eligible(uint64_t now, int ackSendInterval, bool includeRefractoryPeriod = false) {
-		if (includeRefractoryPeriod && _refractoryPeriod) {
-			return false;
-		}
-		bool acceptableAge    = age(now) < ((_monitorInterval * 4) + _downDelay); // Simple RX age (driven by packets of any type and gratuitous VERB_HELLOs)
-		bool acceptableAckAge = ackAge(now) < (ackSendInterval); // Whether the remote peer is actually responding to our outgoing traffic or simply sending stuff to us
-		bool notTooEarly      = (now - _lastAliveToggle) >= _upDelay; // Whether we've waited long enough since the link last came online
-		bool inTrial          = (now - _lastTrialBegin) < _upDelay; // Whether this path is still in its trial period
-		bool currEligibility  = allowed() && (((acceptableAge || acceptableAckAge) && notTooEarly) || inTrial);
-		return currEligibility;
-	}
-
-	/**
-	 * Record when this path first entered the bond. Each path is given a trial period where it is admitted
-	 * to the bond without requiring observations to prove its performance or reliability.
-	 */
-	inline void startTrial(uint64_t now) { _lastTrialBegin = now; }
-
-	/**
-	 * @return True if a path is permitted to be used in a bond (according to user pref.)
-	 */
-	inline bool allowed() {
-		return _enabled
-			&& (!_ipvPref
-				|| ((_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46 || _ipvPref == 64))
-				|| ((_addr.isV6() && (_ipvPref == 6 || _ipvPref == 46 || _ipvPref == 64)))));
-	}
-
-	/**
-	 * @return True if a path is preferred over another on the same physical link (according to user pref.)
-	 */
-	inline bool preferred() {
-		return _onlyPathOnLink
-			|| (_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46))
-			|| (_addr.isV6() && (_ipvPref == 6 || _ipvPref == 64));
-	}
-
-	/**
-	 * @param now Current time
-	 * @return Whether an ACK (VERB_ACK) packet needs to be emitted at this time
-	 */
-	inline bool needsToSendAck(int64_t now, int ackSendInterval) {
-		return ((now - _lastAckSent) >= ackSendInterval ||
-			(_packetsReceivedSinceLastAck == ZT_QOS_TABLE_SIZE)) && _packetsReceivedSinceLastAck;
-	}
-
-	/**
-	 * @param now Current time
-	 * @return Whether a QoS (VERB_QOS_MEASUREMENT) packet needs to be emitted at this time
-	 */
-	inline bool needsToSendQoS(int64_t now, int qosSendInterval) {
-		return ((_packetsReceivedSinceLastQoS >= ZT_QOS_TABLE_SIZE) ||
-			((now - _lastQoSMeasurement) > qosSendInterval)) && _packetsReceivedSinceLastQoS;
-	}
-
-	/**
-	 * Reset packet counters
-	 */
-	inline void resetPacketCounts()
-	{
-		_packetsIn = 0;
-		_packetsOut = 0;
-	}
-
-
-	/**
-	 * The mean latency (computed from a sliding window.)
-	 */
-	float latencyMean() { return _latencyMean; }
-
-	/**
-	 * Packet delay variance (computed from a sliding window.)
-	 */
-	float latencyVariance() { return _latencyVariance; }
-
-	/**
-	 * The ratio of lost packets to received packets.
-	 */
-	float packetLossRatio() { return _packetLossRatio; }
-
-	/**
-	 * The ratio of packets that failed their MAC/CRC checks to those that did not.
-	 */
-	float packetErrorRatio() { return _packetErrorRatio; }
-
-	/**
-	*
-	*/
-	uint8_t allocation() { return _allocation; }
+	void *_bondingMetricPtr;
 
 private:
 
@@ -503,212 +278,6 @@ private:
 	InetAddress _addr;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 	AtomicCounter __refCount;
-
-	std::map<uint64_t,uint64_t> qosStatsOut; // id:egress_time
-	std::map<uint64_t,uint64_t> qosStatsIn; // id:now
-	std::map<uint64_t,uint16_t> ackStatsIn; // id:len
-
-	RingBuffer<int,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordSize;
-	RingBuffer<float,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordLossSamples;
-	RingBuffer<uint64_t,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> throughputSamples;
-	RingBuffer<bool,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> packetValiditySamples;
-	RingBuffer<float,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> _throughputVarianceSamples;
-	RingBuffer<uint16_t,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> latencySamples;
-
-	/**
-	 * Last time that a VERB_ACK was received on this path.
-	 */
-	uint64_t _lastAckReceived;
-
-	/**
-	 * Last time that a VERB_ACK was sent out on this path.
-	 */
-	uint64_t _lastAckSent;
-
-	/**
-	 * Last time that a VERB_QOS_MEASUREMENT was sent out on this path.
-	 */
-	uint64_t _lastQoSMeasurement;
-
-	/**
-	 * Last time that the path's throughput was estimated.
-	 */
-	uint64_t _lastThroughputEstimation;
-
-	/**
-	 * The last time that the refractory period was updated.
-	 */
-	uint64_t _lastRefractoryUpdate;
-
-	/**
-	 * The last time that the path was marked as "alive".
-	 */
-	uint64_t _lastAliveToggle;
-
-	/**
-	 * State of eligibility at last check. Used for determining state changes.
-	 */
-	bool _lastEligibilityState;
-
-	/**
-	 * Timestamp indicating when this path's trial period began.
-	 */
-	uint64_t _lastTrialBegin;
-
-	/**
-	 * Amount of time that this path will be prevented from becoming a member of a bond.
-	 */
-	uint32_t _refractoryPeriod;
-
-	/**
-	 * Monitor interval specific to this path or that was inherited from the bond controller.
-	 */
-	int32_t _monitorInterval;
-
-	/**
-	 * Up delay interval specific to this path or that was inherited from the bond controller.
-	 */
-	uint32_t _upDelay;
-
-	/**
-	 * Down delay interval specific to this path or that was inherited from the bond controller.
-	 */
-	uint32_t _downDelay;
-
-	/**
-	 * IP version preference inherited from the physical link.
-	 */
-	uint8_t _ipvPref;
-
-	/**
-	 * Mode inherited from the physical link.
-	 */
-	uint8_t _mode;
-
-	/**
-	 * IP version preference inherited from the physical link.
-	 */
-	bool _onlyPathOnLink;
-
-	/**
-	 * Enabled state inherited from the physical link.
-	 */
-	bool _enabled;
-
-	/**
-	 * Whether this path is currently part of a bond.
-	 */
-	bool _bonded;
-
-	/**
-	 * Whether this path was intentionally negotiated by either peer.
-	 */
-	bool _negotiated;
-
-	/**
-	 * Whether this path has been deprecated due to performance issues. Current traffic flows
-	 * will be re-allocated to other paths in the most non-disruptive manner (if possible),
-	 * and new traffic will not be allocated to this path.
-	 */
-	bool _deprecated;
-
-	/**
-	 * Whether flows should be moved from this path. Current traffic flows will be re-allocated
-	 * immediately.
-	 */
-	bool _shouldReallocateFlows;
-
-	/**
-	 * The number of flows currently assigned to this path.
-	 */
-	uint16_t _assignedFlowCount;
-
-	/**
-	 * The mean latency (computed from a sliding window.)
-	 */
-	float _latencyMean;
-
-	/**
-	 * Packet delay variance (computed from a sliding window.)
-	 */
-	float _latencyVariance;
-
-	/**
-	 * The ratio of lost packets to received packets.
-	 */
-	float _packetLossRatio;
-
-	/**
-	 * The ratio of packets that failed their MAC/CRC checks to those that did not.
-	 */
-	float _packetErrorRatio;
-
-	/**
-	 * The estimated mean throughput of this path.
-	 */
-	uint64_t _throughputMean;
-
-	/**
-	 * The maximum observed throughput of this path.
-	 */
-	uint64_t _throughputMax;
-
-	/**
-	 * The variance in the estimated throughput of this path.
-	 */
-	float _throughputVariance;
-
-	/**
-	 * The relative quality of this path to all others in the bond, [0-255].
-	 */
-	uint8_t _allocation;
-
-	/**
-	 * How much load this path is under.
-	 */
-	uint64_t _byteLoad;
-
-	/**
-	 * How much load this path is under (relative to other paths in the bond.)
-	 */
-	uint8_t _relativeByteLoad;
-
-	/**
-	 * Relative value expressing how "deserving" this path is of new traffic.
-	 */
-	uint8_t _affinity;
-
-	/**
-	 * Score that indicates to what degree this path is preferred over others that
-	 * are available to the bonding policy. (specifically for active-backup)
-	 */
-	uint32_t _failoverScore;
-
-	/**
-	 * Number of bytes thus far sent that have not been acknowledged by the remote peer.
-	 */
-	int64_t _unackedBytes;
-
-	/**
-	 * Number of packets received since the last VERB_ACK was sent to the remote peer.
-	 */
-	int32_t _packetsReceivedSinceLastAck;
-
-	/**
-	 * Number of packets received since the last VERB_QOS_MEASUREMENT was sent to the remote peer.
-	 */
-	int32_t _packetsReceivedSinceLastQoS;
-
-	/**
-	 * Bytes acknowledged via incoming VERB_ACK since the last estimation of throughput.
-	 */
-	uint64_t _bytesAckedSinceLastThroughputEstimation;
-
-	/**
-	 * Counters used for tracking path load.
-	 */
-	int _packetsIn;
-	int _packetsOut;
 };
 
 } // namespace ZeroTier

node/Peer.cpp

@@ -50,12 +50,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_directPathPushCutoffCount(0),
 	_credentialsCutoffCount(0),
 	_echoRequestCutoffCount(0),
-	_uniqueAlivePathCount(0),
 	_localMultipathSupported(false),
-	_remoteMultipathSupported(false),
-	_canUseMultipath(false),
-	_shouldCollectPathStatistics(0),
-	_bondingPolicy(0),
 	_lastComputedAggregateMeanLatency(0)
 {
 	if (!myIdentity.agree(peerIdentity,_key))
@@ -151,6 +146,10 @@ void Peer::received(
 					_paths[replacePath].lr = now;
 					_paths[replacePath].p = path;
 					_paths[replacePath].priority = 1;
+					Mutex::Lock _l(_bond_m);
+					if(_bond) {
+						_bond->nominatePathToBond(_paths[replacePath].p, now);
+					}
 				}
 			} else {
 				Mutex::Lock ltl(_lastTriedPath_m);
@@ -229,7 +228,8 @@ void Peer::received(
 
 SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int32_t flowId)
 {
-	if (!_bondToPeer) {
+	Mutex::Lock _l(_bond_m);
+	if (!_bond) {
 		Mutex::Lock _l(_paths_m);
 		unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS;
 		/**
@@ -253,7 +253,7 @@ SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int32
 		}
 		return SharedPtr<Path>();
 	}
-	return _bondToPeer->getAppropriatePath(now, flowId);
+	return _bond->getAppropriatePath(now, flowId);
 }
 
 void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &other) const
@@ -444,39 +444,32 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now)
 
 void Peer::performMultipathStateCheck(void *tPtr, int64_t now)
 {
+	Mutex::Lock _l(_bond_m);
+	if (_bond) {
+		// Once enabled the Bond object persists, no need to update state
+		return;
+	}
 	/**
 	 * Check for conditions required for multipath bonding and create a bond
 	 * if allowed.
 	 */
-	_localMultipathSupported = ((RR->bc->inUse()) && (ZT_PROTO_VERSION > 9));
-	if (_localMultipathSupported) {
-		int currAlivePathCount = 0;
-		int duplicatePathsFound = 0;
-		for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
-			if (_paths[i].p) {
-				currAlivePathCount++;
-				for (unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) {
-					if (_paths[i].p && _paths[j].p && _paths[i].p->address().ipsEqual2(_paths[j].p->address()) && i != j) {
-						duplicatePathsFound+=1;
-						break;
-					}
-				}
-			}
+	int numAlivePaths = 0;
+	for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+		if (_paths[i].p && _paths[i].p->alive(now)) {
+			numAlivePaths++;
 		}
-		_uniqueAlivePathCount = (currAlivePathCount - (duplicatePathsFound / 2));
-		_remoteMultipathSupported = _vProto > 9;
-		_canUseMultipath = _localMultipathSupported && _remoteMultipathSupported && (_uniqueAlivePathCount > 1);
 	}
-	if (_canUseMultipath && !_bondToPeer) {
+	_localMultipathSupported = ((numAlivePaths >= 1) && (RR->bc->inUse()) && (ZT_PROTO_VERSION > 9));
+	if (_localMultipathSupported && !_bond) {
 		if (RR->bc) {
-			_bondToPeer = RR->bc->createTransportTriggeredBond(RR, this);
+			_bond = RR->bc->createTransportTriggeredBond(RR, this);
 			/**
 			 * Allow new bond to retroactively learn all paths known to this peer
 			 */
-			if (_bondToPeer) {
+			if (_bond) {
 				for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
 					if (_paths[i].p) {
-						_bondToPeer->nominatePath(_paths[i].p, now);
+						_bond->nominatePathToBond(_paths[i].p, now);
 					}
 				}
 			}
@@ -510,8 +503,7 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
 		if (_paths[i].p) {
 			// Clean expired and reduced priority paths
 			if ( ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) && (_paths[i].priority == maxPriority) ) {
-				if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))
-					|| (_canUseMultipath && _paths[i].p->needsGratuitousHeartbeat(now))) {
+				if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))) {
 					attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,sendFullHello);
 					_paths[i].p->sent(now);
 					sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2;
@@ -591,27 +583,24 @@ void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddres
 void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
 	uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
 {
-	if (!_shouldCollectPathStatistics || !_bondToPeer) {
-		return;
+	if (_localMultipathSupported && _bond) {
+		_bond->recordOutgoingPacket(path, packetId, payloadLength, verb, flowId, now);
 	}
-	_bondToPeer->recordOutgoingPacket(path, packetId, payloadLength, verb, flowId, now);
 }
 
 void Peer::recordIncomingInvalidPacket(const SharedPtr<Path>& path)
 {
-	if (!_shouldCollectPathStatistics || !_bondToPeer) {
-		return;
+	if (_localMultipathSupported && _bond) {
+		_bond->recordIncomingInvalidPacket(path);
 	}
-	_bondToPeer->recordIncomingInvalidPacket(path);
 }
 
 void Peer::recordIncomingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
 	uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
 {
-	if (!_shouldCollectPathStatistics || !_bondToPeer) {
-		return;
+	if (_localMultipathSupported && _bond) {
+		_bond->recordIncomingPacket(path, packetId, payloadLength, verb, flowId, now);
 	}
-	_bondToPeer->recordIncomingPacket(path, packetId, payloadLength, verb, flowId, now);
 }
 
 } // namespace ZeroTier

node/Peer.hpp

@@ -33,7 +33,6 @@
 #include "Hashtable.hpp"
 #include "Mutex.hpp"
 #include "Bond.hpp"
-#include "BondController.hpp"
 #include "AES.hpp"
 
 #define ZT_PEER_MAX_SERIALIZED_STATE_SIZE (sizeof(Peer) + 32 + (sizeof(Path) * 2))
@@ -305,12 +304,13 @@ public:
 	 */
 	inline unsigned int latency(const int64_t now)
 	{
-		if (_canUseMultipath) {
+		if (_localMultipathSupported) {
 			return (int)_lastComputedAggregateMeanLatency;
 		} else {
 			SharedPtr<Path> bp(getAppropriatePath(now,false));
-			if (bp)
+			if (bp) {
 				return bp->latency();
+			}
 			return 0xffff;
 		}
 	}
@@ -419,35 +419,15 @@ public:
 	}
 
 	/**
-	 * Rate limit gate for inbound ECHO requests. This rate limiter works
-	 * by draining a certain number of requests per unit time. Each peer may
-	 * theoretically receive up to ZT_ECHO_CUTOFF_LIMIT requests per second.
+	 * Rate limit gate for inbound ECHO requests
 	 */
 	inline bool rateGateEchoRequest(const int64_t now)
 	{
-		/*
-		// TODO: Rethink this
-		if (_canUseMultipath) {
-			_echoRequestCutoffCount++;
-			int numToDrain = (now - _lastEchoCheck) / ZT_ECHO_DRAINAGE_DIVISOR;
-			_lastEchoCheck = now;
-			fprintf(stderr, "ZT_ECHO_CUTOFF_LIMIT=%d, (now - _lastEchoCheck)=%d, numToDrain=%d, ZT_ECHO_DRAINAGE_DIVISOR=%d\n", ZT_ECHO_CUTOFF_LIMIT, (now - _lastEchoCheck), numToDrain, ZT_ECHO_DRAINAGE_DIVISOR);
-			if (_echoRequestCutoffCount > numToDrain) {
-				_echoRequestCutoffCount-=numToDrain;
-			}
-			else {
-				_echoRequestCutoffCount = 0;
-			}
-			return (_echoRequestCutoffCount < ZT_ECHO_CUTOFF_LIMIT);
-		} else {
-			if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT)) {
-				_lastEchoRequestReceived = now;
-				return true;
-			}
-			return false;
+		if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) {
+			_lastEchoRequestReceived = now;
+			return true;
 		}
-		*/
-		return true;
+		return false;
 	}
 
 	/**
@@ -523,16 +503,20 @@ public:
 	}
 
 	/**
-	 *
-	 * @return
+	 * @return The bonding policy used to reach this peer
 	 */
-	SharedPtr<Bond> bond() { return _bondToPeer; }
+	SharedPtr<Bond> bond() { return _bond; }
 
 	/**
-	 *
-	 * @return
+	 * @return The bonding policy used to reach this peer
 	 */
-	inline int8_t bondingPolicy() { return _bondingPolicy; }
+	inline int8_t bondingPolicy() {
+		Mutex::Lock _l(_paths_m);
+		if (_bond) {
+			return _bond->policy();
+		}
+		return ZT_BOND_POLICY_NONE;
+	}
 
 	//inline const AES *aesKeysIfSupported() const
 	//{ return (const AES *)0; }
@@ -582,6 +566,7 @@ private:
 
 	_PeerPath _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	Mutex _paths_m;
+	Mutex _bond_m;
 
 	Identity _id;
 
@@ -591,18 +576,13 @@ private:
 
 	AtomicCounter __refCount;
 
-	bool _remotePeerMultipathEnabled;
-	int _uniqueAlivePathCount;
 	bool _localMultipathSupported;
-	bool _remoteMultipathSupported;
-	bool _canUseMultipath;
 
 	volatile bool _shouldCollectPathStatistics;
-	volatile int8_t _bondingPolicy;
 
 	int32_t _lastComputedAggregateMeanLatency;
 
-	SharedPtr<Bond> _bondToPeer;
+	SharedPtr<Bond> _bond;
 };
 
 } // namespace ZeroTier

node/RuntimeEnvironment.hpp

@@ -30,7 +30,7 @@ class Multicaster;
 class NetworkController;
 class SelfAwareness;
 class Trace;
-class BondController;
+class Bond;
 
 /**
  * Holds global state for an instance of ZeroTier::Node
@@ -76,7 +76,7 @@ public:
 	Multicaster *mc;
 	Topology *topology;
 	SelfAwareness *sa;
-	BondController *bc;
+	Bond *bc;
 
 	// This node's identity and string representations thereof
 	Identity identity;

node/Switch.cpp

@@ -1003,14 +1003,12 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
 
 	const SharedPtr<Peer> peer(RR->topology->getPeer(tPtr,destination));
 	if (peer) {
-		if ((peer->bondingPolicy() == ZT_BONDING_POLICY_BROADCAST)
+		if ((peer->bondingPolicy() == ZT_BOND_POLICY_BROADCAST)
 			&& (packet.verb() == Packet::VERB_FRAME || packet.verb() == Packet::VERB_EXT_FRAME)) {
 			const SharedPtr<Peer> relay(RR->topology->getUpstreamPeer());
 			Mutex::Lock _l(peer->_paths_m);
 			for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
 				if (peer->_paths[i].p && peer->_paths[i].p->alive(now)) {
-					char pathStr[128];
-					peer->_paths[i].p->address().toString(pathStr);
 					_sendViaSpecificPath(tPtr,peer,peer->_paths[i].p,now,packet,encrypt,flowId);
 				}
 			}
@@ -1047,7 +1045,6 @@ void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path
 	if (trustedPathId) {
 		packet.setTrusted(trustedPathId);
 	} else {
-		Packet::Verb v = packet.verb();
 		packet.armor(peer->key(),encrypt,peer->aesKeysIfSupported());
 		RR->node->expectReplyTo(packet.packetId());
 	}

node/Trace.cpp

@@ -69,7 +69,7 @@ void Trace::peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,
 	char tmp[128];
 	if (!path) return; // sanity check
 
-	ZT_LOCAL_TRACE(tPtr,RR,"trying unknown path %s to %.10llx (packet %.16llx verb %d local socket %lld network %.16llx)",path->address().toString(tmp),peer.address().toInt(),packetId,(double)verb,path->localSocket(),networkId);
+	ZT_LOCAL_TRACE(tPtr,RR,"trying unknown path %s to %.10llx (packet %.16llx verb %d local socket %lld network %.16llx)",path->address().toString(tmp),peer.address().toInt(),packetId,verb,path->localSocket(),networkId);
 
 	std::pair<Address,Trace::Level> byn;
 	if (networkId) { Mutex::Lock l(_byNet_m); _byNet.get(networkId,byn); }

node/Utils.cpp

@@ -28,10 +28,12 @@
 #include <sys/stat.h>
 #include <sys/uio.h>
 #include <dirent.h>
+#ifdef ZT_ARCH_ARM_HAS_NEON
 #ifdef __LINUX__
 #include <sys/auxv.h>
 #endif
 #endif
+#endif
 
 #ifdef __WINDOWS__
 #include <wincrypt.h>

objects.mk

@@ -28,8 +28,7 @@ CORE_OBJS=\
 	node/Topology.o \
 	node/Trace.o \
 	node/Utils.o \
-	node/Bond.o \
-	node/BondController.o
+	node/Bond.o
 
 ONE_OBJS=\
 	controller/EmbeddedNetworkController.o \

one.cpp

@@ -84,7 +84,7 @@
 #include "osdep/Http.hpp"
 #include "osdep/Thread.hpp"
 
-#include "node/BondController.hpp"
+#include "node/Bond.hpp"
 
 #include "service/OneService.hpp"
 
@@ -393,7 +393,9 @@ static int cli(int argc,char **argv)
 									char tmp[256];
 									std::string addr = path["address"];
 									const int64_t now = OSUtils::now();
-									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%lld;%lld",addr.c_str(),now - (int64_t)path["lastSend"],now - (int64_t)path["lastReceive"]);
+									int64_t lastSendDiff = (uint64_t)path["lastSend"] ? now - (uint64_t)path["lastSend"] : -1;
+									int64_t lastReceiveDiff = (uint64_t)path["lastReceive"] ? now - (uint64_t)path["lastReceive"] : -1;
+									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%lld;%lld",addr.c_str(),lastSendDiff,lastReceiveDiff);
 									bestPath = tmp;
 									break;
 								}
@@ -460,7 +462,9 @@ static int cli(int argc,char **argv)
 									char tmp[256];
 									std::string addr = path["address"];
 									const int64_t now = OSUtils::now();
-									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%-8lld %-8lld %s",now - (int64_t)path["lastSend"],now - (int64_t)path["lastReceive"],addr.c_str());
+									int64_t lastSendDiff = (uint64_t)path["lastSend"] ? now - (uint64_t)path["lastSend"] : -1;
+									int64_t lastReceiveDiff = (uint64_t)path["lastReceive"] ? now - (uint64_t)path["lastReceive"] : -1;
+									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%-8lld %-8lld %s",lastSendDiff,lastReceiveDiff,addr.c_str());
 									bestPath = std::string("DIRECT ") + tmp;
 									break;
 								}
@@ -492,14 +496,13 @@ static int cli(int argc,char **argv)
 			return 1;
 		}
 	} else if (command == "bond") {
-		/* zerotier-cli bond */
+		/* zerotier-cli bond <cmd> */
 		if (arg1.empty()) {
-			printf("(bond) command is missing required arugments" ZT_EOL_S);
+			printf("(bond) command is missing required arguments" ZT_EOL_S);
 			return 2;
 		}
 		/* zerotier-cli bond list */
 		if (arg1 == "list") {
-			fprintf(stderr, "zerotier-cli bond list\n");
 			const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/bonds",requestHeaders,responseHeaders,responseBody);
 			if (scode == 0) {
 				printf("Error connecting to the ZeroTier service: %s\n\nPlease check that the service is running and that TCP port 9993 can be contacted via 127.0.0.1." ZT_EOL_S, responseBody.c_str());
@@ -525,11 +528,11 @@ static int cli(int argc,char **argv)
 						for(unsigned long k=0;k<j.size();++k) {
 							nlohmann::json &p = j[k];
 							bool isBonded = p["isBonded"];
-							int8_t bondingPolicy = p["bondingPolicy"];
-							bool isHealthy = p["isHealthy"];
-							int8_t numAliveLinks = p["numAliveLinks"];
-							int8_t numTotalLinks = p["numTotalLinks"];
 							if (isBonded) {
+								int8_t bondingPolicy = p["bondingPolicy"];
+								bool isHealthy = p["isHealthy"];
+								int8_t numAliveLinks = p["numAliveLinks"];
+								int8_t numTotalLinks = p["numTotalLinks"];
 								bFoundBond = true;
 								std::string healthStr;
 								if (isHealthy) {
@@ -538,8 +541,8 @@ static int cli(int argc,char **argv)
 									healthStr = "DEGRADED";
 								}
 								std::string policyStr = "none";
-								if (bondingPolicy >= ZT_BONDING_POLICY_NONE && bondingPolicy <= ZT_BONDING_POLICY_BALANCE_AWARE) {
-									policyStr = BondController::getPolicyStrByCode(bondingPolicy);
+								if (bondingPolicy >= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) {
+									policyStr = Bond::getPolicyStrByCode(bondingPolicy);
 								}
 								printf("%10s  %32s    %8s        %d/%d" ZT_EOL_S,
 									OSUtils::jsonString(p ["address"],"-").c_str(),
@@ -560,11 +563,7 @@ static int cli(int argc,char **argv)
 				return 1;
 			}
 		}
-		else if (arg1.length() == 10) { /* zerotier-cli bond <peerId> enable */
-			if (arg2 == "enable") {
-				fprintf(stderr, "zerotier-cli bond <peerId> enable\n");
-				return 0;
-			}
+		else if (arg1.length() == 10) {
 			if (arg2 == "rotate") { /* zerotier-cli bond <peerId> rotate */
 				fprintf(stderr, "zerotier-cli bond <peerId> rotate\n");
 				requestHeaders["Content-Type"] = "application/json";
@@ -618,7 +617,6 @@ static int cli(int argc,char **argv)
 					if (json) {
 						printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str());
 					} else {
-						bool bFoundBond = false;
 						std::string healthStr;
 						if (OSUtils::jsonInt(j["isHealthy"],0)) {
 							healthStr = "Healthy";
@@ -629,15 +627,15 @@ static int cli(int argc,char **argv)
 						int numTotalLinks = OSUtils::jsonInt(j["numTotalLinks"],0);
 						printf("Peer               : %s\n", arg1.c_str());
 						printf("Bond               : %s\n", OSUtils::jsonString(j["bondingPolicy"],"-").c_str());
-						//if (bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) {
-						printf("Link Select Method : %d\n", OSUtils::jsonInt(j["linkSelectMethod"],0));
+						//if (bondingPolicy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
+						printf("Link Select Method : %d\n", (int)OSUtils::jsonInt(j["linkSelectMethod"],0));
 						//}
 						printf("Status             : %s\n", healthStr.c_str());
 						printf("Links              : %d/%d\n", numAliveLinks, numTotalLinks);
-						printf("Failover Interval  : %d (ms)\n", OSUtils::jsonInt(j["failoverInterval"],0));
-						printf("Up Delay           : %d (ms)\n", OSUtils::jsonInt(j["upDelay"],0));
-						printf("Down Delay         : %d (ms)\n", OSUtils::jsonInt(j["downDelay"],0));
-						printf("Packets Per Link   : %d (ms)\n", OSUtils::jsonInt(j["packetsPerLink"],0));
+						printf("Failover Interval  : %d (ms)\n", (int)OSUtils::jsonInt(j["failoverInterval"],0));
+						printf("Up Delay           : %d (ms)\n", (int)OSUtils::jsonInt(j["upDelay"],0));
+						printf("Down Delay         : %d (ms)\n", (int)OSUtils::jsonInt(j["downDelay"],0));
+						printf("Packets Per Link   : %d (ms)\n", (int)OSUtils::jsonInt(j["packetsPerLink"],0));
 						nlohmann::json &p = j["links"];
 						if (p.is_array()) {
 							printf("\n     Interface Name\t\t\t\t\t     Path\t Alive\n");
@@ -649,7 +647,7 @@ static int cli(int argc,char **argv)
 									i,
 									OSUtils::jsonString(p[i]["ifname"],"-").c_str(),
 									OSUtils::jsonString(p[i]["path"],"-").c_str(),
-									OSUtils::jsonInt(p[i]["alive"],0));
+									(int)OSUtils::jsonInt(p[i]["alive"],0));
 							}
 							printf("\n        Latency     Jitter     Loss     Error        Speed   Alloc\n");
 							for(int i=0; i<80; i++) { printf("-"); }
@@ -662,8 +660,8 @@ static int cli(int argc,char **argv)
 									OSUtils::jsonDouble(p[i]["latencyVariance"], 0),
 									OSUtils::jsonDouble(p[i]["packetLossRatio"], 0),
 									OSUtils::jsonDouble(p[i]["packetErrorRatio"], 0),
-									OSUtils::jsonInt(p[i]["givenLinkSpeed"], 0),
-									OSUtils::jsonInt(p[i]["allocation"], 0));
+									(int)OSUtils::jsonInt(p[i]["givenLinkSpeed"], 0),
+									(int)OSUtils::jsonInt(p[i]["allocation"], 0));
 							}
 						}
 					}
@@ -676,7 +674,7 @@ static int cli(int argc,char **argv)
 			}
 		}
 		/* zerotier-cli bond command was malformed in some way */
-		printf("(bond) command is missing required arugments" ZT_EOL_S);
+		printf("(bond) command is missing required arguments" ZT_EOL_S);
 		return 2;
 		const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/bonds",requestHeaders,responseHeaders,responseBody);
 		if (scode == 0) {
@@ -711,14 +709,13 @@ static int cli(int argc,char **argv)
 				if (j.is_array()) {
 					for(unsigned long k=0;k<j.size();++k) {
 						nlohmann::json &p = j[k];
-
 						bool isBonded = p["isBonded"];
-						int8_t bondingPolicy = p["bondingPolicy"];
-						bool isHealthy = p["isHealthy"];
-						int8_t numAliveLinks = p["numAliveLinks"];
-						int8_t numTotalLinks = p["numTotalLinks"];
-
 						if (isBonded) {
+							int8_t bondingPolicy = p["bondingPolicy"];
+							bool isHealthy = p["isHealthy"];
+							int8_t numAliveLinks = p["numAliveLinks"];
+							int8_t numTotalLinks = p["numTotalLinks"];
+
 							bFoundBond = true;
 							std::string healthStr;
 							if (isHealthy) {
@@ -727,8 +724,8 @@ static int cli(int argc,char **argv)
 								healthStr = "Degraded";
 							}
 							std::string policyStr = "none";
-							if (bondingPolicy >= ZT_BONDING_POLICY_NONE && bondingPolicy <= ZT_BONDING_POLICY_BALANCE_AWARE) {
-								policyStr = BondController::getPolicyStrByCode(bondingPolicy);
+							if (bondingPolicy >= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) {
+								policyStr = Bond::getPolicyStrByCode(bondingPolicy);
 							}
 
 							printf("%10s  %32s    %8s        %d/%d" ZT_EOL_S,
@@ -789,14 +786,23 @@ static int cli(int argc,char **argv)
 								}
 							}
 							if (aa.length() == 0) aa = "-";
+							const std::string status = OSUtils::jsonString(n["status"],"-");
 							printf("200 listnetworks %s %s %s %s %s %s %s" ZT_EOL_S,
 								OSUtils::jsonString(n["nwid"],"-").c_str(),
 								OSUtils::jsonString(n["name"],"-").c_str(),
 								OSUtils::jsonString(n["mac"],"-").c_str(),
-								OSUtils::jsonString(n["status"],"-").c_str(),
+								status.c_str(),
 								OSUtils::jsonString(n["type"],"-").c_str(),
 								OSUtils::jsonString(n["portDeviceName"],"-").c_str(),
 								aa.c_str());
+							if (OSUtils::jsonBool(n["ssoEnabled"], false)) {
+								uint64_t authenticationExpiryTime = n["authenticationExpiryTime"];
+								if (status == "AUTHENTICATION_REQUIRED") {
+									printf("    AUTH EXPIRED, URL: %s" ZT_EOL_S, OSUtils::jsonString(n["authenticationURL"], "(null)").c_str());
+								} else if (status == "OK") {
+									printf("    AUTH OK, expires in: %lld seconds" ZT_EOL_S, ((int64_t)authenticationExpiryTime - OSUtils::now()) / 1000LL);
+								}
+							}
 						}
 					}
 				}
@@ -1310,7 +1316,6 @@ static int cli(int argc,char **argv)
 		struct ifconf ifc;
 		char buf[1024];
 		char stringBuffer[128];
-		int success = 0;
 
 		int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
 

selftest.cpp

@@ -561,8 +561,8 @@ static int testCertificate()
 	std::cout << idA.address().toString(buf) << ", " << idB.address().toString(buf) << std::endl;
 
 	std::cout << "[certificate] Generating certificates A and B...";
-	CertificateOfMembership cA(10000,100,1,idA.address());
-	CertificateOfMembership cB(10099,100,1,idB.address());
+	CertificateOfMembership cA(10000,100,1,idA);
+	CertificateOfMembership cB(10099,100,1,idB);
 	std::cout << std::endl;
 
 	std::cout << "[certificate] Signing certificates A and B with authority...";
@@ -574,13 +574,13 @@ static int testCertificate()
 	//std::cout << "[certificate] B: " << cB.toString() << std::endl;
 
 	std::cout << "[certificate] A agrees with B and B with A... ";
-	if (cA.agreesWith(cB))
+	if (cA.agreesWith(cB, idB))
 		std::cout << "yes, ";
 	else {
 		std::cout << "FAIL" << std::endl;
 		return -1;
 	}
-	if (cB.agreesWith(cA))
+	if (cB.agreesWith(cA, idA))
 		std::cout << "yes." << std::endl;
 	else {
 		std::cout << "FAIL" << std::endl;
@@ -588,18 +588,18 @@ static int testCertificate()
 	}
 
 	std::cout << "[certificate] Generating two certificates that should not agree...";
-	cA = CertificateOfMembership(10000,100,1,idA.address());
-	cB = CertificateOfMembership(10101,100,1,idB.address());
+	cA = CertificateOfMembership(10000,100,1,idA);
+	cB = CertificateOfMembership(10101,100,1,idB);
 	std::cout << std::endl;
 
 	std::cout << "[certificate] A agrees with B and B with A... ";
-	if (!cA.agreesWith(cB))
+	if (!cA.agreesWith(cB, idB))
 		std::cout << "no, ";
 	else {
 		std::cout << "FAIL" << std::endl;
 		return -1;
 	}
-	if (!cB.agreesWith(cA))
+	if (!cB.agreesWith(cA, idA))
 		std::cout << "no." << std::endl;
 	else {
 		std::cout << "FAIL" << std::endl;

service/OneService.cpp

@@ -49,7 +49,6 @@
 #include "../osdep/Binder.hpp"
 #include "../osdep/ManagedRoute.hpp"
 #include "../osdep/BlockingQueue.hpp"
-#include "../osdep/Link.hpp"
 
 #include "OneService.hpp"
 #include "SoftwareUpdater.hpp"
@@ -184,6 +183,7 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc,
 		case ZT_NETWORK_STATUS_NOT_FOUND:                nstatus = "NOT_FOUND"; break;
 		case ZT_NETWORK_STATUS_PORT_ERROR:               nstatus = "PORT_ERROR"; break;
 		case ZT_NETWORK_STATUS_CLIENT_TOO_OLD:           nstatus = "CLIENT_TOO_OLD"; break;
+		case ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED:  nstatus = "AUTHENTICATION_REQUIRED"; break;
 	}
 	switch(nc->type) {
 		case ZT_NETWORK_TYPE_PRIVATE:                    ntype = "PRIVATE"; break;
@@ -251,6 +251,9 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc,
 	}
 	nj["dns"] = m;
 
+	nj["authenticationURL"] = nc->authenticationURL;
+	nj["authenticationExpiryTime"] = nc->authenticationExpiryTime;
+	nj["ssoEnabled"] = nc->ssoEnabled;
 }
 
 static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer)
@@ -300,12 +303,11 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer)
 
 static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 {
-	char tmp[256];
 	uint64_t now = OSUtils::now();
 
-	int bondingPolicy = bond->getPolicy();
-	pj["bondingPolicy"] = BondController::getPolicyStrByCode(bondingPolicy);
-	if (bondingPolicy == ZT_BONDING_POLICY_NONE) {
+	int bondingPolicy = bond->policy();
+	pj["bondingPolicy"] = Bond::getPolicyStrByCode(bondingPolicy);
+	if (bondingPolicy == ZT_BOND_POLICY_NONE) {
 		return;
 	}
 
@@ -315,15 +317,15 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 	pj["failoverInterval"] = bond->getFailoverInterval();
 	pj["downDelay"] = bond->getDownDelay();
 	pj["upDelay"] = bond->getUpDelay();
-	if (bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) {
+	if (bondingPolicy == ZT_BOND_POLICY_BALANCE_RR) {
 		pj["packetsPerLink"] = bond->getPacketsPerLink();
 	}
-	if (bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) {
+	if (bondingPolicy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
 		pj["linkSelectMethod"] = bond->getLinkSelectMethod();
 	}
 
 	nlohmann::json pa = nlohmann::json::array();
-	std::vector< SharedPtr<Path> > paths = bond->getPeer()->paths(now);
+	std::vector< SharedPtr<Path> > paths = bond->paths(now);
 
 	for(unsigned int i=0;i<paths.size();++i) {
 		char pathStr[128];
@@ -332,6 +334,7 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 		nlohmann::json j;
 		j["ifname"] = bond->getLink(paths[i])->ifname();
 		j["path"] = pathStr;
+		/*
 		j["alive"] = paths[i]->alive(now,true);
 		j["bonded"] = paths[i]->bonded();
 		j["latencyMean"] = paths[i]->latencyMean();
@@ -340,6 +343,7 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 		j["packetErrorRatio"] = paths[i]->packetErrorRatio();
 		j["givenLinkSpeed"] = 1000;
 		j["allocation"] = paths[i]->allocation();
+		*/
 		pa.push_back(j);
 	}
 	pj["links"] = pa;
@@ -535,6 +539,12 @@ public:
 			memset(&config, 0, sizeof(ZT_VirtualNetworkConfig));
 		}
 
+		~NetworkState()
+		{
+			this->managedRoutes.clear();
+			this->tap.reset();
+		}
+
 		std::shared_ptr<EthernetTap> tap;
 		ZT_VirtualNetworkConfig config; // memcpy() of raw config from core
 		std::vector<InetAddress> managedIps;
@@ -573,6 +583,7 @@ public:
 	Mutex _run_m;
 
 	RedisConfig *_rc;
+	std::string _ssoRedirectURL;
 
 	// end member variables ----------------------------------------------------
 
@@ -610,6 +621,7 @@ public:
 #endif
 		,_run(true)
 		,_rc(NULL)
+		,_ssoRedirectURL()
 	{
 		_ports[0] = 0;
 		_ports[1] = 0;
@@ -723,25 +735,22 @@ public:
 			OSUtils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]);
 			OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr));
 
-			// Attempt to bind to a secondary port chosen from our ZeroTier address.
+			// Attempt to bind to a secondary port.
 			// This exists because there are buggy NATs out there that fail if more
 			// than one device behind the same NAT tries to use the same internal
 			// private address port number. Buggy NATs are a running theme.
+			//
+			// This used to pick the secondary port based on the node ID until we
+			// discovered another problem: buggy routers and malicious traffic 
+			// "detection".  A lot of routers have such things built in these days
+			// and mis-detect ZeroTier traffic as malicious and block it resulting
+			// in a node that appears to be in a coma.  Secondary ports are now 
+			// randomized on startup.
 			if (_allowSecondaryPort) {
 				if (_secondaryPort) {
 					_ports[1] = _secondaryPort;
 				} else {
-					_ports[1] = 20000 + ((unsigned int)_node->address() % 45500);
-					for(int i=0;;++i) {
-						if (i > 1000) {
-							_ports[1] = 0;
-							break;
-						} else if (++_ports[1] >= 65536) {
-							_ports[1] = 20000;
-						}
-						if (_trialBind(_ports[1]))
-							break;
-					}
+					_ports[1] = _getRandomPort();
 				}
 			}
 #ifdef ZT_USE_MINIUPNPC
@@ -753,7 +762,7 @@ public:
 					if (_tertiaryPort) {
 						_ports[2] = _tertiaryPort;
 					} else {
-						_ports[2] = _ports[1];
+						_ports[2] = 20000 + (_ports[0] % 40000);
 						for(int i=0;;++i) {
 							if (i > 1000) {
 								_ports[2] = 0;
@@ -779,6 +788,9 @@ public:
 
 			// Network controller is now enabled by default for desktop and server
 			_controller = new EmbeddedNetworkController(_node,_homePath.c_str(),_controllerDbPath.c_str(),_ports[0], _rc);
+			if (!_ssoRedirectURL.empty()) {
+				_controller->setSSORedirectURL(_ssoRedirectURL);
+			}
 			_node->setNetconfMaster((void *)_controller);
 
 			// Join existing networks in networks.d
@@ -811,6 +823,7 @@ public:
 			int64_t lastCleanedPeersDb = 0;
 			int64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle
 			int64_t lastLocalConfFileCheck = OSUtils::now();
+			int64_t lastOnline = lastLocalConfFileCheck;
 			for(;;) {
 				_run_m.lock();
 				if (!_run) {
@@ -852,6 +865,16 @@ public:
 					}
 				}
 
+				// If secondary port is not configured to a constant value and we've been offline for a while,
+				// bind a new secondary port. This is a workaround for a "coma" issue caused by buggy NATs that stop
+				// working on one port after a while.
+				if (_node->online()) {
+					lastOnline = now;
+				} else if ((_secondaryPort == 0)&&((now - lastOnline) > ZT_PATH_HEARTBEAT_PERIOD)) {
+					_secondaryPort = _getRandomPort();
+					lastBindRefresh = 0;
+				}
+
 				// Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow routes (e.g. shadow default)
 				if (((now - lastBindRefresh) >= (_node->bondController()->inUse() ? ZT_BINDER_REFRESH_PERIOD / 4 : ZT_BINDER_REFRESH_PERIOD))||(restarted)) {
 					lastBindRefresh = now;
@@ -1012,8 +1035,11 @@ public:
 			}
 		}
 
+		// Make a copy so lookups don't modify in place;
+		json lc(_localConfig);
+
 		// Get any trusted paths in local.conf (we'll parse the rest of physical[] elsewhere)
-		json &physical = _localConfig["physical"];
+		json &physical = lc["physical"];
 		if (physical.is_object()) {
 			for(json::iterator phy(physical.begin());phy!=physical.end();++phy) {
 				InetAddress net(OSUtils::jsonString(phy.key(),"").c_str());
@@ -1030,20 +1056,22 @@ public:
 			}
 		}
 
-		json &settings = _localConfig["settings"];
+		json &settings = lc["settings"];
 		if (settings.is_object()) {
 			// Allow controller DB path to be put somewhere else
 			const std::string cdbp(OSUtils::jsonString(settings["controllerDbPath"],""));
 			if (cdbp.length() > 0)
 				_controllerDbPath = cdbp;
 
+			_ssoRedirectURL = OSUtils::jsonString(settings["ssoRedirectURL"], "");
+
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 			// TODO:  Redis config
 			json &redis = settings["redis"];
 			if (redis.is_object() && _rc == NULL) {
 				_rc = new RedisConfig;
 				_rc->hostname = OSUtils::jsonString(redis["hostname"],"");
-				_rc->port = redis["port"];
+				_rc->port = OSUtils::jsonInt(redis["port"],0);
 				_rc->password = OSUtils::jsonString(redis["password"],"");
 				_rc->clusterMode = OSUtils::jsonBool(redis["clusterMode"], false);
 			}
@@ -1259,7 +1287,7 @@ public:
 										_bondToJson(res,bond);
 										scode = 200;
 									} else {
-										fprintf(stderr, "unable to find bond to peer %llx\n", id);
+										fprintf(stderr, "unable to find bond to peer %llx\n", (unsigned long long)id);
 										scode = 400;
 									}
 								}
@@ -1271,8 +1299,11 @@ public:
 					} else {
 						scode = 400; /* bond controller is not enabled */
 					}
-				}
-				if (ps[0] == "status") {
+				} else if (ps[0] == "config") {
+					Mutex::Lock lc(_localConfig_m);
+					res = _localConfig;
+					scode = 200;
+				} else if (ps[0] == "status") {
 					ZT_NodeStatus status;
 					_node->status(&status);
 
@@ -1466,7 +1497,7 @@ public:
 									if (bond) {
 										scode = bond->abForciblyRotateLink() ? 200 : 400;
 									} else {
-										fprintf(stderr, "unable to find bond to peer %llx\n", id);
+										fprintf(stderr, "unable to find bond to peer %llx\n", (unsigned long long)id);
 										scode = 400;
 									}
 								}
@@ -1478,8 +1509,35 @@ public:
 					} else {
 						scode = 400; /* bond controller is not enabled */
 					}
-				}
-				if (ps[0] == "moon") {
+				} else if (ps[0] == "config") {
+					// Right now we only support writing the things the UI supports changing.
+					if (ps.size() == 2) {
+						if (ps[1] == "settings") {
+							try {
+								json j(OSUtils::jsonParse(body));
+								if (j.is_object()) {
+									Mutex::Lock lcl(_localConfig_m);
+									json lc(_localConfig);
+									for(json::const_iterator s(j.begin());s!=j.end();++s) {
+										lc["settings"][s.key()] = s.value();
+									}
+									std::string lcStr = OSUtils::jsonDump(lc, 4);
+									if (OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(), lcStr)) {
+										_localConfig = lc;
+									}
+								} else {
+									scode = 400;
+								}
+							} catch ( ... ) {
+								scode = 400;
+							}
+						} else {
+							scode = 404;
+						}
+					} else {
+						scode = 404;
+					}
+				} else if (ps[0] == "moon") {
 					if (ps.size() == 2) {
 
 						uint64_t seed = 0;
@@ -1711,11 +1769,11 @@ public:
 				if (basePolicyStr.empty()) {
 					fprintf(stderr, "error: no base policy was specified for custom policy (%s)\n", customPolicyStr.c_str());
 				}
-				if (_node->bondController()->getPolicyCodeByStr(basePolicyStr) == ZT_BONDING_POLICY_NONE) {
+				if (_node->bondController()->getPolicyCodeByStr(basePolicyStr) == ZT_BOND_POLICY_NONE) {
 					fprintf(stderr, "error: custom policy (%s) is invalid, unknown base policy (%s).\n",
 						customPolicyStr.c_str(), basePolicyStr.c_str());
 					continue;
-				} if (_node->bondController()->getPolicyCodeByStr(customPolicyStr) != ZT_BONDING_POLICY_NONE) {
+				} if (_node->bondController()->getPolicyCodeByStr(customPolicyStr) != ZT_BOND_POLICY_NONE) {
 					fprintf(stderr, "error: custom policy (%s) will be ignored, cannot use standard policy names for custom policies.\n",
 						customPolicyStr.c_str());
 					continue;
@@ -1744,20 +1802,12 @@ public:
 					newTemplateBond->setUserQualityWeights(weights,ZT_QOS_WEIGHT_SIZE);
 				}
 				// Bond-specific properties
-				newTemplateBond->setOverflowMode(OSUtils::jsonInt(customPolicy["overflow"],false));
 				newTemplateBond->setUpDelay(OSUtils::jsonInt(customPolicy["upDelay"],-1));
 				newTemplateBond->setDownDelay(OSUtils::jsonInt(customPolicy["downDelay"],-1));
 				newTemplateBond->setFlowRebalanceStrategy(OSUtils::jsonInt(customPolicy["flowRebalanceStrategy"],(uint64_t)0));
 				newTemplateBond->setFailoverInterval(OSUtils::jsonInt(customPolicy["failoverInterval"],(uint64_t)0));
 				newTemplateBond->setPacketsPerLink(OSUtils::jsonInt(customPolicy["packetsPerLink"],-1));
 
-				std::string linkMonitorStrategyStr(OSUtils::jsonString(customPolicy["linkMonitorStrategy"],""));
-				uint8_t linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DEFAULT;
-				if (linkMonitorStrategyStr == "passive") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE; }
-				if (linkMonitorStrategyStr == "active") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_ACTIVE; }
-				if (linkMonitorStrategyStr == "dynamic") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC; }
-				newTemplateBond->setLinkMonitorStrategy(linkMonitorStrategy);
-
 				// Policy-Specific link set
 				json &links = customPolicy["links"];
 				for (json::iterator linkItr = links.begin(); linkItr != links.end();++linkItr) {
@@ -1773,40 +1823,40 @@ public:
 							speed, alloc, linkNameStr.c_str());
 						enabled = false;
 					}
-					uint32_t upDelay = OSUtils::jsonInt(link["upDelay"],-1);
-					uint32_t downDelay = OSUtils::jsonInt(link["downDelay"],-1);
+					//uint32_t upDelay = OSUtils::jsonInt(link["upDelay"],-1);
+					//uint32_t downDelay = OSUtils::jsonInt(link["downDelay"],-1);
 					uint8_t ipvPref = OSUtils::jsonInt(link["ipvPref"],0);
-					uint32_t linkMonitorInterval = OSUtils::jsonInt(link["monitorInterval"],(uint64_t)0);
+					//uint32_t linkMonitorInterval = OSUtils::jsonInt(link["monitorInterval"],(uint64_t)0);
 					std::string failoverToStr(OSUtils::jsonString(link["failoverTo"],""));
 					// Mode
 					std::string linkModeStr(OSUtils::jsonString(link["mode"],"spare"));
-					uint8_t linkMode = ZT_MULTIPATH_SLAVE_MODE_SPARE;
-					if (linkModeStr == "primary") { linkMode = ZT_MULTIPATH_SLAVE_MODE_PRIMARY; }
-					if (linkModeStr == "spare") { linkMode = ZT_MULTIPATH_SLAVE_MODE_SPARE; }
+					uint8_t linkMode = ZT_BOND_SLAVE_MODE_SPARE;
+					if (linkModeStr == "primary") { linkMode = ZT_BOND_SLAVE_MODE_PRIMARY; }
+					if (linkModeStr == "spare") { linkMode = ZT_BOND_SLAVE_MODE_SPARE; }
 					// ipvPref
 					if ((ipvPref != 0) && (ipvPref != 4) && (ipvPref != 6) && (ipvPref != 46) && (ipvPref != 64)) {
 						fprintf(stderr, "error: invalid ipvPref value (%d), link disabled.\n", ipvPref);
 						enabled = false;
 					}
-					if (linkMode == ZT_MULTIPATH_SLAVE_MODE_SPARE && failoverToStr.length()) {
+					if (linkMode == ZT_BOND_SLAVE_MODE_SPARE && failoverToStr.length()) {
 						fprintf(stderr, "error: cannot specify failover links for spares, link disabled.\n");
 						failoverToStr = "";
 						enabled = false;
 					}
-					_node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,speed,linkMonitorInterval,upDelay,downDelay,enabled,linkMode,failoverToStr,alloc));
+					_node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,speed,enabled,linkMode,failoverToStr,alloc));
 				}
 				std::string linkSelectMethodStr(OSUtils::jsonString(customPolicy["activeReselect"],"optimize"));
 				if (linkSelectMethodStr == "always") {
-					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS);
+					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_ALWAYS);
 				}
 				if (linkSelectMethodStr == "better") {
-					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_BETTER);
+					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_BETTER);
 				}
 				if (linkSelectMethodStr == "failure") {
-					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_FAILURE);
+					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_FAILURE);
 				}
 				if (linkSelectMethodStr == "optimize") {
-					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE);
+					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_OPTIMIZE);
 				}
 				if (newTemplateBond->getLinkSelectMethod() < 0 || newTemplateBond->getLinkSelectMethod() > 3) {
 					fprintf(stderr, "warning: invalid value (%s) for linkSelectMethod, assuming mode: always\n", linkSelectMethodStr.c_str());
@@ -1839,7 +1889,7 @@ public:
 		_secondaryPort = (unsigned int)OSUtils::jsonInt(settings["secondaryPort"],0);
 		_tertiaryPort = (unsigned int)OSUtils::jsonInt(settings["tertiaryPort"],0);
 		if (_secondaryPort != 0 || _tertiaryPort != 0) {
-			fprintf(stderr,"WARNING: using manually-specified ports. This can cause NAT issues." ZT_EOL_S);
+			fprintf(stderr,"WARNING: using manually-specified secondary and/or tertiary ports. This can cause NAT issues." ZT_EOL_S);
 		}
 		_portMappingEnabled = OSUtils::jsonBool(settings["portMappingEnabled"],true);
 
@@ -2028,7 +2078,7 @@ public:
 				unsigned int mostMatchingPrefixBits = 0;
 				for(std::set<InetAddress>::const_iterator i(myIps.begin());i!=myIps.end();++i) {
 					const unsigned int matchingPrefixBits = i->matchingPrefixBits(*target);
-					if (matchingPrefixBits >= mostMatchingPrefixBits) {
+					if (matchingPrefixBits >= mostMatchingPrefixBits && ((target->isV4() && i->isV4()) || (target->isV6() && i->isV6()))) {
 						mostMatchingPrefixBits = matchingPrefixBits;
 						src = &(*i);
 					}
@@ -2391,7 +2441,7 @@ public:
 							Dictionary<4096> nc;
 							nc.load(nlcbuf.c_str());
 							Buffer<1024> allowManaged;
-							if (nc.get("allowManaged", allowManaged) && !allowManaged.size() == 0) {
+							if (nc.get("allowManaged", allowManaged) && allowManaged.size() > 0) {
 								std::string addresses (allowManaged.begin(), allowManaged.size());
 								if (allowManaged.size() <= 5) { // untidy parsing for backward compatibility
 									if (allowManaged[0] == '1' || allowManaged[0] == 't' || allowManaged[0] == 'T') {
@@ -2625,7 +2675,6 @@ public:
 			case ZT_STATE_OBJECT_NETWORK_CONFIG:
 				OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "networks.d",_homePath.c_str());
 				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.conf",dirname,(unsigned long long)id[0]);
-				secure = true;
 				break;
 			case ZT_STATE_OBJECT_PEER:
 				OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "peers.d",_homePath.c_str());
@@ -3044,9 +3093,6 @@ public:
 				if (!strncmp(p->c_str(),ifname,p->length()))
 					return false;
 			}
-			if (!_node->bondController()->allowedToBind(std::string(ifname))) {
-				return false;
-			}
 		}
 		{
 			// Check global blacklists
@@ -3084,6 +3130,23 @@ public:
 		return true;
 	}
 
+	unsigned int _getRandomPort()
+	{
+		unsigned int randp = 0;
+		Utils::getSecureRandom(&randp,sizeof(randp));
+		randp = 20000 + (randp % 45500);
+		for(int i=0;;++i) {
+			if (i > 1000) {
+				return 0;
+			} else if (++randp >= 65536) {
+				randp = 20000;
+			}
+			if (_trialBind(randp))
+				break;
+		}
+		return randp;
+	}
+
 	bool _trialBind(unsigned int port)
 	{
 		struct sockaddr_in in4;