Rework how paths are set as remote cluster preferred. The code is now clearer and cluster preference indications are now very sticky as they should be.

node/Hashtable.hpp

@@ -362,7 +362,7 @@ private:
 	template<typename O>
 	static inline unsigned long _hc(const O &obj)
 	{
-		return obj.hashCode();
+		return (unsigned long)obj.hashCode();
 	}
 	static inline unsigned long _hc(const uint64_t i)
 	{

node/Peer.cpp

@@ -33,7 +33,6 @@ namespace ZeroTier {
 static uint32_t _natKeepaliveBuf = 0;
 
 Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) :
-	RR(renv),
 	_lastUsed(0),
 	_lastReceive(0),
 	_lastUnicastFrame(0),
@@ -41,6 +40,8 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_lastAnnouncedTo(0),
 	_lastDirectPathPushSent(0),
 	_lastDirectPathPushReceive(0),
+	RR(renv),
+	_remoteClusterOptimal4(0),
 	_vProto(0),
 	_vMajor(0),
 	_vMinor(0),
@@ -50,6 +51,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_latency(0),
 	_directPathPushCutoffCount(0)
 {
+	memset(_remoteClusterOptimal6,0,sizeof(_remoteClusterOptimal6));
 	if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
 		throw std::runtime_error("new peer identity key agreement failed");
 }
@@ -126,7 +128,7 @@ void Peer::received(
 					_paths[p].lastReceive = now;
 					_paths[p].path = path; // local address may have changed!
 #ifdef ZT_ENABLE_CLUSTER
-					_paths[p].clusterWeights = (unsigned int)(!suboptimalPath);
+					_paths[p].localClusterSuboptimal = suboptimalPath;
 #endif
 					pathIsConfirmed = true;
 					break;
@@ -173,11 +175,9 @@ void Peer::received(
 				_paths[slot].lastReceive = now;
 				_paths[slot].path = path;
 #ifdef ZT_ENABLE_CLUSTER
-				_paths[slot].clusterWeights = (unsigned int)(!suboptimalPath);
+				_paths[p].localClusterSuboptimal = suboptimalPath;
 				if (RR->cluster)
 					RR->cluster->broadcastHavePeer(_id);
-#else
-				_paths[slot].clusterWeights = 1;
 #endif
 			} else {
 				TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str());
@@ -216,26 +216,6 @@ bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const
 	return false;
 }
 
-void Peer::setClusterOptimal(const InetAddress &addr)
-{
-	Mutex::Lock _l(_paths_m);
-
-	int opt = -1;
-	for(unsigned int p=0;p<_numPaths;++p) {
-		if (_paths[p].path->address() == addr) {
-			opt = (int)p;
-			break;
-		}
-	}
-
-	if (opt >= 0) { // only change anything if we have the optimal path
-		for(unsigned int p=0;p<_numPaths;++p) {
-			if (_paths[p].path->address().ss_family == addr.ss_family)
-				_paths[p].clusterWeights = ((int)p == opt) ? 2 : 0;
-		}
-	}
-}
-
 bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead)
 {
 	Mutex::Lock _l(_paths_m);
@@ -350,7 +330,9 @@ bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now)
 			if (x != y) {
 				_paths[y].lastReceive = _paths[x].lastReceive;
 				_paths[y].path = _paths[x].path;
-				_paths[y].clusterWeights = _paths[x].clusterWeights;
+#ifdef ZT_ENABLE_CLUSTER
+				_paths[y].localClusterSuboptimal = _paths[x].localClusterSuboptimal;
+#endif
 			}
 			++y;
 		}
@@ -399,7 +381,9 @@ void Peer::clean(uint64_t now)
 			if (y != x) {
 				_paths[y].lastReceive = _paths[x].lastReceive;
 				_paths[y].path = _paths[x].path;
-				_paths[y].clusterWeights = _paths[x].clusterWeights;
+#ifdef ZT_ENABLE_CLUSTER
+				_paths[y].localClusterSuboptimal = _paths[x].localClusterSuboptimal;
+#endif
 			}
 			++y;
 		}

node/Peer.hpp

@@ -123,11 +123,16 @@ public:
 	/**
 	 * Set which known path for an address family is optimal
 	 *
-	 * This only modifies paths within the same address family
-	 *
 	 * @param addr Address to make exclusive
 	 */
-	void setClusterOptimal(const InetAddress &addr);
+	inline void setClusterOptimal(const InetAddress &addr)
+	{
+		if (addr.ss_family == AF_INET) {
+			_remoteClusterOptimal4 = (uint32_t)reinterpret_cast<const struct sockaddr_in *>(&addr)->sin_addr.s_addr;
+		} else if (addr.ss_family == AF_INET6) {
+			memcpy(_remoteClusterOptimal6,reinterpret_cast<const struct sockaddr_in6 *>(&addr)->sin6_addr.s6_addr,16);
+		}
+	}
 
 	/**
 	 * Send via best direct path
@@ -367,14 +372,30 @@ private:
 
 	inline uint64_t _pathScore(const unsigned int p) const
 	{
-		return ( _paths[p].lastReceive +
-		         (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) +
-						 (uint64_t)(_paths[p].clusterWeights * ZT_PEER_PING_PERIOD) );
+		uint64_t s = ZT_PEER_PING_PERIOD;
+		if (_paths[p].path->address().ss_family == AF_INET) {
+			s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + (uint64_t)(ZT_PEER_PING_PERIOD * (unsigned long)(reinterpret_cast<const struct sockaddr_in *>(&(_paths[p].path->address()))->sin_addr.s_addr == _remoteClusterOptimal4));
+		} else if (_paths[p].path->address().ss_family == AF_INET6) {
+			uint64_t clusterWeight = ZT_PEER_PING_PERIOD;
+			const uint8_t *a = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(&(_paths[p].path->address()))->sin6_addr.s6_addr);
+			for(long i=0;i<16;++i) {
+				if (a[i] != _remoteClusterOptimal6[i]) {
+					clusterWeight = 0;
+					break;
+				}
+			}
+			s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + clusterWeight;
+		} else {
+			s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK));
+		}
+#ifdef ZT_ENABLE_CLUSTER
+		s -= ZT_PEER_PING_PERIOD * (uint64_t)_paths[p].localClusterSuboptimal;
+#endif
+		return s;
 	}
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
-
-	const RuntimeEnvironment *RR;
+	uint8_t _remoteClusterOptimal6[16];
 	uint64_t _lastUsed;
 	uint64_t _lastReceive; // direct or indirect
 	uint64_t _lastUnicastFrame;
@@ -382,6 +403,8 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastDirectPathPushSent;
 	uint64_t _lastDirectPathPushReceive;
+	const RuntimeEnvironment *RR;
+	uint32_t _remoteClusterOptimal4;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
@@ -390,7 +413,9 @@ private:
 	struct {
 		uint64_t lastReceive;
 		SharedPtr<Path> path;
-		unsigned int clusterWeights;
+#ifdef ZT_ENABLE_CLUSTER
+		bool localClusterSuboptimal;
+#endif
 	} _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	Mutex _paths_m;
 	unsigned int _numPaths;