Fix several things:

(1) The changes to path learning in the two previous releases were poorly thought out,
and this version should remedy that by introducing PROBE. This is basically a kind of
ECHO request and is used to authenticate endpoints that are not learned via a valid
request/response pair. Thus we will still passively learn endpoints, but securely.

(2) Turns out there was a security oversight in _doHELLO() that could have permitted...
well... I'm not sure it was exploitable to do anything particularly interesting since
a bad identity would be discarded anyway, but fix it just the same.

node/Packet.cpp

@@ -48,6 +48,7 @@ const char *Packet::verbString(Verb v)
 		case VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return "NETWORK_MEMBERSHIP_CERTIFICATE";
 		case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST";
 		case VERB_NETWORK_CONFIG_REFRESH: return "NETWORK_CONFIG_REFRESH";
+		case VERB_PROBE: return "PROBE";
 	}
 	return "(unknown)";
 }

node/Packet.hpp

@@ -225,6 +225,16 @@
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID + 8)
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN + 2)
 
+#define ZT_PROTO_VERB_PROBE_IDX_TIMESTAMP (ZT_PACKET_IDX_PAYLOAD)
+#define ZT_PROTO_VERB_PROBE_LEN_TIMESTAMP 8
+#define ZT_PROTO_VERB_PROBE_IDX_MS_SINCE_LAST_SEND (ZT_PROTO_VERB_PROBE_IDX_TIMESTAMP + ZT_PROTO_VERB_PROBE_LEN_TIMESTAMP)
+#define ZT_PROTO_VERB_PROBE_LEN_MS_SINCE_LAST_SEND 8
+
+#define ZT_PROTO_VERB_PROBE__OK__IDX_TIMESTAMP (ZT_PACKET_IDX_PAYLOAD)
+#define ZT_PROTO_VERB_PROBE__OK__LEN_TIMESTAMP 8
+#define ZT_PROTO_VERB_PROBE__OK__IDX_MS_SINCE_LAST_SEND (ZT_PROTO_VERB_PROBE_IDX_TIMESTAMP + ZT_PROTO_VERB_PROBE_LEN_TIMESTAMP)
+#define ZT_PROTO_VERB_PROBE__OK__LEN_MS_SINCE_LAST_SEND 8
+
 // ---------------------------------------------------------------------------
 
 namespace ZeroTier {
@@ -457,11 +467,7 @@ public:
 		 * send this to both peers at the same time on a periodic basis, telling
 		 * each where it might find the other on the network.
 		 *
-		 * Upon receipt, a peer sends a message such as NOP or HELLO to the other
+		 * Upon receipt a peer sends HELLO to establish a direct link.
-		 * peer. Peers only "learn" one anothers' direct addresses when they
-		 * successfully *receive* a message and authenticate it. Optionally, peers
-		 * will usually preface these messages with one or more firewall openers
-		 * to clear the path.
 		 *
 		 * Nodes should implement rate control, limiting the rate at which they
 		 * respond to these packets to prevent their use in DDOS attacks. Nodes
@@ -615,7 +621,28 @@ public:
 		 * It does not generate an OK or ERROR message, and is treated only as
 		 * a hint to refresh now.
 		 */
-		VERB_NETWORK_CONFIG_REFRESH = 12
+		VERB_NETWORK_CONFIG_REFRESH = 12,
+
+		/* Probe peer connection status:
+		 *    <[8] 64-bit timestamp>
+		 *    <[8] 64-bit milliseconds since last send to this peer>
+		 *
+		 * This message is sent to probe the status of a peer and to confirm
+		 * new link-layer addresses. Upon receipt an OK is generated which
+		 * echoes the time and responds with the number of milliseconds since
+		 * the recipient has last sent a packet to the sender.
+		 *
+		 * Using these delay times, a peer may determine if its current route
+		 * to another peer is likely dead and default to another route (e.g.
+		 * reverting to relaying).
+		 *
+		 * OK response payload:
+		 *    <[8] 64-bit timestamp echoed from request>
+		 *    <[8] 64-bit milliseconds since last send to requesitng peer>
+		 *
+		 * ERROR is not generated.
+		 */
+		VERB_PROBE = 13
 	};
 
 	/**

node/PacketDecoder.cpp

@@ -106,6 +106,8 @@ bool PacketDecoder::tryDecode(const RuntimeEnvironment *_r)
 				return _doNETWORK_CONFIG_REQUEST(_r,peer);
 			case Packet::VERB_NETWORK_CONFIG_REFRESH:
 				return _doNETWORK_CONFIG_REFRESH(_r,peer);
+			case Packet::VERB_PROBE:
+				return _doPROBE(_r,peer);
 			default:
 				// This might be something from a new or old version of the protocol.
 				// Technically it passed MAC so the packet is still valid, but we
@@ -195,16 +197,25 @@ bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
 			if (peer->identity() != id) {
 				unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
 				if (_r->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
-					TRACE("rejected HELLO from %s(%s): address already claimed",source().toString().c_str(),_remoteAddress.toString().c_str());
+					if (dearmor(key)) { // ensure packet is authentic, otherwise drop
-					Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+						TRACE("rejected HELLO from %s(%s): address already claimed",source().toString().c_str(),_remoteAddress.toString().c_str());
-					outp.append((unsigned char)Packet::VERB_HELLO);
+						Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
-					outp.append(packetId());
+						outp.append((unsigned char)Packet::VERB_HELLO);
-					outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
+						outp.append(packetId());
-					outp.armor(key,true);
+						outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
-					_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+						outp.armor(key,true);
+						_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+					} else {
+						LOG("rejected HELLO from %s(%s): packet failed authentication",source().toString().c_str(),_remoteAddress.toString().c_str());
+					}
+				} else {
+					TRACE("rejected HELLO from %s(%s): key agreement failed",source().toString().c_str(),_remoteAddress.toString().c_str());
 				}
 				return true;
-			} // else continue and send OK since we already know thee...
+			} else if (!dearmor(peer->key())) {
+				TRACE("rejected HELLO from %s(%s): packet failed authentication",source().toString().c_str(),_remoteAddress.toString().c_str());
+				return true;
+			} // else continue and respond
 		} else {
 			// If we don't have a peer record on file, check the identity cache (if
 			// we have one) to see if we have a cached identity. Then check that for
@@ -213,20 +224,30 @@ bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
 			if ((alreadyHaveCachedId)&&(id != alreadyHaveCachedId)) {
 				unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
 				if (_r->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
-					TRACE("rejected HELLO from %s(%s): address already claimed",source().toString().c_str(),_remoteAddress.toString().c_str());
+					if (dearmor(key)) { // ensure packet is authentic, otherwise drop
-					Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+						TRACE("rejected HELLO from %s(%s): address already claimed",source().toString().c_str(),_remoteAddress.toString().c_str());
-					outp.append((unsigned char)Packet::VERB_HELLO);
+						Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
-					outp.append(packetId());
+						outp.append((unsigned char)Packet::VERB_HELLO);
-					outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
+						outp.append(packetId());
-					outp.armor(key,true);
+						outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
-					_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+						outp.armor(key,true);
+						_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+					} else {
+						LOG("rejected HELLO from %s(%s): packet failed authentication",source().toString().c_str(),_remoteAddress.toString().c_str());
+					}
+				} else {
+					TRACE("rejected HELLO from %s(%s): key agreement failed",source().toString().c_str(),_remoteAddress.toString().c_str());
 				}
 				return true;
 			} // else continue since identity is already known and matches
 
-			// Learn a new peer if it's new. This also adds it to the identity
+			// If this is a new peer, learn it
-			// cache if that's enabled.
+			SharedPtr<Peer> newPeer(new Peer(_r->identity,id));
-			peer = _r->topology->addPeer(SharedPtr<Peer>(new Peer(_r->identity,id)));
+			if (!dearmor(newPeer->key())) {
+				LOG("rejected HELLO from %s(%s): packet failed authentication",source().toString().c_str(),_remoteAddress.toString().c_str());
+				return true;
+			}
+			peer = _r->topology->addPeer(newPeer);
 		}
 
 		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP,Utils::now());
@@ -908,4 +929,23 @@ bool PacketDecoder::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *_r,const
 	return true;
 }
 
+bool PacketDecoder::_doPROBE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		uint64_t ts = at<uint64_t>(ZT_PROTO_VERB_PROBE_IDX_TIMESTAMP);
+		//uint64_t msSinceLastSend = at<uint64_t>(ZT_PROTO_VERB_PROBE_IDX_MS_SINCE_LAST_SEND);
+		Packet outp(source(),_r->identity.address(),Packet::VERB_OK);
+		outp.append((unsigned char)Packet::VERB_PROBE);
+		outp.append(ts);
+		outp.append(peer->lastDirectSend()); // FIXME: need to refactor to also track relayed sends
+		outp.armor(peer->key(),true);
+		_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+	} catch (std::exception &exc) {
+		TRACE("dropped PROBE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("dropped PROBE from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
 } // namespace ZeroTier

node/PacketDecoder.hpp

@@ -122,6 +122,7 @@ private:
 	bool _doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doPROBE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 
 	uint64_t _receiveTime;
 	Demarc::Port _localPort;

node/Peer.cpp

@@ -40,10 +40,10 @@ Peer::Peer() :
 	_lastUnicastFrame(0),
 	_lastMulticastFrame(0),
 	_lastAnnouncedTo(0),
-	_latency(0),
 	_vMajor(0),
 	_vMinor(0),
 	_vRevision(0),
+	_latency(0),
 	_requestHistoryPtr(0)
 {
 }
@@ -91,7 +91,7 @@ void Peer::onReceive(
 		// Do things like learn latency or endpoints on OK or ERROR replies
 		if (inReVerb != Packet::VERB_NOP) {
 			for(unsigned int p=0;p<ZT_PEER_REQUEST_HISTORY_LENGTH;++p) {
-				if ((_requestHistory[p].packetId == inRePacketId)&&(_requestHistory[p].verb == inReVerb)) {
+				if ((_requestHistory[p].timestamp)&&(_requestHistory[p].packetId == inRePacketId)&&(_requestHistory[p].verb == inReVerb)) {
 					_latency = std::min((unsigned int)(now - _requestHistory[p].timestamp),(unsigned int)0xffff);
 
 					// Only learn paths on replies to packets we have sent, otherwise
@@ -100,11 +100,17 @@ void Peer::onReceive(
 					if (!wp->fixed)
 						wp->addr = remoteAddr;
 
-					_requestHistory[p].packetId = 0;
+					_requestHistory[p].timestamp = 0;
 					break;
 				}
 			}
 		}
+
+		// If we get a valid packet with a different address that is not a response
+		// to a request, send a PROBE to authenticate this endpoint and determine if
+		// it is reachable.
+		if ((!wp->fixed)&&(wp->addr != remoteAddr))
+			_r->sw->sendPROBE(SharedPtr<Peer>(this),localPort,remoteAddr);
 	}
 
 	if (verb == Packet::VERB_FRAME) {

node/Peer.hpp

@@ -49,7 +49,7 @@
 #include "Mutex.hpp"
 
 // Increment if serialization has changed
-#define ZT_PEER_SERIALIZATION_VERSION 5
+#define ZT_PEER_SERIALIZATION_VERSION 6
 
 namespace ZeroTier {
 
@@ -129,7 +129,7 @@ public:
 		uint64_t now);
 
 	/**
-	 * Send a UDP packet to this peer
+	 * Send a UDP packet to this peer directly (not via relaying)
 	 * 
 	 * @param _r Runtime environment
 	 * @param data Data to send
@@ -236,9 +236,19 @@ public:
 	}
 
 	/**
-	 * @return Lowest of measured latencies of all paths or 0 if unknown
+	 * @return Current latency or 0 if unknown (max: 65535)
 	 */
-	inline unsigned int latency() const throw() { return _latency; }
+	inline unsigned int latency() const
+		throw()
+	{
+		uint64_t now = Utils::now();
+		uint64_t latestOutstandingReq = 0;
+		for(unsigned int p=0;p<ZT_PEER_REQUEST_HISTORY_LENGTH;++p)
+			latestOutstandingReq = std::max(latestOutstandingReq,_requestHistory[p].timestamp);
+		if (latestOutstandingReq)
+			return std::min(std::max((unsigned int)(now - latestOutstandingReq),(unsigned int)_latency),(unsigned int)0xffff);
+		else return _latency;
+	}
 
 	/**
 	 * @return True if this peer has at least one direct IP address path
@@ -513,12 +523,12 @@ private:
 	WanPath _ipv4p;
 	WanPath _ipv6p;
 
-	uint64_t _lastUsed;
+	volatile uint64_t _lastUsed;
-	uint64_t _lastUnicastFrame;
+	volatile uint64_t _lastUnicastFrame;
-	uint64_t _lastMulticastFrame;
+	volatile uint64_t _lastMulticastFrame;
-	uint64_t _lastAnnouncedTo;
+	volatile uint64_t _lastAnnouncedTo;
-	unsigned int _latency; // milliseconds, 0 if not known
 	unsigned int _vMajor,_vMinor,_vRevision;
+	volatile unsigned int _latency;
 
 	// not persisted
 	RequestHistoryItem _requestHistory[ZT_PEER_REQUEST_HISTORY_LENGTH];

node/Switch.cpp

@@ -226,6 +226,20 @@ bool Switch::sendHELLO(const SharedPtr<Peer> &dest,Demarc::Port localPort,const
 	} else return false;
 }
 
+bool Switch::sendPROBE(const SharedPtr<Peer> &dest,Demarc::Port localPort,const InetAddress &remoteAddr)
+{
+	uint64_t now = Utils::now();
+	Packet outp(dest->address(),_r->identity.address(),Packet::VERB_PROBE);
+	outp.append(now);
+	outp.append(dest->lastDirectSend()); // FIXME: need to refactor to also track relayed sends
+	outp.armor(dest->key(),true);
+
+	if (_r->demarc->send(localPort,remoteAddr,outp.data(),outp.size(),-1)) {
+		dest->expectResponseTo(outp.packetId(),Packet::VERB_PROBE,localPort,now);
+		return true;
+	} else return false;
+}
+
 bool Switch::unite(const Address &p1,const Address &p2,bool force)
 {
 	if ((p1 == _r->identity.address())||(p2 == _r->identity.address()))

node/Switch.hpp

@@ -129,6 +129,16 @@ public:
 	 */
 	bool sendHELLO(const SharedPtr<Peer> &dest,Demarc::Port localPort,const InetAddress &remoteAddr);
 
+	/**
+	 * Send a PROBE immediately to the indicated address
+	 *
+	 * @param localPort Originating local port or ANY_PORT to pick
+	 * @param remoteAddr IP address to send to
+	 * @param dest Destination peer
+	 * @return True if send appears successful
+	 */
+	bool sendPROBE(const SharedPtr<Peer> &dest,Demarc::Port localPort,const InetAddress &remoteAddr);
+
 	/**
 	 * Send RENDEZVOUS to two peers to permit them to directly connect
 	 *