Fix for GitHub issue #20 (untested)

node/Constants.hpp

@@ -302,6 +302,11 @@ error_no_byte_order_defined;
  */
 #define ZT_PEER_LINK_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 2) + 1000)
 
+/**
+ * Number of outgoing verb/packetId pairs to keep for sends expecting responses
+ */
+#define ZT_PEER_REQUEST_HISTORY_LENGTH 8
+
 /**
  * IP hops (a.k.a. TTL) to set for firewall opener packets
  *

node/Demarc.hpp

@@ -60,6 +60,9 @@ class Demarc
 public:
 	/**
 	 * Local demarcation port
+	 *
+	 * NULL_PORT is zero so this can be used in if(port) to check for
+	 * a valid/known port.
 	 */
 	typedef uint64_t Port;
 

node/PacketDecoder.cpp

@@ -77,16 +77,9 @@ bool PacketDecoder::tryDecode(const RuntimeEnvironment *_r)
 			return true;
 		}
 
-		Packet::Verb v = verb();
-
-		// Once a packet is determined to be basically valid, it can be used
-		// to passively learn a new network path to the sending peer. It
-		// also results in statistics updates.
-		peer->onReceive(_r,_localPort,_remoteAddress,hops(),v,Utils::now());
-
-		switch(v) {
+		switch(verb()) {
 			case Packet::VERB_NOP:
-				TRACE("NOP from %s(%s)",source().toString().c_str(),_remoteAddress.toString().c_str());
+				peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_NOP,0,Packet::VERB_NOP,Utils::now());
 				return true;
 			case Packet::VERB_HELLO:
 				return _doHELLO(_r); // legal, but why? :)
@@ -130,7 +123,9 @@ bool PacketDecoder::_doERROR(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 {
 	try {
 		Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB];
+		uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID);
 		Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE];
+
 		TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
 
 		switch(errorCode) {
@@ -161,6 +156,8 @@ bool PacketDecoder::_doERROR(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 			default:
 				break;
 		}
+
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,Utils::now());
 	} catch (std::exception &ex) {
 		TRACE("dropped ERROR from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -183,7 +180,6 @@ bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
 			TRACE("dropped HELLO from %s(%s): protocol version mismatch (%u, expected %u)",source().toString().c_str(),_remoteAddress.toString().c_str(),protoVersion,(unsigned int)ZT_PROTO_VERSION);
 			return true;
 		}
-
 		if (!id.locallyValidate()) {
 			TRACE("dropped HELLO from %s(%s): identity invalid",source().toString().c_str(),_remoteAddress.toString().c_str());
 			return true;
@@ -225,14 +221,14 @@ bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
 					_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
 				}
 				return true;
-			}
+			} // else continue since identity is already known and matches
 
 			// Learn a new peer if it's new. This also adds it to the identity
 			// cache if that's enabled.
 			peer = _r->topology->addPeer(SharedPtr<Peer>(new Peer(_r->identity,id)));
 		}
 
-		peer->onReceive(_r,_localPort,_remoteAddress,hops(),Packet::VERB_HELLO,Utils::now());
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP,Utils::now());
 		peer->setRemoteVersion(vMajor,vMinor,vRevision);
 
 		Packet outp(source(),_r->identity.address(),Packet::VERB_OK);
@@ -257,16 +253,17 @@ bool PacketDecoder::_doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &pe
 {
 	try {
 		Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB];
+		uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID);
+
 		//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
+
 		switch(inReVerb) {
 			case Packet::VERB_HELLO: {
-				// OK from HELLO permits computation of latency.
-				unsigned int latency = std::min((unsigned int)(Utils::now() - at<uint64_t>(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)),(unsigned int)0xffff);
+				//unsigned int latency = std::min((unsigned int)(Utils::now() - at<uint64_t>(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)),(unsigned int)0xffff);
 				unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION];
 				unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION];
 				unsigned int vRevision = at<uint16_t>(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION);
-				TRACE("%s(%s): OK(HELLO), latency: %u, version %u.%u.%u",source().toString().c_str(),_remoteAddress.toString().c_str(),latency,vMajor,vMinor,vRevision);
-				peer->setLatency(_remoteAddress,latency);
+				TRACE("%s(%s): OK(HELLO), version %u.%u.%u",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision);
 				peer->setRemoteVersion(vMajor,vMinor,vRevision);
 			}	break;
 			case Packet::VERB_WHOIS: {
@@ -292,8 +289,11 @@ bool PacketDecoder::_doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &pe
 					}
 				}
 			}	break;
-			default: break;
+			default:
+				break;
 		}
+
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,Utils::now());
 	} catch (std::exception &ex) {
 		TRACE("dropped OK from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -327,6 +327,7 @@ bool PacketDecoder::_doWHOIS(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 	} else {
 		TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
+	peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,Utils::now());
 	return true;
 }
 
@@ -355,6 +356,7 @@ bool PacketDecoder::_doRENDEZVOUS(const RuntimeEnvironment *_r,const SharedPtr<P
 				if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
 					InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
 					TRACE("RENDEZVOUS from %s says %s might be at %s, starting NAT-t",source().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
+					peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,Utils::now());
 					_r->sw->contact(withPeer,atAddr);
 				} else {
 					TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",source().toString().c_str(),_remoteAddress.toString().c_str());
@@ -380,18 +382,22 @@ bool PacketDecoder::_doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 		if (network) {
 			if (network->isAllowed(source())) {
 				unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
-				if (network->config()->permitsEtherType(etherType)) {
-					network->tap().put(source().toMAC(),network->tap().mac(),etherType,data() + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD);
-				} else if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
-					TRACE("dropped FRAME from %s: ethernet type %u not allowed on network %.16llx",source().toString().c_str(),etherType,(unsigned long long)network->id());
-				}
+				if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
+					if (network->config()->permitsEtherType(etherType)) {
+						network->tap().put(source().toMAC(),network->tap().mac(),etherType,data() + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD);
+					} else {
+						TRACE("dropped FRAME from %s: ethernet type %u not allowed on network %.16llx",source().toString().c_str(),etherType,(unsigned long long)network->id());
+						return true;
+					}
+				} else return true; // ignore empty frames
 
 				// Source moves "closer" to us in multicast propagation priority when
 				// we receive unicast frames from it. This is called "implicit social
 				// ordering" in other docs.
 				_r->mc->bringCloser(network->id(),source());
+				peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,Utils::now());
 			} else {
-				TRACE("dropped FRAME from %s(%s): not a member of closed network %llu",source().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
+				TRACE("dropped FRAME from %s(%s): sender not a member of closed network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
 
 				Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
 				outp.append((unsigned char)Packet::VERB_FRAME);
@@ -400,9 +406,12 @@ bool PacketDecoder::_doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 				outp.append(network->id());
 				outp.armor(peer->key(),true);
 				_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+
+				return true;
 			}
 		} else {
-			TRACE("dropped FRAME from %s(%s): network %llu unknown",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
+			TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
+			return true;
 		}
 	} catch (std::exception &ex) {
 		TRACE("dropped FRAME from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
@@ -532,6 +541,9 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		_r->demarc->send(Demarc::ANY_PORT,ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct),-1);
 #endif
 
+		// At this point the frame is basically valid, so we can call it a receive
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,Utils::now());
+
 		// This gets updated later in most cases but start with the global limit.
 		unsigned int maxDepth = ZT_MULTICAST_GLOBAL_MAX_DEPTH;
 
@@ -780,6 +792,8 @@ bool PacketDecoder::_doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedP
 					network->pushMembershipCertificate(peer->address(),false,now);
 			}
 		}
+
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,now);
 	} catch (std::exception &ex) {
 		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -792,44 +806,30 @@ bool PacketDecoder::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *
 {
 	try {
 		CertificateOfMembership com;
+
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
 		while (ptr < size()) {
 			ptr += com.deserialize(*this,ptr);
-			if (!com.hasRequiredFields()) {
-				TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): invalid cert: at least one required field is missing",source().toString().c_str(),_remoteAddress.toString().c_str());
-				return true;
-			} else if (com.signedBy()) {
+			if ((com.hasRequiredFields())&&(com.signedBy())) {
 				SharedPtr<Peer> signer(_r->topology->getPeer(com.signedBy()));
 				if (signer) {
 					if (com.verify(signer->identity())) {
 						uint64_t nwid = com.networkId();
 						SharedPtr<Network> network(_r->nc->network(nwid));
 						if (network) {
-							if (network->controller() == signer) {
+							if (network->controller() == signer)
 								network->addMembershipCertificate(com);
-								return true;
-							} else {
-								TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): signer %s is not the controller for network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),signer->address().toString().c_str(),(unsigned long long)nwid);
-								return true;
-							}
-						} else {
-							TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): not a member of network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)nwid);
-							return true;
 						}
-					} else {
-						TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): failed signature verification for signer %s",source().toString().c_str(),_remoteAddress.toString().c_str(),signer->address().toString().c_str());
-						return true;
 					}
 				} else {
 					_r->sw->requestWhois(com.signedBy());
 					_step = DECODE_WAITING_FOR_NETWORK_MEMBERSHIP_CERTIFICATE_SIGNER_LOOKUP;
 					return false;
 				}
-			} else {
-				TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): invalid cert: no signature",source().toString().c_str(),_remoteAddress.toString().c_str());
-				return true;
 			}
 		}
+
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP,Utils::now());
 	} catch (std::exception &ex) {
 		TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -872,6 +872,7 @@ bool PacketDecoder::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *_r,const
 #ifndef __WINDOWS__
 		}
 #endif // !__WINDOWS__
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,Utils::now());
 	} catch (std::exception &exc) {
 		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -892,6 +893,7 @@ bool PacketDecoder::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *_r,const
 				nw->requestConfiguration();
 			}
 		}
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,Utils::now());
 	} catch (std::exception &exc) {
 		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {

node/Peer.cpp

@@ -28,6 +28,8 @@
 #include "Peer.hpp"
 #include "Switch.hpp"
 
+#include <algorithm>
+
 namespace ZeroTier {
 
 Peer::Peer() :
@@ -38,9 +40,11 @@ Peer::Peer() :
 	_lastUnicastFrame(0),
 	_lastMulticastFrame(0),
 	_lastAnnouncedTo(0),
+	_latency(0),
 	_vMajor(0),
 	_vMinor(0),
-	_vRevision(0)
+	_vRevision(0),
+	_requestHistoryPtr(0)
 {
 }
 
@@ -61,19 +65,44 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 		throw std::runtime_error("new peer identity key agreement failed");
 }
 
-void Peer::onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const InetAddress &remoteAddr,unsigned int hops,Packet::Verb verb,uint64_t now)
+void Peer::onReceive(
+	const RuntimeEnvironment *_r,
+	Demarc::Port localPort,
+	const InetAddress &remoteAddr,
+	unsigned int hops,
+	uint64_t packetId,
+	Packet::Verb verb,
+	uint64_t inRePacketId,
+	Packet::Verb inReVerb,
+	uint64_t now)
 {
 	if (!hops) { // direct packet
-		WanPath *wp = (remoteAddr.isV4() ? &_ipv4p : &_ipv6p);
-		wp->lastReceive = now;
-		wp->localPort = localPort;
-		if (!wp->fixed)
-			wp->addr = remoteAddr;
-
+		// Announce multicast LIKEs to peers to whom we have a direct link
 		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
 			_lastAnnouncedTo = now;
 			_r->sw->announceMulticastGroups(SharedPtr<Peer>(this));
 		}
+
+		// Do things like learn latency or endpoints on OK or ERROR replies
+		if (inReVerb != Packet::VERB_NOP) {
+			for(unsigned int p=0;p<ZT_PEER_REQUEST_HISTORY_LENGTH;++p) {
+				if ((_requestHistory[p].packetId == inRePacketId)&&(_requestHistory[p].verb == inReVerb)) {
+					_latency = std::min((unsigned int)(now - _requestHistory[p].timestamp),(unsigned int)0xffff);
+
+					// Only learn paths on replies to packets we have sent, otherwise paths
+					// this introduces both an asymmetry problem in NAT-t and a potential
+					// reply DOS attack.
+					WanPath *const wp = (remoteAddr.isV4() ? &_ipv4p : &_ipv6p);
+					wp->lastReceive = now;
+					wp->localPort = ((localPort) ? localPort : Demarc::ANY_PORT);
+					if (!wp->fixed)
+						wp->addr = remoteAddr;
+
+					_requestHistory[p].packetId = 0;
+					break;
+				}
+			}
+		}
 	}
 
 	if (verb == Packet::VERB_FRAME) {
@@ -83,23 +112,23 @@ void Peer::onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const I
 	}
 }
 
-bool Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,uint64_t now)
+Demarc::Port Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,uint64_t now)
 {
 	if ((_ipv6p.isActive(now))||((!(_ipv4p.addr))&&(_ipv6p.addr))) {
 		if (_r->demarc->send(_ipv6p.localPort,_ipv6p.addr,data,len,-1)) {
 			_ipv6p.lastSend = now;
-			return true;
+			return _ipv6p.localPort;
 		}
 	}
 
 	if (_ipv4p.addr) {
 		if (_r->demarc->send(_ipv4p.localPort,_ipv4p.addr,data,len,-1)) {
 			_ipv4p.lastSend = now;
-			return true;
+			return _ipv4p.localPort;
 		}
 	}
 
-	return false;
+	return Demarc::NULL_PORT;
 }
 
 bool Peer::sendFirewallOpener(const RuntimeEnvironment *_r,uint64_t now)

node/Peer.hpp

@@ -48,6 +48,9 @@
 #include "NonCopyable.hpp"
 #include "Mutex.hpp"
 
+// Increment if serialization has changed
+#define ZT_PEER_SERIALIZATION_VERSION 5
+
 namespace ZeroTier {
 
 /**
@@ -104,28 +107,37 @@ public:
 	/**
 	 * Must be called on authenticated packet receive from this peer
 	 * 
-	 * This must be called only after a packet has passed authentication
-	 * checking. Packets that fail are silently discarded.
-	 *
 	 * @param _r Runtime environment
 	 * @param localPort Local port on which packet was received
 	 * @param remoteAddr Internet address of sender
 	 * @param hops ZeroTier (not IP) hops
+	 * @param packetId Packet ID
 	 * @param verb Packet verb
+	 * @param inRePacketId Packet ID in reply to (for OK/ERROR, 0 otherwise)
+	 * @param inReVerb Verb in reply to (for OK/ERROR, VERB_NOP otherwise)
 	 * @param now Current time
 	 */
-	void onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const InetAddress &remoteAddr,unsigned int hops,Packet::Verb verb,uint64_t now);
+	void onReceive(
+		const RuntimeEnvironment *_r,
+		Demarc::Port localPort,
+		const InetAddress &remoteAddr,
+		unsigned int hops,
+		uint64_t packetId,
+		Packet::Verb verb,
+		uint64_t inRePacketId,
+		Packet::Verb inReVerb,
+		uint64_t now);
 
 	/**
-	 * Send a packet to this peer
+	 * Send a UDP packet to this peer
 	 * 
 	 * @param _r Runtime environment
 	 * @param data Data to send
 	 * @param len Length of packet
 	 * @param now Current time
-	 * @return True if packet appears to have been sent, false on local failure
+	 * @return NULL_PORT or port packet was sent from
 	 */
-	bool send(const RuntimeEnvironment *_r,const void *data,unsigned int len,uint64_t now);
+	Demarc::Port send(const RuntimeEnvironment *_r,const void *data,unsigned int len,uint64_t now);
 
 	/**
 	 * Send firewall opener to active link
@@ -226,68 +238,29 @@ public:
 	/**
 	 * @return Lowest of measured latencies of all paths or 0 if unknown
 	 */
-	inline unsigned int latency() const
-		throw()
-	{
-		if (_ipv4p.latency) {
-			if (_ipv6p.latency)
-				return std::min(_ipv4p.latency,_ipv6p.latency);
-			else return _ipv4p.latency;
-		} else if (_ipv6p.latency)
-			return _ipv6p.latency;
-		return 0;
-	}
-
-	/**
-	 * @param addr Remote address
-	 * @param latency Latency measurment
-	 */
-	inline void setLatency(const InetAddress &addr,unsigned int latency)
-	{
-		if (addr == _ipv4p.addr) {
-			_ipv4p.latency = latency;
-		} else if (addr == _ipv6p.addr) {
-			_ipv6p.latency = latency;
-		}
-	}
+	inline unsigned int latency() const throw() { return _latency; }
 
 	/**
 	 * @return True if this peer has at least one direct IP address path
 	 */
-	inline bool hasDirectPath() const
-		throw()
-	{
-		return ((_ipv4p.addr)||(_ipv6p.addr));
-	}
+	inline bool hasDirectPath() const throw() { return ((_ipv4p.addr)||(_ipv6p.addr)); }
 
 	/**
 	 * @return True if this peer has at least one direct IP address path that looks active
 	 *
 	 * @param now Current time
 	 */
-	inline bool hasActiveDirectPath(uint64_t now) const
-		throw()
-	{
-		return ((_ipv4p.isActive(now))||(_ipv6p.isActive(now)));
-	}
+	inline bool hasActiveDirectPath(uint64_t now) const throw() { return ((_ipv4p.isActive(now))||(_ipv6p.isActive(now))); }
 
 	/**
 	 * @return IPv4 direct address or null InetAddress if none
 	 */
-	inline InetAddress ipv4Path() const
-		throw()
-	{
-		return _ipv4p.addr;
-	}
+	inline InetAddress ipv4Path() const throw() { return _ipv4p.addr; }
 
 	/**
 	 * @return IPv6 direct address or null InetAddress if none
 	 */
-	inline InetAddress ipv6Path() const
-		throw()
-	{
-		return _ipv4p.addr;
-	}
+	inline InetAddress ipv6Path() const throw() { return _ipv4p.addr; }
 
 	/**
 	 * @return IPv4 direct address or null InetAddress if none
@@ -312,13 +285,9 @@ public:
 	}
 
 	/**
-	 * @return 256-bit encryption key
+	 * @return 256-bit secret symmetric encryption key
 	 */
-	inline const unsigned char *key() const
-		throw()
-	{
-		return _key;
-	}
+	inline const unsigned char *key() const throw() { return _key; }
 
 	/**
 	 * Set the remote version of the peer (not persisted)
@@ -347,10 +316,55 @@ public:
 		return std::string("?");
 	}
 
+	/**
+	 * Called when certain packet types are sent that expect OK responses
+	 *
+	 * @param packetId ID of sent packet
+	 * @param verb Verb of sent packet
+	 * @param sentFromLocalPort Outgoing local port
+	 * @param now Current time
+	 */
+	inline void expectResponseTo(uint64_t packetId,Packet::Verb verb,Demarc::Port sentFromLocalPort,uint64_t now)
+		throw()
+	{
+		unsigned int p = _requestHistoryPtr++ % ZT_PEER_REQUEST_HISTORY_LENGTH;
+		_requestHistory[p].timestamp = now;
+		_requestHistory[p].packetId = packetId;
+		_requestHistory[p].localPort = sentFromLocalPort;
+		_requestHistory[p].verb = verb;
+	}
+
+	/**
+	 * @return True if this Peer is initialized with something
+	 */
+	inline operator bool() const throw() { return (_id); }
+
+	/**
+	 * Find a common set of addresses by which two peers can link, if any
+	 *
+	 * @param a Peer A
+	 * @param b Peer B
+	 * @param now Current time
+	 * @return Pair: B's address to send to A, A's address to send to B
+	 */
+	static inline std::pair<InetAddress,InetAddress> findCommonGround(const Peer &a,const Peer &b,uint64_t now)
+		throw()
+	{
+		if ((a._ipv6p.isActive(now))&&(b._ipv6p.isActive(now)))
+			return std::pair<InetAddress,InetAddress>(b._ipv6p.addr,a._ipv6p.addr);
+		else if ((a._ipv4p.isActive(now))&&(b._ipv4p.isActive(now)))
+			return std::pair<InetAddress,InetAddress>(b._ipv4p.addr,a._ipv4p.addr);
+		else if ((a._ipv6p.addr)&&(b._ipv6p.addr))
+			return std::pair<InetAddress,InetAddress>(b._ipv6p.addr,a._ipv6p.addr);
+		else if ((a._ipv4p.addr)&&(b._ipv4p.addr))
+			return std::pair<InetAddress,InetAddress>(b._ipv4p.addr,a._ipv4p.addr);
+		return std::pair<InetAddress,InetAddress>();
+	}
+
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b)
 	{
-		b.append((unsigned char)4); // version
+		b.append((unsigned char)ZT_PEER_SERIALIZATION_VERSION);
 		b.append(_key,sizeof(_key));
 		_id.serialize(b,false);
 		_ipv4p.serialize(b);
@@ -362,14 +376,14 @@ public:
 		b.append((uint16_t)_vMajor);
 		b.append((uint16_t)_vMinor);
 		b.append((uint16_t)_vRevision);
+		b.append((uint16_t)_latency);
 	}
-
 	template<unsigned int C>
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		unsigned int p = startAt;
 
-		if (b[p++] != 4)
+		if (b[p++] != ZT_PEER_SERIALIZATION_VERSION)
 			throw std::invalid_argument("Peer: deserialize(): version mismatch");
 
 		memcpy(_key,b.field(p,sizeof(_key)),sizeof(_key)); p += sizeof(_key);
@@ -383,38 +397,14 @@ public:
 		_vMajor = b.template at<uint16_t>(p); p += sizeof(uint16_t);
 		_vMinor = b.template at<uint16_t>(p); p += sizeof(uint16_t);
 		_vRevision = b.template at<uint16_t>(p); p += sizeof(uint16_t);
+		_latency = b.template at<uint16_t>(p); p += sizeof(uint16_t);
 
 		return (p - startAt);
 	}
-
-	/**
-	 * @return True if this Peer is initialized with something
-	 */
-	inline operator bool() const throw() { return (_id); }
-
+private:
 	/**
-	 * Find a common set of addresses by which two peers can link, if any
-	 *
-	 * @param a Peer A
-	 * @param b Peer B
-	 * @param now Current time
-	 * @return Pair: B's address to send to A, A's address to send to B
+	 * A direct IP path to a peer
 	 */
-	static inline std::pair<InetAddress,InetAddress> findCommonGround(const Peer &a,const Peer &b,uint64_t now)
-		throw()
-	{
-		if ((a._ipv6p.isActive(now))&&(b._ipv6p.isActive(now)))
-			return std::pair<InetAddress,InetAddress>(b._ipv6p.addr,a._ipv6p.addr);
-		else if ((a._ipv4p.isActive(now))&&(b._ipv4p.isActive(now)))
-			return std::pair<InetAddress,InetAddress>(b._ipv4p.addr,a._ipv4p.addr);
-		else if ((a._ipv6p.addr)&&(b._ipv6p.addr))
-			return std::pair<InetAddress,InetAddress>(b._ipv6p.addr,a._ipv6p.addr);
-		else if ((a._ipv4p.addr)&&(b._ipv4p.addr))
-			return std::pair<InetAddress,InetAddress>(b._ipv4p.addr,a._ipv4p.addr);
-		return std::pair<InetAddress,InetAddress>();
-	}
-
-private:
 	class WanPath
 	{
 	public:
@@ -423,7 +413,6 @@ private:
 			lastReceive(0),
 			lastFirewallOpener(0),
 			localPort(Demarc::ANY_PORT),
-			latency(0),
 			addr(),
 			fixed(false)
 		{
@@ -443,7 +432,6 @@ private:
 			b.append(lastReceive);
 			b.append(lastFirewallOpener);
 			b.append(Demarc::portToInt(localPort));
-			b.append((uint16_t)latency);
 
 			b.append((unsigned char)addr.type());
 			switch(addr.type()) {
@@ -472,7 +460,6 @@ private:
 			lastReceive = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 			lastFirewallOpener = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 			localPort = Demarc::intToPort(b.template at<uint64_t>(p)); p += sizeof(uint64_t);
-			latency = b.template at<uint16_t>(p); p += sizeof(uint16_t);
 
 			switch ((InetAddress::AddressType)b[p++]) {
 				case InetAddress::TYPE_NULL:
@@ -497,11 +484,29 @@ private:
 		uint64_t lastReceive;
 		uint64_t lastFirewallOpener;
 		Demarc::Port localPort; // ANY_PORT if not defined (size: uint64_t)
-		unsigned int latency; // 0 if never determined
 		InetAddress addr; // null InetAddress if path is undefined
 		bool fixed; // do not learn address from received packets
 	};
 
+	/**
+	 * A history of a packet sent to a peer expecing a response (e.g. HELLO)
+	 */
+	class RequestHistoryItem
+	{
+	public:
+		RequestHistoryItem() :
+			timestamp(0),
+			packetId(0),
+			verb(Packet::VERB_NOP)
+		{
+		}
+
+		uint64_t timestamp;
+		uint64_t packetId;
+		Demarc::Port localPort;
+		Packet::Verb verb;
+	};
+
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	Identity _id;
 
@@ -512,8 +517,13 @@ private:
 	uint64_t _lastUnicastFrame;
 	uint64_t _lastMulticastFrame;
 	uint64_t _lastAnnouncedTo;
+	unsigned int _latency; // milliseconds, 0 if not known
 	unsigned int _vMajor,_vMinor,_vRevision;
 
+	// not persisted
+	RequestHistoryItem _requestHistory[ZT_PEER_REQUEST_HISTORY_LENGTH];
+	volatile unsigned int _requestHistoryPtr;
+
 	AtomicCounter __refCount;
 };
 

node/Switch.cpp

@@ -219,7 +219,11 @@ bool Switch::sendHELLO(const SharedPtr<Peer> &dest,Demarc::Port localPort,const
 	outp.append(now);
 	_r->identity.serialize(outp,false);
 	outp.armor(dest->key(),false);
-	return _r->demarc->send(localPort,remoteAddr,outp.data(),outp.size(),-1);
+
+	if (_r->demarc->send(localPort,remoteAddr,outp.data(),outp.size(),-1)) {
+		dest->expectResponseTo(outp.packetId(),Packet::VERB_HELLO,localPort,now);
+		return true;
+	} else return false;
 }
 
 bool Switch::unite(const Address &p1,const Address &p2,bool force)
@@ -696,7 +700,8 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 
 		tmp.armor(peer->key(),encrypt);
 
-		if (via->send(_r,tmp.data(),chunkSize,now)) {
+		Demarc::Port localPort;
+		if ((localPort = via->send(_r,tmp.data(),chunkSize,now))) {
 			if (chunkSize < tmp.size()) {
 				// Too big for one bite, fragment the rest
 				unsigned int fragStart = chunkSize;
@@ -716,6 +721,15 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 					remaining -= chunkSize;
 				}
 			}
+
+			switch(packet.verb()) {
+				case Packet::VERB_HELLO:
+					peer->expectResponseTo(packet.packetId(),Packet::VERB_HELLO,localPort,now);
+					break;
+				default:
+					break;
+			}
+
 			return true;
 		}
 		return false;