Whew, it builds!

netconf-service/netconf.cpp

@@ -204,19 +204,15 @@ int main(int argc,char **argv)
 				}
 
 				bool isOpen = false;
-				unsigned int mcb = 3;
-				unsigned int mcd = 6;
 				std::string name,desc;
 				{
 					Query q = dbCon->query();
-					q << "SELECT name,`desc`,isOpen,multicastPropagationBreadth,multicastPropagationDepth FROM Network WHERE id = " << nwid;
+					q << "SELECT name,`desc`,isOpen FROM Network WHERE id = " << nwid;
 					StoreQueryResult rs = q.store();
 					if (rs.num_rows() > 0) {
 						name = rs[0]["name"].c_str();
 						desc = rs[0]["desc"].c_str();
 						isOpen = ((int)rs[0]["isOpen"] > 0);
-						mcb = (unsigned int)rs[0]["multicastPropagationBreadth"];
-						mcd = (unsigned int)rs[0]["multicastPropagationDepth"];
 					} else {
 						Dictionary response;
 						response["peer"] = peerIdentity.address().toString();
@@ -235,18 +231,6 @@ int main(int argc,char **argv)
 					}
 				}
 
-				std::string etherTypeWhitelistOld;
-				{
-					Query q = dbCon->query();
-					q << "SELECT DISTINCT etherType FROM NetworkEthertypes WHERE Network_id = " << nwid;
-					StoreQueryResult rs = q.store();
-					for(unsigned long i=0;i<rs.num_rows();++i) {
-						if (etherTypeWhitelistOld.length() > 0)
-							etherTypeWhitelistOld.push_back(',');
-						etherTypeWhitelistOld.append(rs[i]["etherType"].c_str());
-					}
-				}
-
 				std::string etherTypeWhitelist;
 				{
 					Query q = dbCon->query();
@@ -287,20 +271,14 @@ int main(int argc,char **argv)
 
 				sprintf(buf,"%.16llx",(unsigned long long)nwid);
 				netconf["nwid"] = buf;
-				netconf["isOpen"] = (isOpen ? "1" : "0"); // TODO: remove, old name
 				netconf["o"] = (isOpen ? "1" : "0");
 				netconf["name"] = name;
 				netconf["desc"] = desc;
-				netconf["etherTypes"] = etherTypeWhitelistOld; // TODO: remove, old name
 				netconf["et"] = etherTypeWhitelist;
 				netconf["mr"] = multicastRates.toString();
 				sprintf(buf,"%llx",(unsigned long long)Utils::now());
 				netconf["ts"] = buf;
 				netconf["peer"] = peerIdentity.address().toString();
-				sprintf(buf,"%x",mcb);
-				netconf["mcb"] = buf;
-				sprintf(buf,"%x",mcd);
-				netconf["mcd"] = buf;
 
 				if (!isOpen) {
 					// TODO: handle closed networks, look up private membership,

node/Constants.hpp

@@ -233,7 +233,26 @@ error_no_ZT_ARCH_defined;
 /**
  * Size of multicast deduplication ring buffer in 64-bit ints
  */
-#define ZT_MULTICAST_DEDUP_HISTORY_LENGTH 512
+#define ZT_MULTICAST_DEDUP_HISTORY_LENGTH 1024
+
+/**
+ * Maximum propagation depth for multicasts
+ *
+ * TODO: in the future make this configurable per-network.
+ */
+#define ZT_MULTICAST_MAX_PROPAGATION_DEPTH 128
+
+/**
+ * Number of frames to send when originating a multicast
+ *
+ * This MUST be a power of two.
+ */
+#define ZT_MULTICAST_NUM_PROPAGATION_PREFIXES 4
+
+/**
+ * Power of two used for ZT_MULTICAST_NUM_PROPAGATION_PREFIXES
+ */
+#define ZT_MULTICAST_NUM_PROPAGATION_PREFIX_BITS 2
 
 /**
  * Period between announcements of all multicast 'likes' in ms

node/Demarc.cpp

@@ -41,6 +41,7 @@
 #include "InetAddress.hpp"
 #include "Switch.hpp"
 #include "Buffer.hpp"
+#include "CMWC4096.hpp"
 
 namespace ZeroTier {
 

node/Multicaster.cpp

@@ -93,7 +93,7 @@ void Multicaster::clean()
 			} else ++s;
 		}
 
-		if (n->second.got.empty()&&n->second.proximity.empty()&&n->second.subscriptions.empty())
+		if (n->second.proximity.empty()&&n->second.subscriptions.empty())
 			_nets.erase(n++);
 		else ++n;
 	}

node/Multicaster.hpp

@@ -97,7 +97,7 @@ public:
 			if (n.multicastHistory[i] == mcGuid)
 				return true;
 		}
-		n.multicastHistory[n.multicastHistoryPtr++ % ZT_NETWORK_MULTICAST_DEDUP_HISTORY_LENGTH] = mcGuid;
+		n.multicastHistory[n.multicastHistoryPtr++ % ZT_MULTICAST_DEDUP_HISTORY_LENGTH] = mcGuid;
 		return false;
 	}
 
@@ -110,28 +110,92 @@ public:
 	 * @param nwid Network ID
 	 * @param mg Multicast group
 	 * @param nextHopFunc Function to call for each address, search stops if it returns false
-	 * @return Number of results returned through function
 	 */
 	template<typename F>
-	inline unsigned int getNextHops(uint64_t nwid,const MulticastGroup &mg,F nextHopFunc)
+	inline void getNextHops(uint64_t nwid,const MulticastGroup &mg,F nextHopFunc)
 	{
 		Mutex::Lock _l(_lock);
 
 		std::map< uint64_t,_NetInfo >::iterator n(_nets.find(nwid));
 		if (n == _nets.end())
-			return 0;
+			return;
 		std::map< MulticastGroup,std::list< Address > >::iterator p(n->second.proximity.find(mg));
 		if (p == n->second.proximity.end())
-			return 0;
+			return;
 
-		unsigned int cnt = 0;
 		for(std::list< Address >::iterator a(p->second.begin());a!=p->second.end();++a) {
 			if (!nextHopFunc(*a))
 				break;
 		}
-		return cnt;
 	}
 
+	/**
+	 * Functor to add addresses to multicast frame propagation queues
+	 *
+	 * This function object checks the origin, bloom filter, and restriction
+	 * prefix for each address and if all these pass it adds the address and
+	 * increments the pointer pointed to by ptr. It stops (returns false) when
+	 * *ptr reaches end. It's used in PacketDecoder and Switch with getNextHops()
+	 * to compose multicast frame headers.
+	 */
+	class AddToPropagationQueue
+	{
+	public:
+		/**
+		 * @param ptr Pointer to pointer to current position in queue
+		 * @param end End of queue
+		 * @param bloom Bloom filter field (must be 1024 bytes in length)
+		 * @param bloomNonce Random nonce for bloom filter randomization
+		 * @param origin Originating address
+		 * @param prefixBits Number of bits in propagation restriction prefix
+		 * @param prefix Propagation restrition prefix
+		 */
+		AddToPropagationQueue(unsigned char **ptr,unsigned char *end,unsigned char *bloom,uint16_t bloomNonce,const Address &origin,unsigned int prefixBits,unsigned int prefix)
+			throw() :
+			_origin(origin),
+			_bloomNonce((uint64_t)bloomNonce),
+			_ptr(ptr),
+			_end(end),
+			_bloom(bloom),
+			_prefix(prefix),
+			_prefixBits(std::min(prefixBits,(unsigned int)16)) {}
+
+		inline bool operator()(const Address &a)
+			throw()
+		{
+			// Exclude original sender -- obviously they've already seen it
+			if (a == _origin)
+				return true;
+
+			// Prefixes match if N least significant bits in address are equal to the
+			// prefix. (e.g. 0 bits and 0 prefix would match all, 1 bit and 0 prefix
+			// would match addresses with LSB == 0)
+			if (((unsigned int)a.toInt() & (0xffff >> (16 - _prefixBits))) != _prefix)
+				return true;
+
+			// Exclude addresses remembered in bloom filter -- or else remember them
+			uint64_t aint = a.toInt() + _bloomNonce;
+			const unsigned int bit = (unsigned int)(aint ^ (aint >> 13) ^ (aint >> 26) ^ (aint >> 39)) & 0x1fff;
+			unsigned char *const bbyte = _bloom + (bit >> 3); // note: bloom filter size == 1024 is hard-coded here
+			const unsigned char bmask = 0x80 >> (bit & 7);
+			if ((*bbyte & bmask))
+				return true;
+			else *bbyte |= bmask;
+
+			a.copyTo(*_ptr,ZT_ADDRESS_LENGTH);
+			return ((*_ptr += ZT_ADDRESS_LENGTH) != _end);
+		}
+
+	private:
+		const Address _origin;
+		const uint64_t _bloomNonce;
+		unsigned char **const _ptr;
+		unsigned char *const _end;
+		unsigned char *const _bloom;
+		const unsigned int _prefix;
+		const unsigned int _prefixBits;
+	};
+
 private:
 	// Information about a subscription
 	struct _SubInfo

node/Packet.cpp

@@ -29,7 +29,7 @@
 
 namespace ZeroTier {
 
-const unsigned char Packet::ZERO_KEY[32] { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
+const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
 
 const char *Packet::verbString(Verb v)
 	throw()

node/Packet.hpp

@@ -165,7 +165,10 @@
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO 320
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_FIFO + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM 1024
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FLAGS 1
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FLAGS)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_NETWORK_ID 8
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM_NONCE (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_NETWORK_ID)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM_NONCE 2
@@ -189,7 +192,7 @@
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_ETHERTYPE 2
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME_LEN (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_ETHERTYPE)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FRAME_LEN 2
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PAYLOAD_LEN + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PAYLOAD_LEN)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME_LEN + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FRAME_LEN)
 
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID + 8)
@@ -465,7 +468,8 @@ public:
 		 * ZeroTier addresses. ZeroTier does not support VLANs or other extensions
 		 * beyond core Ethernet.
 		 *
-		 * No OK or ERROR is generated.
+		 * ERROR may be generated if a membership certificate is needed for a
+		 * closed network. Payload will be network ID.
 		 */
 		VERB_FRAME = 6,
 
@@ -477,10 +481,11 @@ public:
 		 *   <[320] propagation FIFO>
 		 *   <[1024] propagation bloom filter>
 		 *   [... begin signed portion ...]
+		 *   <[1] 8-bit flags, currently unused and must be 0>
 		 *   <[8] 64-bit network ID>
 		 *   <[2] 16-bit random propagation bloom filter nonce>
 		 *   <[1] number of significant bits in propagation restrict prefix>
-		 *   <[2] 16-bit propagation restriction prefix (left to right)>
+		 *   <[2] 16-bit propagation restriction prefix (sig bits right to left)>
 		 *   <[5] ZeroTier address of node of origin>
 		 *   <[3] 24-bit multicast ID, together with origin forms GUID>
 		 *   <[6] source MAC address>
@@ -524,11 +529,10 @@ public:
 		 * Propagation occurs within a restrict prefix. The restrict prefix is
 		 * applied to the least significant 16 bits of an address. The original
 		 * sender of the multicast sets the restrict prefix and sends 2^N copies
-		 * of the multicast frame, one for each address prefix. This permits
-		 * propagation to be partitioned into realms, and places the majority of
-		 * the burden for this upon the sender.
+		 * of the multicast frame, one for each address prefix.
 		 *
-		 * OK/ERROR are not generated.
+		 * ERROR may be generated if a membership certificate is needed for a
+		 * closed network. Payload will be network ID.
 		 */
 		VERB_MULTICAST_FRAME = 8,
 
@@ -550,7 +554,7 @@ public:
 		 *   <[2] 16-bit length of certificate>
 		 *   <[2] 16-bit length of signature>
 		 *   <[...] string-serialized certificate dictionary>
-		 *   <[...] ECDSA signature of certificate>
+		 *   <[...] signature of certificate>
 		 *
 		 * OK is generated on acceptance. ERROR is returned on failure. In both
 		 * cases the payload is the network ID.

node/PacketDecoder.cpp

@@ -92,12 +92,12 @@ bool PacketDecoder::tryDecode(const RuntimeEnvironment *_r)
 				return _doRENDEZVOUS(_r,peer);
 			case Packet::VERB_FRAME:
 				return _doFRAME(_r,peer);
-			case Packet::VERB_MULTICAST_LIKE:
-				return _doMULTICAST_LIKE(_r,peer);
-			case Packet::VERB_MULTICAST_GOT:
-				return _doMULTICAST_GOT(_r,peer);
+			case Packet::VERB_PROXY_FRAME:
+				return _doPROXY_FRAME(_r,peer);
 			case Packet::VERB_MULTICAST_FRAME:
 				return _doMULTICAST_FRAME(_r,peer);
+			case Packet::VERB_MULTICAST_LIKE:
+				return _doMULTICAST_LIKE(_r,peer);
 			case Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE:
 				return _doNETWORK_MEMBERSHIP_CERTIFICATE(_r,peer);
 			case Packet::VERB_NETWORK_CONFIG_REQUEST:
@@ -137,8 +137,7 @@ void PacketDecoder::_CBaddPeerFromHello(void *arg,const SharedPtr<Peer> &p,Topol
 				outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
 				outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
 				outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
-				outp.encrypt(p->cryptKey());
-				outp.macSet(p->macKey());
+				outp.armor(p->key(),true);
 				_r->demarc->send(req->localPort,req->remoteAddress,outp.data(),outp.size(),-1);
 			}	break;
 
@@ -147,8 +146,7 @@ void PacketDecoder::_CBaddPeerFromHello(void *arg,const SharedPtr<Peer> &p,Topol
 				outp.append((unsigned char)Packet::VERB_HELLO);
 				outp.append(req->helloPacketId);
 				outp.append((unsigned char)Packet::ERROR_IDENTITY_INVALID);
-				outp.encrypt(p->cryptKey());
-				outp.macSet(p->macKey());
+				outp.armor(p->key(),true);
 				_r->demarc->send(req->localPort,req->remoteAddress,outp.data(),outp.size(),-1);
 			}	break;
 
@@ -158,8 +156,7 @@ void PacketDecoder::_CBaddPeerFromHello(void *arg,const SharedPtr<Peer> &p,Topol
 				outp.append((unsigned char)Packet::VERB_HELLO);
 				outp.append(req->helloPacketId);
 				outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
-				outp.encrypt(p->cryptKey());
-				outp.macSet(p->macKey());
+				outp.armor(p->key(),true);
 				_r->demarc->send(req->localPort,req->remoteAddress,outp.data(),outp.size(),-1);
 			}	break;
 		}
@@ -191,16 +188,26 @@ void PacketDecoder::_CBaddPeerFromWhois(void *arg,const SharedPtr<Peer> &p,Topol
 bool PacketDecoder::_doERROR(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
 {
 	try {
-#ifdef ZT_TRACE
 		Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB];
 		Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE];
 		TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
-#endif
-		// TODO (sorta):
-		// The fact is that the protocol works fine without error handling.
-		// The only error that really needs to be handled here is duplicate
-		// identity collision, which if it comes from a supernode should cause
-		// us to restart and regenerate a new identity.
+
+		switch(errorCode) {
+			case Packet::ERROR_OBJ_NOT_FOUND:
+				if (inReVerb == Packet::VERB_WHOIS) {
+					// TODO: abort WHOIS if sender is a supernode
+				}
+				break;
+			case Packet::ERROR_IDENTITY_COLLISION:
+			case Packet::ERROR_IDENTITY_INVALID:
+				// TODO: if it comes from a supernode, regenerate a new identity
+				break;
+			case Packet::ERROR_NO_MEMBER_CERTIFICATE:
+				// TODO: send member certificate
+				break;
+			default:
+				break;
+		}
 	} catch (std::exception &ex) {
 		TRACE("dropped ERROR from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -243,8 +250,7 @@ bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
 			outp.append((unsigned char)Packet::VERB_HELLO);
 			outp.append(packetId());
 			outp.append(timestamp);
-			outp.encrypt(existingPeer->cryptKey());
-			outp.macSet(existingPeer->macKey());
+			outp.armor(existingPeer->key(),true);
 			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
 			return true;
 		}
@@ -281,8 +287,12 @@ bool PacketDecoder::_doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &pe
 			case Packet::VERB_HELLO: {
 				// OK from HELLO permits computation of latency.
 				unsigned int latency = std::min((unsigned int)(Utils::now() - at<uint64_t>(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)),(unsigned int)0xffff);
-				TRACE("%s(%s): OK(HELLO), latency: %u",source().toString().c_str(),_remoteAddress.toString().c_str(),latency);
+				unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION];
+				unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION];
+				unsigned int vRevision = at<uint16_t>(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION);
+				TRACE("%s(%s): OK(HELLO), latency: %u, version %u.%u.%u",source().toString().c_str(),_remoteAddress.toString().c_str(),latency,vMajor,vMinor,vRevision);
 				peer->setLatency(_remoteAddress,latency);
+				peer->setRemoteVersion(vMajor,vMinor,vRevision);
 			}	break;
 			case Packet::VERB_WHOIS: {
 				TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
@@ -328,8 +338,7 @@ bool PacketDecoder::_doWHOIS(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 			outp.append((unsigned char)Packet::VERB_WHOIS);
 			outp.append(packetId());
 			p->identity().serialize(outp,false);
-			outp.encrypt(peer->cryptKey());
-			outp.macSet(peer->macKey());
+			outp.armor(peer->key(),true);
 			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
 			TRACE("sent WHOIS response to %s for %s",source().toString().c_str(),Address(payload(),ZT_ADDRESS_LENGTH).toString().c_str());
 		} else {
@@ -338,8 +347,7 @@ bool PacketDecoder::_doWHOIS(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 			outp.append(packetId());
 			outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 			outp.append(payload(),ZT_ADDRESS_LENGTH);
-			outp.encrypt(peer->cryptKey());
-			outp.macSet(peer->macKey());
+			outp.armor(peer->key(),true);
 			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
 			TRACE("sent WHOIS ERROR to %s for %s (not found)",source().toString().c_str(),Address(payload(),ZT_ADDRESS_LENGTH).toString().c_str());
 		}
@@ -404,8 +412,21 @@ bool PacketDecoder::_doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 				} else if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 					TRACE("dropped FRAME from %s: ethernet type %u not allowed on network %.16llx",source().toString().c_str(),etherType,(unsigned long long)network->id());
 				}
+
+				// Source moves "closer" to us in multicast propagation priority when
+				// we receive unicast frames from it. This is called "implicit social
+				// ordering" in other docs.
+				_r->mc->bringCloser(network->id(),source());
 			} else {
 				TRACE("dropped FRAME from %s(%s): not a member of closed network %llu",source().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
+
+				Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+				outp.append((unsigned char)Packet::VERB_FRAME);
+				outp.append(packetId());
+				outp.append((unsigned char)Packet::ERROR_NO_MEMBER_CERTIFICATE);
+				outp.append(network->id());
+				outp.armor(peer->key(),true);
+				_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
 			}
 		} else {
 			TRACE("dropped FRAME from %s(%s): network %llu unknown",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
@@ -418,97 +439,12 @@ bool PacketDecoder::_doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 	return true;
 }
 
-bool PacketDecoder::_doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+bool PacketDecoder::_doPROXY_FRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
 {
-	try {
-		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
-		if (ptr >= size())
-			return true;
-		uint64_t now = Utils::now();
-		Address src(source());
-
-		// Iterate through 18-byte network,MAC,ADI tuples
-		for(;;) {
-			_r->mc->likesGroup(at<uint64_t>(ptr),src,MulticastGroup(MAC(field(ptr + 8,6)),at<uint32_t>(ptr + 14)),now);
-			if ((ptr += 18) >= size())
-				break;
-		}
-	} catch (std::exception &ex) {
-		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
-	} catch ( ... ) {
-		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
-	}
-
-	return true;
-}
-
-bool PacketDecoder::_doMULTICAST_GOT(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
-{
-	// Right now only supernodes act as propagation hubs
-	if (!_r->topology->amSupernode()) {
-		TRACE("dropped MULTICAST_GOT from %s: I am not a supernode",source().toString().c_str());
-		return true;
-	}
-
-	try {
-		_r->mc->got(at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GOT_IDX_NETWORK_ID),source(),at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GOT_IDX_MULTICAST_GUID));
-	} catch (std::exception &ex) {
-		TRACE("dropped MULTICAST_GOT from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
-	} catch ( ... ) {
-		TRACE("dropped MULTICAST_GOT from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
-	}
-
+	// TODO: bridging is not implemented yet
 	return true;
 }
 
-// Function used in _doMULTICAST_FRAME
-static inline unsigned int _bloomBit(const Address &a,uint16_t bloomNonce)
-	throw()
-{
-	uint64_t a = a.toInt() + (uint64_t)bloomNonce;
-	unsigned int bit = (unsigned int)(a & 0x1fff);
-	bit ^= (unsigned int)((a >> 13) & 0x1fff);
-	bit ^= (unsigned int)((a >> 26) & 0x1fff);
-	bit ^= (unsigned int)((a >> 39) & 0x1fff);
-	return bit;
-}
-
-// Function object used in _doMULTICAST_FRAME
-struct _PushNextHops
-{
-	_PushNextHops(unsigned char **ptr_,unsigned char *end_,unsigned char *bloom_,uint16_t bloomNonce_const Address &origin_)
-		ptr(ptr_),
-		end(end_),
-		bloom(bloom_),
-		origin(origin_),
-		bloomNonce(bloomNonce_) throw() {}
-
-	inline bool operator()(const Address &a) const
-		throw()
-	{
-		if (a == origin)
-			return true;
-
-		unsigned int bb = _bloomBit(a,bloomNonce);
-		unsigned char *bbyte = bloom + (bb >> 3);
-		unsigned char bmask = 0x80 >> (bb & 7);
-		if ((*bbyte & bmask))
-			return true;
-		else *bbyte |= bmask;
-
-		a.copyTo(*ptr,ZT_ADDRESS_LENGTH);
-		*ptr += ZT_ADDRESS_LENGTH;
-
-		return (*ptr != end);
-	}
-
-	unsigned char **ptr;
-	unsigned char *end;
-	unsigned char *bloom;
-	Address origin;
-	uint16_t bloomNonce;
-};
-
 bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
 {
 	try {
@@ -526,7 +462,7 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID);
 		uint16_t bloomNonce = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM_NONCE);
 		unsigned int prefixBits = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_PREFIX_BITS];
-		uint16_t prefix = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_PREFIX);
+		unsigned int prefix = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_PREFIX);
 		uint64_t guid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GUID);
 		MAC sourceMac(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_SOURCE_MAC));
 		MulticastGroup dest(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_DEST_MAC)),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI));
@@ -536,13 +472,18 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		unsigned int signatureLen = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME + frameLen);
 		unsigned char *signature = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME + frameLen + 2,signatureLen);
 
-		unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID) + frameLen;
-		if (!submitter->identity().verify(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID,signedPartLen),signedPartLen,signature,signatureLen)) {
+		unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION) + frameLen;
+		if (!originPeer->identity().verify(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION,signedPartLen),signedPartLen,signature,signatureLen)) {
 			TRACE("dropped MULTICAST_FRAME from %s(%s): failed signature verification, claims to be from %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
 			return true;
 		}
 
-		if (_r->mc->deduplicate(nwid,guid)) {
+		if (!dest.mac().isMulticast()) {
+			TRACE("dropped MULTICAST_FRAME from %s(%s): %s is not a multicast/broadcast address",source().toString().c_str(),_remoteAddress.toString().c_str(),dest.mac().toString().c_str());
+			return true;
+		}
+
+		if ((origin == _r->identity.address())||(_r->mc->deduplicate(nwid,guid))) {
 			TRACE("dropped MULTICAST_FRAME from %s(%s): duplicate",source().toString().c_str(),_remoteAddress.toString().c_str());
 			return true;
 		}
@@ -551,21 +492,44 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 
 		SharedPtr<Network> network(_r->nc->network(nwid));
 		if (network) {
-			if (!network->isAllowed(submitterAddr)) {
-			} else if (!dest.mac().isMulticast()) {
-			} else if ((!network->permitsBridging())&&(!submitterAddr.wouldHaveMac(sourceMac))) {
+			if (!network->isAllowed(origin)) {
+				TRACE("didn't inject MULTICAST_FRAME from %s(%s) into %.16llx: sender %s not allowed or we don't have a certificate",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),origin.toString().c_str());
+
+				Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+				outp.append((unsigned char)Packet::VERB_FRAME);
+				outp.append(packetId());
+				outp.append((unsigned char)Packet::ERROR_NO_MEMBER_CERTIFICATE);
+				outp.append(nwid);
+				outp.armor(peer->key(),true);
+				_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+
+				// We do not terminate here, since if the member just has an out of
+				// date cert or hasn't sent us a cert yet we still want to propagate
+				// the message so multicast works.
+			} else if ((!network->permitsBridging())&&(!origin.wouldHaveMac(sourceMac))) {
+				TRACE("didn't inject MULTICAST_FRAME from %s(%s) into %.16llx: source mac %s doesn't belong to %s, and bridging is not supported on network",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),sourceMac.toString().c_str(),origin.toString().c_str());
 			} else if (!network->permitsEtherType(etherType)) {
-			} else if (network->updateAndCheckMulticastBalance(submitterAddr,dest,frameLen)) {
+				TRACE("didn't inject MULTICAST_FRAME from %s(%s) into %.16llx: ethertype %u is not allowed",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),etherType);
+			} else if (network->updateAndCheckMulticastBalance(origin,dest,frameLen)) {
 				network->tap().put(sourceMac,dest.mac(),etherType,frame,frameLen);
-			} else rateLimitsExceeded = true;
+			} else {
+				rateLimitsExceeded = true;
+			}
 		}
 
-		if ((rateLimitsExceeded)&&(!_r->topology->amSupernode())) {
-			TRACE("dropped MULTICAST_FRAME from %s(%s): rate limit exceeded for sender %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
+		// We can only really know if rate limit was exceeded if we're a member of
+		// this network. This will nearly always be true for anyone getting a
+		// multicast except supernodes, so the net effect will be to truncate
+		// multicast propagation if the rate limit is exceeded.
+		if (rateLimitsExceeded) {
+			TRACE("dropped MULTICAST_FRAME from %s(%s): rate limits exceeded for sender %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
 			return true;
 		}
 
-		++depth; // TODO: implement max depth
+		if (++depth > ZT_MULTICAST_MAX_PROPAGATION_DEPTH) {
+			TRACE("dropped MULTICAST_FRAME from %s(%s): max propagation depth reached",source().toString().c_str(),_remoteAddress.toString().c_str());
+			return true;
+		}
 		setAt(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_DEPTH,(uint16_t)depth);
 
 		// New FIFO with room for one extra, since head will be next hop
@@ -573,17 +537,17 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		unsigned char *newFifoPtr = newFifo;
 		unsigned char *newFifoEnd = newFifoPtr + sizeof(newFifo);
 		for(unsigned int i=0;i<ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO;) {
-			unsigned char zm = 0;
 			unsigned int j = i;
 			i += ZT_ADDRESS_LENGTH;
+			unsigned char zm = 0;
 			while (j != i)
 				zm |= (*(newFifoPtr++) = fifo[j++]);
 			if (!zm) // stop at zero address
 				break;
 		}
 
-		// Fill remaining part of new fifo
-		_r->mc->getNextHops(nwid,dest,_PushNextHops(&newFifoPtr,newFifoEnd,bloom,bloomNonce,origin));
+		// Add any next hops we know about to FIFO
+		_r->mc->getNextHops(nwid,dest,Multicaster::AddToPropagationQueue(&newFifoPtr,newFifoEnd,bloom,bloomNonce,origin,prefixBits,prefix));
 
 		// Zero-terminate new FIFO if not completely full
 		while (newFifoPtr != newFifoEnd)
@@ -591,16 +555,23 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 
 		// First element in newFifo[] is next hop
 		Address nextHop(newFifo,ZT_ADDRESS_LENGTH);
-
-		// Send to next hop, unless it's us of course
-		if (nextHop != _r->identity.address()) {
-			newInitializationVector();
-			setDestination(nextHop);
-			setSource(_r->identity.address());
-			compress();
-			_r->sw->send(*this,true);
+		if (!nextHop)
+			nextHop = _r->topology->getBestSupernode(&origin,1,true); // exclude origin in case it's itself a supernode
+		if ((!nextHop)||(nextHop == _r->identity.address())) { // check against our addr is a sanity check
+			TRACE("not forwarding MULTICAST_FRAME from %s(%s): no next hop",source().toString().c_str(),_remoteAddress.toString().c_str());
+			return true;
 		}
 
+		// The rest of newFifo[] goes back into the packet
+		memcpy(fifo,newFifo + ZT_ADDRESS_LENGTH,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO);
+
+		// Send to next hop, reusing this packet as scratch space
+		newInitializationVector();
+		setDestination(nextHop);
+		setSource(_r->identity.address());
+		compress(); // note: bloom filters and empty FIFOs are highly compressable!
+		_r->sw->send(*this,true);
+
 		return true;
 	} catch (std::exception &ex) {
 		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
@@ -611,10 +582,33 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 	return true;
 }
 
+bool PacketDecoder::_doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
+		if (ptr >= size())
+			return true;
+		uint64_t now = Utils::now();
+		Address src(source());
+
+		// Iterate through 18-byte network,MAC,ADI tuples
+		for(;;) {
+			_r->mc->likesGroup(at<uint64_t>(ptr),src,MulticastGroup(MAC(field(ptr + 8,6)),at<uint32_t>(ptr + 14)),now);
+			if ((ptr += 18) >= size())
+				break;
+		}
+	} catch (std::exception &ex) {
+		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+
+	return true;
+}
+
 bool PacketDecoder::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
 {
 	// TODO: not implemented yet, will be needed for private networks.
-
 	return true;
 }
 
@@ -645,8 +639,7 @@ bool PacketDecoder::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *_r,const
 			outp.append(packetId());
 			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
 			outp.append(nwid);
-			outp.encrypt(peer->cryptKey());
-			outp.macSet(peer->macKey());
+			outp.armor(peer->key(),true);
 			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
 #ifndef __WINDOWS__
 		}

node/PacketDecoder.hpp

@@ -137,9 +137,9 @@ private:
 	bool _doWHOIS(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doRENDEZVOUS(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
-	bool _doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
-	bool _doMULTICAST_GOT(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doPROXY_FRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_FRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);

node/Switch.cpp

@@ -49,13 +49,15 @@
 #include "NodeConfig.hpp"
 #include "Demarc.hpp"
 #include "Filter.hpp"
+#include "CMWC4096.hpp"
 
 #include "../version.h"
 
 namespace ZeroTier {
 
 Switch::Switch(const RuntimeEnvironment *renv) :
-	_r(renv)
+	_r(renv),
+	_multicastIdCounter((unsigned int)renv->prng->next32()) // start a random spot to minimize possible collisions on startup
 {
 }
 
@@ -81,18 +83,18 @@ void Switch::onRemotePacket(Demarc::Port localPort,const InetAddress &fromAddr,c
 
 void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data)
 {
-	if (from != network->tap().mac()) {
-		LOG("ignored tap: %s -> %s %s (bridging is not (yet?) supported)",from.toString().c_str(),to.toString().c_str(),Filter::etherTypeName(etherType));
+	if (to == network->tap().mac()) {
+		LOG("%s: frame received from self, ignoring (bridge loop? OS bug?)",network->tap().deviceName().c_str());
 		return;
 	}
 
-	if (to == network->tap().mac()) {
-		LOG("%s: frame received from self, ignoring (bridge loop? OS bug?)",network->tap().deviceName().c_str());
+	if (from != network->tap().mac()) {
+		LOG("ignored tap: %s -> %s %s (bridging not supported)",from.toString().c_str(),to.toString().c_str(),Filter::etherTypeName(etherType));
 		return;
 	}
 
 	if (!network->permitsEtherType(etherType)) {
-		LOG("ignored tap: %s -> %s: ethernet type %s not allowed on network %.16llx",from.toString().c_str(),to.toString().c_str(),Filter::etherTypeName(etherType),(unsigned long long)network->id());
+		LOG("ignored tap: %s -> %s: ethertype %s not allowed on network %.16llx",from.toString().c_str(),to.toString().c_str(),Filter::etherTypeName(etherType),(unsigned long long)network->id());
 		return;
 	}
 
@@ -105,58 +107,55 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 				mg = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(data.field(24,4),4,0));
 		}
 
-		uint64_t crc = Multicaster::computeMulticastDedupCrc(network->id(),from,mg,etherType,data.data(),data.size());
-		uint64_t now = Utils::now();
+		unsigned int mcid = ++_multicastIdCounter & 0xffffff;
+		uint16_t bloomNonce = (uint16_t)_r->prng->next32(); // doesn't need to be cryptographically strong
+		unsigned char bloom[ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM];
+		unsigned char fifo[ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO + ZT_ADDRESS_LENGTH];
+		for(unsigned int prefix=0;prefix<ZT_MULTICAST_NUM_PROPAGATION_PREFIXES;++prefix) {
+			memset(bloom,0,sizeof(bloom));
+
+			unsigned char *fifoPtr = fifo;
+			unsigned char *fifoEnd = fifo + sizeof(fifo);
+
+			_r->mc->getNextHops(network->id(),mg,Multicaster::AddToPropagationQueue(&fifoPtr,fifoEnd,bloom,bloomNonce,_r->identity.address(),ZT_MULTICAST_NUM_PROPAGATION_PREFIX_BITS,prefix));
+			while (fifoPtr != fifoEnd)
+				*(fifoPtr++) = (unsigned char)0;
+
+			Address firstHop(fifo,ZT_ADDRESS_LENGTH); // fifo is +1 in size, with first element being used here
+			if (!firstHop) {
+				SharedPtr<Peer> sn(_r->topology->getBestSupernode());
+				if (sn)
+					firstHop = sn->address();
+				else break;
+			}
 
-		if (_r->multicaster->checkDuplicate(crc,now)) {
-			LOG("%s/%.16llx: multicast group %s: dropped %u bytes, duplicate multicast in too short a time frame",network->tap().deviceName().c_str(),(unsigned long long)network->id(),mg.toString().c_str(),(unsigned int)data.size());
-			return;
-		}
-		_r->multicaster->addToDedupHistory(crc,now);
-		if (!network->updateAndCheckMulticastBalance(_r->identity.address(),mg,data.size())) {
-			LOG("%s/%.16llx: multicast group %s: dropped %u bytes, out of budget",network->tap().deviceName().c_str(),(unsigned long long)network->id(),mg.toString().c_str(),(unsigned int)data.size());
-			return;
-		}
+			Packet outp(firstHop,_r->identity.address(),Packet::VERB_MULTICAST_FRAME);
+			outp.append((uint16_t)0);
+			outp.append(fifo + ZT_ADDRESS_LENGTH,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO); // remainder of fifo is loaded into packet
+			outp.append(bloom,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM);
+			outp.append((unsigned char)0);
+			outp.append(network->id());
+			outp.append(bloomNonce);
+			outp.append((unsigned char)ZT_MULTICAST_NUM_PROPAGATION_PREFIX_BITS);
+			outp.append((uint16_t)prefix);
+			_r->identity.address().appendTo(outp);
+			outp.append((unsigned char)((mcid >> 16) & 0xff));
+			outp.append((unsigned char)((mcid >> 8) & 0xff));
+			outp.append((unsigned char)(mcid & 0xff));
+			outp.append(from.data,6);
+			outp.append(mg.mac().data,6);
+			outp.append(mg.adi());
+			outp.append((uint16_t)etherType);
+			outp.append((uint16_t)data.size());
+			outp.append(data);
 
-		Multicaster::MulticastBloomFilter bloom;
-		SharedPtr<Peer> propPeers[16];
-		unsigned int np = _r->multicaster->pickSocialPropagationPeers(
-			*(_r->prng),
-			*(_r->topology),
-			network->id(),
-			mg,
-			_r->identity.address(),
-			Address(),
-			bloom,
-			std::min(network->multicastPropagationBreadth(),(unsigned int)16), // 16 is a sanity check
-			propPeers,
-			now);
-
-		if (!np)
-			return;
-
-		C25519::Signature signature(Multicaster::signMulticastPacket(_r->identity,network->id(),from,mg,etherType,data.data(),data.size()));
-
-		Packet outpTmpl(propPeers[0]->address(),_r->identity.address(),Packet::VERB_MULTICAST_FRAME);
-		outpTmpl.append((uint8_t)0);
-		outpTmpl.append((uint64_t)network->id());
-		_r->identity.address().appendTo(outpTmpl);
-		outpTmpl.append(from.data,6);
-		outpTmpl.append(mg.mac().data,6);
-		outpTmpl.append((uint32_t)mg.adi());
-		outpTmpl.append(bloom.data(),ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE_BYTES);
-		outpTmpl.append((uint8_t)0); // 0 hops
-		outpTmpl.append((uint16_t)etherType);
-		outpTmpl.append((uint16_t)data.size());
-		outpTmpl.append((uint16_t)signature.size());
-		outpTmpl.append(data.data(),data.size());
-		outpTmpl.append(signature.data,(unsigned int)signature.size());
-		outpTmpl.compress();
-		send(outpTmpl,true);
-		for(unsigned int i=1;i<np;++i) {
-			outpTmpl.newInitializationVector();
-			outpTmpl.setDestination(propPeers[i]->address());
-			send(outpTmpl,true);
+			unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION) + data.size();
+			C25519::Signature sig(_r->identity.sign(outp.field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION,signedPartLen),signedPartLen));
+			outp.append((uint16_t)sig.size());
+			outp.append(sig.data,sig.size());
+
+			outp.compress();
+			send(outp,true);
 		}
 	} else if (to.isZeroTier()) {
 		// Simple unicast frame from us to another node
@@ -206,7 +205,6 @@ void Switch::sendHELLO(const Address &dest)
 bool Switch::sendHELLO(const SharedPtr<Peer> &dest,Demarc::Port localPort,const InetAddress &remoteAddr)
 {
 	uint64_t now = Utils::now();
-
 	Packet outp(dest->address(),_r->identity.address(),Packet::VERB_HELLO);
 	outp.append((unsigned char)ZT_PROTO_VERSION);
 	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
@@ -214,8 +212,7 @@ bool Switch::sendHELLO(const SharedPtr<Peer> &dest,Demarc::Port localPort,const
 	outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
 	outp.append(now);
 	_r->identity.serialize(outp,false);
-	outp.macSet(dest->macKey());
-
+	outp.armor(dest->key(),false);
 	return _r->demarc->send(localPort,remoteAddr,outp.data(),outp.size(),-1);
 }
 
@@ -268,8 +265,7 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 			outp.append((unsigned char)4);
 			outp.append(cg.first.rawIpData(),4);
 		}
-		outp.encrypt(p1p->cryptKey());
-		outp.macSet(p1p->macKey());
+		outp.armor(p1p->key(),true);
 		p1p->send(_r,outp.data(),outp.size(),now);
 	}
 	{	// tell p2 where to find p1
@@ -283,8 +279,7 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 			outp.append((unsigned char)4);
 			outp.append(cg.second.rawIpData(),4);
 		}
-		outp.encrypt(p2p->cryptKey());
-		outp.macSet(p2p->macKey());
+		outp.armor(p2p->key(),true);
 		p2p->send(_r,outp.data(),outp.size(),now);
 	}
 
@@ -606,9 +601,7 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread
 	if (supernode) {
 		Packet outp(supernode->address(),_r->identity.address(),Packet::VERB_WHOIS);
 		addr.appendTo(outp);
-		outp.encrypt(supernode->cryptKey());
-		outp.macSet(supernode->macKey());
-
+		outp.armor(supernode->key(),true);
 		uint64_t now = Utils::now();
 		if (supernode->send(_r,outp.data(),outp.size(),now))
 			return supernode->address();
@@ -623,13 +616,10 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 	if (peer) {
 		uint64_t now = Utils::now();
 
-		bool isRelay;
 		SharedPtr<Peer> via;
 		if ((_r->topology->isSupernode(peer->address()))||(peer->hasActiveDirectPath(now))) {
-			isRelay = false;
 			via = peer;
 		} else {
-			isRelay = true;
 			via = _r->topology->getBestSupernode();
 			if (!via)
 				return false;
@@ -640,9 +630,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 		unsigned int chunkSize = std::min(tmp.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU);
 		tmp.setFragmented(chunkSize < tmp.size());
 
-		if (encrypt)
-			tmp.encrypt(peer->cryptKey());
-		tmp.macSet(peer->macKey());
+		tmp.armor(peer->key(),encrypt);
 
 		if (via->send(_r,tmp.data(),chunkSize,now)) {
 			if (chunkSize < tmp.size()) {

node/Switch.hpp

@@ -197,6 +197,7 @@ private:
 		bool encrypt);
 
 	const RuntimeEnvironment *const _r;
+	volatile unsigned int _multicastIdCounter;
 
 	struct WhoisRequest
 	{