File transfer work, add identities for validation of updates.

node/Defaults.cpp

@@ -98,12 +98,37 @@ static inline std::string _mkDefaultHomePath()
 #endif
 }
 
+static inline std::map< Address,Identity > _mkUpdateAuth()
+{
+	std::map< Address,Identity > ua;
+
+	{ // 0001
+		Identity id("e9bc3707b5:0:c4cef17bde99eadf9748c4fd11b9b06dc5cd8eb429227811d2c336e6b96a8d329e8abd0a4f45e47fe1bcebf878c004c822d952ff77fc2833af4c74e65985c435");
+		ua[id.address()] = id;
+	}
+	{ // 0002
+		Identity id("56520eaf93:0:7d858b47988b34399a9a31136de07b46104d7edb4a98fa1d6da3e583d3a33e48be531532b886f0b12cd16794a66ab9220749ec5112cbe96296b18fe0cc79ca05");
+		ua[id.address()] = id;
+	}
+	{ // 0003
+		Identity id("7c195de2e0:0:9f659071c960f9b0f0b96f9f9ecdaa27c7295feed9c79b7db6eedcc11feb705e6dd85c70fa21655204d24c897865b99eb946b753a2bbcf2be5f5e006ae618c54");
+		ua[id.address()] = id;
+	}
+	{ // 0004
+		Identity id("415f4cfde7:0:54118e87777b0ea5d922c10b337c4f4bd1db7141845bd54004b3255551a6e356ba6b9e1e85357dbfafc45630b8faa2ebf992f31479e9005f0472685f2d8cbd6e");
+		ua[id.address()] = id;
+	}
+
+	return ua;
+}
+
 Defaults::Defaults() :
 #ifdef ZT_TRACE_MULTICAST
 	multicastTraceWatcher(ZT_TRACE_MULTICAST),
 #endif
 	defaultHomePath(_mkDefaultHomePath()),
-	supernodes(_mkSupernodeMap())
+	supernodes(_mkSupernodeMap()),
+	updateAuthorities(_mkUpdateAuth())
 {
 }
 

node/Defaults.hpp

@@ -70,6 +70,12 @@ public:
 
 	/**
 	 * Identities permitted to sign software updates
+	 *
+	 * ZTN can keep multiple signing identities and rotate them, keeping some in
+	 * "cold storage" and obsoleting others gradually.
+	 *
+	 * If you don't build with ZT_OFFICIAL_BUILD, this isn't used since your
+	 * build will not auto-update.
 	 */
 	const std::map< Address,Identity > updateAuthorities;
 };

node/Packet.hpp

@@ -619,12 +619,12 @@ public:
 
 		/* Request information about a shared file (for software updates):
 		 *   <[1] flags, currently unused and must be 0>
-		 *   <[2] 16-bit length of filename>
+		 *   <[1] 8-bit length of filename>
 		 *   <[...] name of file being requested>
 		 *
 		 * OK response payload (indicates that we have and will share):
 		 *   <[1] flags, currently unused and must be 0>
-		 *   <[2] 16-bit length of filename>
+		 *   <[1] 8-bit length of filename>
 		 *   <[...] name of file being requested>
 		 *   <[64] full length SHA-512 hash of file contents>
 		 *   <[4] 32-bit length of file in bytes>
@@ -636,6 +636,10 @@ public:
 		 *   <[2] 16-bit length of filename>
 		 *   <[...] name of file being requested>
 		 *
+		 * This is used for distribution of software updates and in the future may
+		 * be used for anything else that needs to be globally distributed. It
+		 * is not designed for end-user use for other purposes.
+		 *
 		 * Support is optional. Nodes should return UNSUPPORTED_OPERATION if
 		 * not supported or enabled.
 		 */
@@ -657,6 +661,10 @@ public:
 		 *   <[4] 32-bit index of desired chunk>
 		 *   <[2] 16-bit length of desired chunk>
 		 *
+		 * This is used for distribution of software updates and in the future may
+		 * be used for anything else that needs to be globally distributed. It
+		 * is not designed for end-user use for other purposes.
+		 *
 		 * Support is optional. Nodes should return UNSUPPORTED_OPERATION if
 		 * not supported or enabled.
 		 */

node/Updater.cpp

@@ -31,6 +31,8 @@
 #include "Defaults.hpp"
 #include "Utils.hpp"
 #include "Topology.hpp"
+#include "Switch.hpp"
+#include "SHA512.hpp"
 
 #include "../version.h"
 
@@ -69,8 +71,9 @@ void Updater::refreshShared()
 		if (Utils::readFile(nfoPath.c_str(),buf)) {
 			Dictionary nfo(buf);
 
-			_Shared shared;
-			shared.filename = fullPath;
+			SharedUpdate shared;
+			shared.fullPath = fullPath;
+			shared.filename = u->first;
 
 			std::string sha512(Utils::unhex(nfo.get("sha512",std::string())));
 			if (sha512.length() < sizeof(shared.sha512)) {
@@ -104,9 +107,7 @@ void Updater::refreshShared()
 			}
 			shared.size = (unsigned long)fs;
 
-			Array<unsigned char,16> first16Bytes;
-			memcpy(first16Bytes.data,sha512.data(),16);
-			_sharedUpdates[first16Bytes] = shared;
+			_sharedUpdates.push_back(shared);
 		} else {
 			TRACE("skipped shareable update due to missing companion .nfo: %s",fullPath.c_str());
 			continue;
@@ -127,9 +128,9 @@ void Updater::getUpdateIfThisIsNewer(unsigned int vMajor,unsigned int vMinor,uns
 		}
 	}
 
-	std::string updateFilename(generateUpdateFilename());
+	std::string updateFilename(generateUpdateFilename(vMajor,vMinor,revision));
 	if (!updateFilename.length()) {
-		TRACE("a new update to %u.%u.%u is available, but this platform doesn't support auto updates",vMajor,vMinor,revision);
+		TRACE("an update to %u.%u.%u is available, but this platform or build doesn't support auto-update",vMajor,vMinor,revision);
 		return;
 	}
 
@@ -138,11 +139,8 @@ void Updater::getUpdateIfThisIsNewer(unsigned int vMajor,unsigned int vMinor,uns
 
 	TRACE("new update available to %u.%u.%u, looking for %s from %u peers",vMajor,vMinor,revision,updateFilename.c_str(),(unsigned int)peers.size());
 
-	if (!peers.size())
-		return;
-
 	for(std::vector< SharedPtr<Peer> >::iterator p(peers.begin());p!=peers.end();++p) {
-		Packet outp(p->address(),_r->identity.address(),Packet::VERB_FILE_INFO_REQUEST);
+		Packet outp((*p)->address(),_r->identity.address(),Packet::VERB_FILE_INFO_REQUEST);
 		outp.append((unsigned char)0);
 		outp.append((uint16_t)updateFilename.length());
 		outp.append(updateFilename.data(),updateFilename.length());
@@ -152,14 +150,167 @@ void Updater::getUpdateIfThisIsNewer(unsigned int vMajor,unsigned int vMinor,uns
 
 void Updater::retryIfNeeded()
 {
+	Mutex::Lock _l(_lock);
+
+	if (_download) {
+		uint64_t elapsed = Utils::now() - _download->lastChunkReceivedAt;
+		if ((elapsed >= ZT_UPDATER_PEER_TIMEOUT)||(!_download->currentlyReceivingFrom)) {
+			if (_download->peersThatHave.empty()) {
+				// Search for more sources if we have no more possibilities queued
+				_download->currentlyReceivingFrom.zero();
+
+				std::vector< SharedPtr<Peer> > peers;
+				_r->topology->eachPeer(Topology::CollectPeersWithActiveDirectPath(peers,Utils::now()));
+
+				for(std::vector< SharedPtr<Peer> >::iterator p(peers.begin());p!=peers.end();++p) {
+					Packet outp((*p)->address(),_r->identity.address(),Packet::VERB_FILE_INFO_REQUEST);
+					outp.append((unsigned char)0);
+					outp.append((uint16_t)_download->filename.length());
+					outp.append(_download->filename.data(),_download->filename.length());
+					_r->sw->send(outp,true);
+				}
+			} else {
+				// If that peer isn't answering, try the next queued source
+				_download->currentlyReceivingFrom = _download->peersThatHave.front();
+				_download->peersThatHave.pop_front();
+			}
+		} else if (elapsed >= ZT_UPDATER_RETRY_TIMEOUT) {
+			// Re-request next chunk we don't have from current source
+			_requestNextChunk();
+		}
+	}
 }
 
-void Updater::handleChunk(const void *sha512First16,unsigned long at,const void *chunk,unsigned long len)
+void Updater::handleChunk(const Address &from,const void *sha512,unsigned int shalen,unsigned long at,const void *chunk,unsigned long len)
 {
+	Mutex::Lock _l(_lock);
+
+	if (!_download) {
+		TRACE("got chunk from %s while no download is in progress, ignored",from.toString().c_str());
+		return;
+	}
+
+	if (memcmp(_download->sha512,sha512,(shalen > 64) ? 64 : shalen)) {
+		TRACE("got chunk from %s for wrong download (SHA mismatch), ignored",from.toString().c_str());
+		return;
+	}
+
+	unsigned long whichChunk = at / ZT_UPDATER_CHUNK_SIZE;
+
+	if (at != (ZT_UPDATER_CHUNK_SIZE * whichChunk))
+		return; // not at chunk boundary
+	if (whichChunk >= _download->haveChunks.size())
+		return; // overflow
+	if ((whichChunk == (_download->haveChunks.size() - 1))&&(len != _download->lastChunkSize))
+		return; // last chunk, size wrong
+	else if (len != ZT_UPDATER_CHUNK_SIZE)
+		return; // chunk size wrong
+
+	for(unsigned long i=0;i<len;++i)
+		_download->data[at + i] = ((const char *)chunk)[i];
+
+	_download->haveChunks[whichChunk] = true;
+	_download->lastChunkReceivedAt = Utils::now();
+
+	_requestNextChunk();
+}
+
+void Updater::handleAvailable(const Address &from,const char *filename,const void *sha512,unsigned long filesize,const Address &signedBy,const void *signature,unsigned int siglen)
+{
+	unsigned int vMajor = 0,vMinor = 0,revision = 0;
+	if (!parseUpdateFilename(filename,vMajor,vMinor,revision)) {
+		TRACE("rejected offer of %s from %s: could not parse version information",filename,from.toString().c_str());
+		return;
+	}
+
+	if (filesize > ZT_UPDATER_MAX_SUPPORTED_SIZE) {
+		TRACE("rejected offer of %s from %s: file too large (%u)",filename,from.toString().c_str(),(unsigned int)filesize);
+		return;
+	}
+
+	if (vMajor < ZEROTIER_ONE_VERSION_MAJOR)
+		return;
+	else if (vMajor == ZEROTIER_ONE_VERSION_MAJOR) {
+		if (vMinor < ZEROTIER_ONE_VERSION_MINOR)
+			return;
+		else if (vMinor == ZEROTIER_ONE_VERSION_MINOR) {
+			if (revision <= ZEROTIER_ONE_VERSION_REVISION)
+				return;
+		}
+	}
+
+	Mutex::Lock _l(_lock);
+
+	if (_download) {
+		// If a download is in progress, only accept this as another source if
+		// it matches the size, hash, and version. Also check if this is a newer
+		// version and if so replace download with this.
+	} else {
+		// If there is no download in progress, create one provided the signature
+		// for the SHA-512 hash verifies as being from a valid signer.
+	}
+}
+
+bool Updater::findSharedUpdate(const char *filename,SharedUpdate &update) const
+{
+	Mutex::Lock _l(_lock);
+	for(std::list<SharedUpdate>::const_iterator u(_sharedUpdates.begin());u!=_sharedUpdates.end();++u) {
+		if (u->filename == filename) {
+			update = *u;
+			return true;
+		}
+	}
+	return false;
+}
+
+bool Updater::findSharedUpdate(const void *sha512,unsigned int shalen,SharedUpdate &update) const
+{
+	if (!shalen)
+		return false;
+	Mutex::Lock _l(_lock);
+	for(std::list<SharedUpdate>::const_iterator u(_sharedUpdates.begin());u!=_sharedUpdates.end();++u) {
+		if (!memcmp(u->sha512,sha512,(shalen > 64) ? 64 : shalen)) {
+			update = *u;
+			return true;
+		}
+	}
+	return false;
+}
+
+bool Updater::getSharedChunk(const void *sha512,unsigned int shalen,unsigned long at,void *chunk,unsigned long chunklen) const
+{
+	if (!chunklen)
+		return true;
+	if (!shalen)
+		return false;
+	Mutex::Lock _l(_lock);
+	for(std::list<SharedUpdate>::const_iterator u(_sharedUpdates.begin());u!=_sharedUpdates.end();++u) {
+		if (!memcmp(u->sha512,sha512,(shalen > 64) ? 64 : shalen)) {
+			FILE *f = fopen(u->fullPath.c_str(),"rb");
+			if (!f)
+				return false;
+			if (!fseek(f,(long)at,SEEK_SET)) {
+				fclose(f);
+				return false;
+			}
+			if (fread(chunk,chunklen,1,f) != 1) {
+				fclose(f);
+				return false;
+			}
+			fclose(f);
+			return true;
+		}
+	}
+	return false;
 }
 
 std::string Updater::generateUpdateFilename(unsigned int vMajor,unsigned int vMinor,unsigned int revision)
 {
+	// Defining ZT_OFFICIAL_BUILD enables this cascade of macros, which will
+	// make your build auto-update itself if it's for an officially supported
+	// architecture. The signing identity for auto-updates is in Defaults.
+#ifdef ZT_OFFICIAL_BUILD
+
 	// Not supported... yet? Get it first cause it might identify as Linux too.
 #ifdef __ANDROID__
 #define _updSupported 1
@@ -202,6 +353,10 @@ std::string Updater::generateUpdateFilename(unsigned int vMajor,unsigned int vMi
 #ifndef _updSupported
 	return std::string();
 #endif
+
+#else
+	return std::string();
+#endif // ZT_OFFICIAL_BUILD
 }
 
 bool Updater::parseUpdateFilename(const char *filename,unsigned int &vMajor,unsigned int &vMinor,unsigned int &revision)
@@ -218,5 +373,42 @@ bool Updater::parseUpdateFilename(const char *filename,unsigned int &vMajor,unsi
 	return true;
 }
 
+void Updater::_requestNextChunk()
+{
+	// assumes _lock is locked
+
+	if (!_download)
+		return;
+
+	unsigned long whichChunk = 0;
+	std::vector<bool>::iterator ptr(std::find(_download->haveChunks.begin(),_download->haveChunks.end(),false));
+	if (ptr == _download->haveChunks.end()) {
+		unsigned char digest[64];
+		SHA512::hash(digest,_download->data.data(),_download->data.length());
+		if (memcmp(digest,_download->sha512,64)) {
+			LOG("retrying download of %s -- SHA-512 mismatch, file corrupt!",_download->filename.c_str());
+			std::fill(_download->haveChunks.begin(),_download->haveChunks.end(),false);
+			whichChunk = 0;
+		} else {
+			LOG("successfully downloaded and authenticated %s, launching update...",_download->filename.c_str());
+			delete _download;
+			_download = (_Download *)0;
+			return;
+		}
+	} else {
+		whichChunk = std::distance(_download->haveChunks.begin(),ptr);
+	}
+
+	TRACE("requesting chunk %u/%u of %s from %s",(unsigned int)whichChunk,(unsigned int)_download->haveChunks.size(),_download->filename.c_str()_download->currentlyReceivingFrom.toString().c_str());
+
+	Packet outp(_download->currentlyReceivingFrom,_r->identity.address(),Packet::VERB_FILE_BLOCK_REQUEST);
+	outp.append(_download->sha512,16);
+	outp.append((uint32_t)(whichChunk * ZT_UPDATER_CHUNK_SIZE));
+	if (whichChunk == (_download->haveChunks.size() - 1))
+		outp.append((uint16_t)_download->lastChunkSize);
+	else outp.append((uint16_t)ZT_UPDATER_CHUNK_SIZE);
+	_r->sw->send(outp,true);
+}
+
 } // namespace ZeroTier
 

node/Updater.hpp

@@ -38,6 +38,7 @@
 #include <iterator>
 #include <stdexcept>
 #include <string>
+#include <list>
 
 #include "Constants.hpp"
 #include "Packet.hpp"
@@ -55,10 +56,10 @@
 #define ZT_UPDATER_MAX_SUPPORTED_SIZE (1024 * 1024 * 16)
 
 // Retry timeout in ms.
-#define ZT_UPDATER_RETRY_TIMEOUT 30000
+#define ZT_UPDATER_RETRY_TIMEOUT 15000
 
-// After this long, look for a new set of peers that have the download shared.
-#define ZT_UPDATER_REPOLL_TIMEOUT 60000
+// After this long, look for a new peer to download from
+#define ZT_UPDATER_PEER_TIMEOUT 65000
 
 namespace ZeroTier {
 
@@ -67,10 +68,12 @@ class RuntimeEnvironment;
 /**
  * Software update downloader and executer
  *
- * FYI: downloads occur via the protocol rather than out of band via http so
+ * Downloads occur via the ZT1 protocol rather than out of band via http so
  * that ZeroTier One can be run in secure jailed environments where it is the
- * only protocol permitted over the "real" Internet. This is required for a
- * number of potentially popular use cases.
+ * only protocol permitted over the "real" Internet. This is wanted for a
+ * number of potentially popular use cases, like private LANs that connect
+ * nodes in hostile environments or playing attack/defend on the future CTF
+ * network.
  *
  * The protocol is a simple chunk-pulling "trivial FTP" like thing that should
  * be suitable for core engine software updates. Software updates themselves
@@ -84,6 +87,19 @@ class RuntimeEnvironment;
 class Updater
 {
 public:
+	/**
+	 * Contains information about a shared update available to other peers
+	 */
+	struct SharedUpdate
+	{
+		std::string fullPath;
+		std::string filename;
+		unsigned char sha512[64];
+		C25519::Signature sig;
+		Address signedBy;
+		unsigned long size;
+	};
+
 	Updater(const RuntimeEnvironment *renv);
 	~Updater();
 
@@ -108,18 +124,72 @@ public:
 
 	/**
 	 * Called periodically from main loop
+	 *
+	 * This retries downloads if they're stalled and performs other cleanup.
 	 */
 	void retryIfNeeded();
 
 	/**
 	 * Called when a chunk is received
 	 *
-	 * @param sha512First16 First 16 bytes of SHA-512 hash
+	 * If the chunk is a final chunk and we now have an update, this may result
+	 * in the commencement of the update process and the shutdown of ZT1.
+	 *
+	 * @param from Originating peer
+	 * @param sha512 Up to 64 bytes of hash to match
+	 * @param shalen Length of sha512[]
 	 * @param at Position of chunk
 	 * @param chunk Chunk data
 	 * @param len Length of chunk
 	 */
-	void handleChunk(const void *sha512First16,unsigned long at,const void *chunk,unsigned long len);
+	void handleChunk(const Address &from,const void *sha512,unsigned int shalen,unsigned long at,const void *chunk,unsigned long len);
+
+	/**
+	 * Called when a reply to a search for an update is received
+	 *
+	 * This checks SHA-512 hash signature and version as parsed from filename
+	 * before starting the transfer.
+	 *
+	 * @param from Node that sent reply saying it has the file
+	 * @param filename Name of file (can be parsed for version info)
+	 * @param sha512 64-byte SHA-512 hash of file's contents
+	 * @param filesize Size of file in bytes
+	 * @param signedBy Address of signer of hash
+	 * @param signature Signature (currently must be Ed25519)
+	 * @param siglen Length of signature in bytes
+	 */
+	void handleAvailable(const Address &from,const char *filename,const void *sha512,unsigned long filesize,const Address &signedBy,const void *signature,unsigned int siglen);
+
+	/**
+	 * Get data about a shared update if found
+	 *
+	 * @param filename File name
+	 * @param update Empty structure to be filled with update info
+	 * @return True if found (if false, 'update' is unmodified)
+	 */
+	bool findSharedUpdate(const char *filename,SharedUpdate &update) const;
+
+	/**
+	 * Get data about a shared update if found
+	 *
+	 * @param sha512 Up to 64 bytes of hash to match
+	 * @param shalen Length of sha512[]
+	 * @param update Empty structure to be filled with update info
+	 * @return True if found (if false, 'update' is unmodified)
+	 */
+	bool findSharedUpdate(const void *sha512,unsigned int shalen,SharedUpdate &update) const;
+
+	/**
+	 * Get a chunk of a shared update
+	 *
+	 * @param sha512 Up to 64 bytes of hash to match
+	 * @param shalen Length of sha512[]
+	 * @param at Position in file
+	 * @param chunk Buffer to store data
+	 * @param chunklen Number of bytes to get
+	 * @return True if chunk[] was successfully filled, false if not found or other error
+	 */
+	bool getSharedChunk(const void *sha512,unsigned int shalen,unsigned long at,void *chunk,unsigned long chunklen) const;
 
 	/**
 	 * @return Canonical update filename for this platform or empty string if unsupported
@@ -135,48 +205,13 @@ public:
 	static bool parseUpdateFilename(const char *filename,unsigned int &vMajor,unsigned int &vMinor,unsigned int &revision);
 
 private:
+	void _requestNextChunk();
+
 	struct _Download
 	{
-		_Download(const void *s512,const std::string &fn,unsigned long len,unsigned int vMajor,unsigned int vMinor,unsigned int rev)
-		{
-			data.resize(len);
-			haveChunks.resize((len / ZT_UPDATER_CHUNK_SIZE) + 1,false);
-			filename = fn;
-			memcpy(sha512,s512,64);
-			lastChunkSize = len % ZT_UPDATER_CHUNK_SIZE;
-			versionMajor = vMajor;
-			versionMinor = vMinor;
-			revision = rev;
-		}
-
-		long nextChunk() const
-		{
-			std::vector<bool>::const_iterator ptr(std::find(haveChunks.begin(),haveChunks.end(),false));
-			if (ptr != haveChunks.end())
-				return std::distance(haveChunks.begin(),ptr);
-			else return -1;
-		}
-
-		bool gotChunk(unsigned long at,const void *chunk,unsigned long len)
-		{
-			unsigned long whichChunk = at / ZT_UPDATER_CHUNK_SIZE;
-			if (at != (ZT_UPDATER_CHUNK_SIZE * whichChunk))
-				return false; // not at chunk boundary
-			if (whichChunk >= haveChunks.size())
-				return false; // overflow
-			if ((whichChunk == (haveChunks.size() - 1))&&(len != lastChunkSize))
-				return false; // last chunk, size wrong
-			else if (len != ZT_UPDATER_CHUNK_SIZE)
-				return false; // chunk size wrong
-			for(unsigned long i=0;i<len;++i)
-				data[at + i] = ((const char *)chunk)[i];
-			haveChunks[whichChunk] = true;
-			return true;
-		}
-
 		std::string data;
 		std::vector<bool> haveChunks;
-		std::vector<Address> peersThatHave;
+		std::list<Address> peersThatHave; // excluding current
 		std::string filename;
 		unsigned char sha512[64];
 		Address currentlyReceivingFrom;
@@ -185,18 +220,9 @@ private:
 		unsigned int versionMajor,versionMinor,revision;
 	};
 
-	struct _Shared
-	{
-		std::string filename;
-		unsigned char sha512[64];
-		C25519::Signature sig;
-		Address signedBy;
-		unsigned long size;
-	};
-
 	const RuntimeEnvironment *_r;
 	_Download *_download;
-	std::map< Array<unsigned char,16>,_Shared > _sharedUpdates;
+	std::list<SharedUpdate> _sharedUpdates; // usually not more than 1 or 2 of these
 	Mutex _lock;
 };
 

node/Utils.cpp

@@ -265,7 +265,7 @@ uint64_t Utils::getLastModified(const char *path)
 	return (((uint64_t)s.st_mtime) * 1000ULL);
 }
 
-static int64_t getFileSize(const char *path)
+int64_t Utils::getFileSize(const char *path)
 {
 	struct stat s;
 	if (stat(path,&s))