More clustering work.

include/ZeroTierOne.h

@@ -271,22 +271,27 @@ enum ZT_ResultCode
 	 */
 	ZT_RESULT_OK = 0,
 
-	// Fatal errors (>0, <1000)
+	/**
+	 * Call produced no error but no action was taken
+	 */
+	ZT_RESULT_OK_IGNORED = 1,
+
+	// Fatal errors (>100, <1000)
 
 	/**
 	 * Ran out of memory
 	 */
-	ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY = 1,
+	ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY = 100,
 
 	/**
 	 * Data store is not writable or has failed
 	 */
-	ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED = 2,
+	ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED = 101,
 
 	/**
 	 * Internal error (e.g. unexpected exception indicating bug or build problem)
 	 */
-	ZT_RESULT_FATAL_ERROR_INTERNAL = 3,
+	ZT_RESULT_FATAL_ERROR_INTERNAL = 102,
 
 	// Non-fatal errors (>1000)
 
@@ -1090,59 +1095,97 @@ typedef struct
 } ZT_PeerList;
 
 /**
- * Types of stored objects that the core may wish to save or load
+ * ZeroTier core state objects
+ *
+ * All of these objects can be persisted if desired. To preserve the
+ * identity of a node and its address, the identity (public and secret)
+ * must be saved at a minimum.
+ *
+ * The reference service implementation currently persists identity,
+ * peer identities (for a period of time), planet, moons, and network
+ * configurations. Other state is treated as ephemeral.
+ *
+ * All state objects should be replicated in cluster mode. The reference
+ * clustering implementation uses a rumor mill algorithm in which state
+ * updates that are accepted with RESULT_OK (but not RESULT_OK_IGNORED)
+ * are flooded to all connected cluster peers. This results in updates
+ * being flooded across the cluster until all cluster members have the
+ * latest.
  */
-enum ZT_StoredObjectType
+enum ZT_StateObjectType
 {
 	/**
-	 * Node status information (reserved, not currently used)
-	 */
-	ZT_STORED_OBJECT_STATUS = 0,
-
-	/**
-	 * String serialized public identity
+	 * Null object -- ignored
 	 */
-	ZT_STORED_OBJECT_IDENTITY_PUBLIC = 1,
+	ZT_STATE_OBJECT_NULL = 0,
 
 	/**
-	 * String serialized secret identity
+	 * identity.public
+	 *
+	 * Object ID: this node's address if known, or 0 if unknown (first query)
+	 * Canonical path: <HOME>/identity.public
+   * Persistence: required
 	 */
-	ZT_STORED_OBJECT_IDENTITY_SECRET = 1,
+	ZT_STATE_OBJECT_IDENTITY_PUBLIC = 1,
 
 	/**
-	 * Binary serialized peer state
+	 * identity.secret
+	 *
+	 * Object ID: this node's address if known, or 0 if unknown (first query)
+	 * Canonical path: <HOME>/identity.public
+   * Persistence: required, should be stored with restricted permissions e.g. mode 0600 on *nix
 	 */
-	ZT_STORED_OBJECT_PEER = 3,
+	ZT_STATE_OBJECT_IDENTITY_SECRET = 2,
 
 	/**
-	 * Identity (other node, not this one)
+	 * A peer to which this node is communicating
+	 *
+	 * Object ID: peer address
+	 * Canonical path: <HOME>/peers.d/<ADDRESS> (10-digit hex address)
+	 * Persistence: optional, can be purged at any time
 	 */
-	ZT_STORED_OBJECT_IDENTITY = 4,
+	ZT_STATE_OBJECT_PEER = 3,
 
 	/**
-	 * Network configuration object
+	 * The identity of a known peer
+	 *
+	 * Object ID: peer address
+	 * Canonical path: <HOME>/iddb.d/<ADDRESS> (10-digit hex address)
+	 * Persistence: optional, can be purged at any time, recommended ttl 30-60 days
 	 */
-	ZT_STORED_OBJECT_NETWORK_CONFIG = 5,
+	ZT_STATE_OBJECT_PEER_IDENTITY = 4,
 
 	/**
-	 * Planet definition (object ID will be zero and should be ignored since there's only one)
+	 * Network configuration
+	 *
+	 * Object ID: peer address
+	 * Canonical path: <HOME>/networks.d/<NETWORKID>.conf (16-digit hex ID)
+	 * Persistence: required if network memberships should persist
 	 */
-	ZT_STORED_OBJECT_PLANET = 6,
+	ZT_STATE_OBJECT_NETWORK_CONFIG = 5,
 
 	/**
-	 * Moon definition
+	 * The planet (there is only one per... well... planet!)
+	 *
+	 * Object ID: world ID of planet, or 0 if unknown (first query)
+	 * Canonical path: <HOME>/planet
+	 * Persistence: recommended
 	 */
-	ZT_STORED_OBJECT_MOON = 7,
+	ZT_STATE_OBJECT_PLANET = 6,
 
 	/**
-	 * Multicast membership
+	 * A moon (federated root set)
+	 *
+	 * Object ID: world ID of moon
+	 * Canonical path: <HOME>/moons.d/<ID>.moon (16-digit hex ID)
+	 * Persistence: required if moon memberships should persist
 	 */
-	ZT_STORED_OBJECT_MULTICAST_MEMBERSHIP = 8,
+	ZT_STATE_OBJECT_MOON = 7,
 
 	/**
-	 * IDs above this are never used by the core and are available for implementation use
+	 * IDs above this value will not be used by the core (and could be used as implementation-specific IDs)
 	 */
-	ZT_STORED_OBJECT__MAX_TYPE_ID = 255
+	ZT_STATE_OBJECT__MAX_ID = 255
 };
 
 /**
@@ -1221,59 +1264,38 @@ typedef void (*ZT_EventCallback)(
 	const void *);                         /* Event payload (if applicable) */
 
 /**
- * Function to get an object from the data store
- *
- * Parameters: (1) object name, (2) buffer to fill, (3) size of buffer, (4)
- * index in object to start reading, (5) result parameter that must be set
- * to the actual size of the object if it exists.
- *
- * Object names can contain forward slash (/) path separators. They will
- * never contain .. or backslash (\), so this is safe to map as a Unix-style
- * path if the underlying storage permits. For security reasons we recommend
- * returning errors if .. or \ are used.
+ * Callback for storing and/or publishing state information
  *
- * The function must return the actual number of bytes read. If the object
- * doesn't exist, it should return -1. -2 should be returned on other errors
- * such as errors accessing underlying storage.
+ * See ZT_StateObjectType docs for information about each state object type
+ * and when and if it needs to be persisted.
  *
- * If the read doesn't fit in the buffer, the max number of bytes should be
- * read. The caller may call the function multiple times to read the whole
- * object.
+ * An object of length -1 is sent to indicate that an object should be
+ * deleted.
  */
-typedef long (*ZT_DataStoreGetFunction)(
+typedef void (*ZT_StatePutFunction)(
 	ZT_Node *,                             /* Node */
 	void *,                                /* User ptr */
 	void *,                                /* Thread ptr */
-	const char *,
-	void *,
-	unsigned long,
-	unsigned long,
-	unsigned long *);
+	enum ZT_StateObjectType,               /* State object type */
+	uint64_t,                              /* State object ID (if applicable) */
+	const void *,                          /* State object data */
+	int);                                  /* Length of data or -1 to delete */
 
 /**
- * Function to store an object in the data store
+ * Callback for retrieving stored state information
  *
- * Parameters: (1) node, (2) user ptr, (3) object name, (4) object data,
- * (5) object size, (6) secure? (bool).
- *
- * If secure is true, the file should be set readable and writable only
- * to the user running ZeroTier One. What this means is platform-specific.
- *
- * Name semantics are the same as the get function. This must return zero on
- * success. You can return any OS-specific error code on failure, as these
- * may be visible in logs or error messages and might aid in debugging.
- *
- * If the data pointer is null, this must be interpreted as a delete
- * operation.
+ * This function should return the number of bytes actually stored to the
+ * buffer or -1 if the state object was not found or the buffer was too
+ * small to store it.
  */
-typedef int (*ZT_DataStorePutFunction)(
-	ZT_Node *,
-	void *,
+typedef int (*ZT_StateGetFunction)(
+	ZT_Node *,                             /* Node */
+	void *,                                /* User ptr */
 	void *,                                /* Thread ptr */
-	const char *,
-	const void *,
-	unsigned long,
-	int);
+	enum ZT_StateObjectType,               /* State object type */
+	uint64_t,                              /* State object ID (if applicable) */
+	void *,                                /* Buffer to store state object data */
+	unsigned int);                         /* Length of data buffer in bytes */
 
 /**
  * Function to send a ZeroTier packet out over the wire
@@ -1381,14 +1403,14 @@ struct ZT_Node_Callbacks
 	long version;
 
 	/**
-	 * REQUIRED: Function to get objects from persistent storage
+	 * REQUIRED: Function to store and/or replicate state objects
 	 */
-	ZT_DataStoreGetFunction dataStoreGetFunction;
+	ZT_StatePutFunction statePutFunction;
 
 	/**
-	 * REQUIRED: Function to store objects in persistent storage
+	 * REQUIRED: Function to retrieve state objects from an object store
 	 */
-	ZT_DataStorePutFunction dataStorePutFunction;
+	ZT_StateGetFunction stateGetFunction;
 
 	/**
 	 * REQUIRED: Function to send packets over the physical wire
@@ -1449,6 +1471,49 @@ enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct
  */
 void ZT_Node_delete(ZT_Node *node);
 
+/**
+ * Notify node of an update to a state object
+ *
+ * This can be called after node startup to restore cached state objects such
+ * as network configurations for joined networks, planet, moons, etc. See
+ * the documentation of ZT_StateObjectType for more information. It's okay
+ * to call this for everything in the object store, but note that the node
+ * will automatically query for some core objects like identities so supplying
+ * these via this function is not necessary.
+ *
+ * Unless clustering is being implemented this function doesn't need to be
+ * used after startup. It could be called in response to filesystem changes
+ * to allow some degree of live configurability by filesystem observation.
+ *
+ * The return value of this function indicates whether the update was accepted
+ * as new. A return value of ZT_RESULT_OK indicates that the node gleaned new
+ * information from this update and that therefore (in cluster rumor mill mode)
+ * this update should be distributed to other members of a cluster. A return
+ * value of ZT_RESULT_OK_IGNORED indicates that the object did not provide any
+ * new information and therefore should not be propagated in a cluster.
+ *
+ * If clustering isn't being implemented the return value of this function can
+ * generally be ignored.
+ *
+ * ZT_RESULT_ERROR_BAD_PARAMETER can be returned if the parameter was invalid
+ * or not applicable. Object stores may delete the object in this case.
+ *
+ * @param node Node instance
+ * @param tptr Thread pointer to pass to functions/callbacks resulting from this call
+ * @param type State object type
+ * @param id State object ID
+ * @param data State object data
+ * @param len Length of state object data in bytes
+ * @return ZT_RESULT_OK if object was accepted or ZT_RESULT_OK_IGNORED if non-informative, error if object was invalid
+ */
+enum ZT_ResultCode ZT_Node_processStateUpdate(
+	ZT_Node *node,
+	void *tptr,
+	ZT_StateObjectType type,
+	uint64_t id,
+	const void *data,
+	unsigned int len);
+
 /**
  * Process a packet received from the physical wire
  *

node/Network.cpp

@@ -682,7 +682,7 @@ static _doZtFilterResult _doZtFilter(
 
 const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0);
 
-Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr) :
+Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr,const NetworkConfig *nconf) :
 	RR(renv),
 	_uPtr(uptr),
 	_id(nwid),
@@ -697,29 +697,11 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
 	for(int i=0;i<ZT_NETWORK_MAX_INCOMING_UPDATES;++i)
 		_incomingConfigChunks[i].ts = 0;
 
-	char confn[128];
-	Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id);
-
-	bool gotConf = false;
-	Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *dconf = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
-	NetworkConfig *nconf = new NetworkConfig();
-	try {
-		std::string conf(RR->node->dataStoreGet(tPtr,confn));
-		if (conf.length()) {
-			dconf->load(conf.c_str());
-			if (nconf->fromDictionary(*dconf)) {
-				this->setConfiguration(tPtr,*nconf,false);
-				_lastConfigUpdate = 0; // we still want to re-request a new config from the network
-				gotConf = true;
-			}
-		}
-	} catch ( ... ) {} // ignore invalids, we'll re-request
-	delete nconf;
-	delete dconf;
-
-	if (!gotConf) {
-		// Save a one-byte CR to persist membership while we request a real netconf
-		RR->node->dataStorePut(tPtr,confn,"\n",1,false);
+	if (nconf) {
+		this->setConfiguration(tPtr,*nconf,false);
+		_lastConfigUpdate = 0; // still want to re-request since it's likely outdated
+	} else {
+		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,"\n",1);
 	}
 
 	if (!_portInitialized) {
@@ -735,12 +717,9 @@ Network::~Network()
 	ZT_VirtualNetworkConfig ctmp;
 	_externalConfig(&ctmp);
 
-	char n[128];
 	if (_destroyed) {
-		// This is done in Node::leave() so we can pass tPtr
+		// This is done in Node::leave() so we can pass tPtr properly
 		//RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
-		RR->node->dataStoreDelete((void *)0,n);
 	} else {
 		RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp);
 	}
@@ -1188,10 +1167,8 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD
 		if (saveToDisk) {
 			Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *d = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
 			try {
-				char n[64];
-				Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
 				if (nconf.toDictionary(*d,false))
-					RR->node->dataStorePut(tPtr,n,(const void *)d->data(),d->sizeBytes(),true);
+					RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,_id,d->data(),d->sizeBytes());
 			} catch ( ... ) {}
 			delete d;
 		}

node/Network.hpp

@@ -88,8 +88,9 @@ public:
 	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param nwid Network ID
 	 * @param uptr Arbitrary pointer used by externally-facing API (for user use)
+	 * @param nconf Network config, if known
 	 */
-	Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr);
+	Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr,const NetworkConfig *nconf);
 
 	~Network();
 

node/Node.cpp

@@ -33,6 +33,7 @@
 #include "../version.h"
 
 #include "Constants.hpp"
+#include "SharedPtr.hpp"
 #include "Node.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "NetworkController.hpp"
@@ -45,6 +46,7 @@
 #include "Identity.hpp"
 #include "SelfAwareness.hpp"
 #include "Cluster.hpp"
+#include "Network.hpp"
 
 const struct sockaddr_storage ZT_SOCKADDR_NULL = {0};
 
@@ -58,6 +60,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6
 	_RR(this),
 	RR(&_RR),
 	_uPtr(uptr),
+	_networks(8),
 	_now(now),
 	_lastPingCheck(0),
 	_lastHousekeepingRun(0)
@@ -74,20 +77,31 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6
 	memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo));
 	memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification));
 
-	std::string idtmp(dataStoreGet(tptr,"identity.secret"));
-	if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) {
-		TRACE("identity.secret not found, generating...");
-		RR->identity.generate();
-		idtmp = RR->identity.toString(true);
-		if (!dataStorePut(tptr,"identity.secret",idtmp,true))
-			throw std::runtime_error("unable to write identity.secret");
+	char tmp[512];
+	std::string tmp2;
+	int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,0,tmp,sizeof(tmp) - 1);
+	if (n > 0) {
+		tmp[n] = (char)0;
+		if (!RR->identity.fromString(tmp))
+			n = -1;
 	}
-	RR->publicIdentityStr = RR->identity.toString(false);
-	RR->secretIdentityStr = RR->identity.toString(true);
-	idtmp = dataStoreGet(tptr,"identity.public");
-	if (idtmp != RR->publicIdentityStr) {
-		if (!dataStorePut(tptr,"identity.public",RR->publicIdentityStr,false))
-			throw std::runtime_error("unable to write identity.public");
+	if (n <= 0) {
+		RR->identity.generate();
+		tmp2 = RR->identity.toString(true);
+		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+		tmp2 = RR->identity.toString(false);
+		stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+	} else {
+		n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp,sizeof(tmp) - 1);
+		if (n > 0) {
+			tmp[n] = (char)0;
+			if (RR->identity.toString(false) != tmp)
+				n = -1;
+		}
+		if (n <= 0) {
+			tmp2 = RR->identity.toString(false);
+			stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length());
+		}
 	}
 
 	try {
@@ -110,7 +124,7 @@ Node::~Node()
 {
 	Mutex::Lock _l(_networks_m);
 
-	_networks.clear(); // ensure that networks are destroyed before shutdow
+	_networks.clear(); // destroy all networks before shutdown
 
 	delete RR->sa;
 	delete RR->topology;
@@ -122,6 +136,88 @@ Node::~Node()
 #endif
 }
 
+ZT_ResultCode Node::processStateUpdate(
+	void *tptr,
+	ZT_StateObjectType type,
+	uint64_t id,
+	const void *data,
+	unsigned int len)
+{
+	ZT_ResultCode r = ZT_RESULT_OK_IGNORED;
+	switch(type) {
+
+		case ZT_STATE_OBJECT_PEER: {
+		}	break;
+
+		case ZT_STATE_OBJECT_NETWORK_CONFIG:
+			if (len <= (ZT_NETWORKCONFIG_DICT_CAPACITY - 1)) {
+				if (len < 2) {
+					Mutex::Lock _l(_networks_m);
+					SharedPtr<Network> &nw = _networks[id];
+					if (!nw)
+						nw = SharedPtr<Network>(new Network(RR,tptr,id,(void *)0,(const NetworkConfig *)0));
+				} else {
+					Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *dict = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>(reinterpret_cast<const char *>(data),len);
+					try {
+						NetworkConfig *nconf = new NetworkConfig();
+						try {
+							if (nconf->fromDictionary(*dict)) {
+								Mutex::Lock _l(_networks_m);
+								SharedPtr<Network> &nw = _networks[id];
+								if (nw) {
+									switch (nw->setConfiguration(tptr,*nconf,false)) {
+										default:
+											r = ZT_RESULT_ERROR_BAD_PARAMETER;
+											break;
+										case 1:
+											r = ZT_RESULT_OK_IGNORED;
+											break;
+										case 2:
+											r = ZT_RESULT_OK;
+											break;
+									}
+								} else {
+									nw = SharedPtr<Network>(new Network(RR,tptr,id,(void *)0,nconf));
+								}
+							} else {
+								r = ZT_RESULT_ERROR_BAD_PARAMETER;
+							}
+						} catch ( ... ) {
+							r = ZT_RESULT_ERROR_BAD_PARAMETER;
+						}
+						delete nconf;
+					} catch ( ... ) {
+						r = ZT_RESULT_ERROR_BAD_PARAMETER;
+					}
+					delete dict;
+				}
+			} else {
+				r = ZT_RESULT_ERROR_BAD_PARAMETER;
+			}
+			break;
+
+		case ZT_STATE_OBJECT_PLANET:
+		case ZT_STATE_OBJECT_MOON:
+			if (len <= ZT_WORLD_MAX_SERIALIZED_LENGTH) {
+				World w;
+				try {
+					w.deserialize(Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH>(data,len));
+					if (( (w.type() == World::TYPE_MOON)&&(type == ZT_STATE_OBJECT_MOON) )||( (w.type() == World::TYPE_PLANET)&&(type == ZT_STATE_OBJECT_PLANET) )) {
+						r = (RR->topology->addWorld(tptr,w,false)) ? ZT_RESULT_OK : ZT_RESULT_OK_IGNORED;
+					}
+				} catch ( ... ) {
+					r = ZT_RESULT_ERROR_BAD_PARAMETER;
+				}
+			} else {
+				r = ZT_RESULT_ERROR_BAD_PARAMETER;
+			}
+			break;
+
+		default: break;
+	}
+	return r;
+}
+
 ZT_ResultCode Node::processWirePacket(
 	void *tptr,
 	uint64_t now,
@@ -311,7 +407,7 @@ ZT_ResultCode Node::join(uint64_t nwid,void *uptr,void *tptr)
 	Mutex::Lock _l(_networks_m);
 	SharedPtr<Network> &nw = _networks[nwid];
 	if (!nw)
-		nw = SharedPtr<Network>(new Network(RR,tptr,nwid,uptr));
+		nw = SharedPtr<Network>(new Network(RR,tptr,nwid,uptr,(const NetworkConfig *)0));
 	return ZT_RESULT_OK;
 }
 
@@ -319,7 +415,6 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr)
 {
 	ZT_VirtualNetworkConfig ctmp;
 	void **nUserPtr = (void **)0;
-
 	{
 		Mutex::Lock _l(_networks_m);
 		SharedPtr<Network> *nw = _networks.get(nwid);
@@ -330,12 +425,18 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr)
 		(*nw)->externalConfig(&ctmp);
 		(*nw)->destroy();
 		nUserPtr = (*nw)->userPtr();
-		_networks.erase(nwid);
 	}
 
 	if (nUserPtr)
 		RR->node->configureVirtualNetworkPort(tptr,nwid,nUserPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
 
+	{
+		Mutex::Lock _l(_networks_m);
+		_networks.erase(nwid);
+	}
+
+	RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid);
+
 	return ZT_RESULT_OK;
 }
 
@@ -579,20 +680,6 @@ void Node::clusterStatus(ZT_ClusterStatus *cs)
 /* Node methods used only within node/                                      */
 /****************************************************************************/
 
-std::string Node::dataStoreGet(void *tPtr,const char *name)
-{
-	char buf[1024];
-	std::string r;
-	unsigned long olen = 0;
-	do {
-		long n = _cb.dataStoreGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen);
-		if (n <= 0)
-			return std::string();
-		r.append(buf,n);
-	} while (r.length() < olen);
-	return r;
-}
-
 bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress)
 {
 	if (!Path::isAddressValidForPath(remoteAddress))
@@ -813,6 +900,23 @@ void ZT_Node_delete(ZT_Node *node)
 	} catch ( ... ) {}
 }
 
+enum ZT_ResultCode ZT_Node_processStateUpdate(
+	ZT_Node *node,
+	void *tptr,
+	ZT_StateObjectType type,
+	uint64_t id,
+	const void *data,
+	unsigned int len)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->processStateUpdate(tptr,type,id,data,len);
+	} catch (std::bad_alloc &exc) {
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
 enum ZT_ResultCode ZT_Node_processWirePacket(
 	ZT_Node *node,
 	void *tptr,

node/Node.hpp

@@ -82,6 +82,12 @@ public:
 
 	// Public API Functions ----------------------------------------------------
 
+	ZT_ResultCode processStateUpdate(
+		void *tptr,
+		ZT_StateObjectType type,
+		uint64_t id,
+		const void *data,
+		unsigned int len);
 	ZT_ResultCode processWirePacket(
 		void *tptr,
 		uint64_t now,
@@ -185,17 +191,16 @@ public:
 		return _directPaths;
 	}
 
-	inline bool dataStorePut(void *tPtr,const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,name,data,len,(int)secure) == 0); }
-	inline bool dataStorePut(void *tPtr,const char *name,const std::string &data,bool secure) { return dataStorePut(tPtr,name,(const void *)data.data(),(unsigned int)data.length(),secure); }
-	inline void dataStoreDelete(void *tPtr,const char *name) { _cb.dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,name,(const void *)0,0,0); }
-	std::string dataStoreGet(void *tPtr,const char *name);
-
 	inline void postEvent(void *tPtr,ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,ev,md); }
 
 	inline int configureVirtualNetworkPort(void *tPtr,uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,nwid,nuptr,op,nc); }
 
 	inline bool online() const throw() { return _online; }
 
+	inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id,void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,maxlen); }
+	inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,data,(int)len); }
+	inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id) { _cb.statePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,type,id,(const void *)0,-1); }
+
 #ifdef ZT_TRACE
 	void postTrace(const char *module,unsigned int line,const char *fmt,...);
 #endif

node/Topology.cpp

@@ -68,15 +68,15 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) :
 	_trustedPathCount(0),
 	_amRoot(false)
 {
-	try {
-		World cachedPlanet;
-		std::string buf(RR->node->dataStoreGet(tPtr,"planet"));
-		if (buf.length() > 0) {
-			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp(buf.data(),(unsigned int)buf.length());
-			cachedPlanet.deserialize(dswtmp,0);
-		}
-		addWorld(tPtr,cachedPlanet,false);
-	} catch ( ... ) {}
+	uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,0,tmp,sizeof(tmp));
+	if (n > 0) {
+		try {
+			World cachedPlanet;
+			cachedPlanet.deserialize(Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH>(tmp,(unsigned int)n),0);
+			addWorld(tPtr,cachedPlanet,false);
+		} catch ( ... ) {} // ignore invalid cached planets
+	}
 
 	World defaultPlanet;
 	{
@@ -158,9 +158,8 @@ Identity Topology::getIdentity(void *tPtr,const Address &zta)
 void Topology::saveIdentity(void *tPtr,const Identity &id)
 {
 	if (id) {
-		char p[128];
-		Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)id.address().toInt());
-		RR->node->dataStorePut(tPtr,p,id.toString(false),false);
+		const std::string tmp(id.toString(false));
+		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,id.address().toInt(),tmp.data(),(unsigned int)tmp.length());
 	}
 }
 
@@ -327,19 +326,11 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew)
 		return false;
 	}
 
-	char savePath[64];
-	if (existing->type() == World::TYPE_MOON) {
-		Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",existing->id());
-	} else {
-		Utils::scopy(savePath,sizeof(savePath),"planet");
-	}
 	try {
-		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp;
-		existing->serialize(dswtmp,false);
-		RR->node->dataStorePut(tPtr,savePath,dswtmp.data(),dswtmp.size(),false);
-	} catch ( ... ) {
-		RR->node->dataStoreDelete(tPtr,savePath);
-	}
+		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> sbuf;
+		existing->serialize(sbuf,false);
+		RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,existing->id(),sbuf.data(),sbuf.size());
+	} catch ( ... ) {}
 
 	_memoizeUpstreams(tPtr);
 
@@ -348,21 +339,18 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew)
 
 void Topology::addMoon(void *tPtr,const uint64_t id,const Address &seed)
 {
-	char savePath[64];
-	Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id);
-
-	try {
-		std::string moonBin(RR->node->dataStoreGet(tPtr,savePath));
-		if (moonBin.length() > 1) {
-			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> wtmp(moonBin.data(),(unsigned int)moonBin.length());
+	char tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,id,tmp,sizeof(tmp));
+	if (n > 0) {
+		try {
 			World w;
-			w.deserialize(wtmp);
+			w.deserialize(Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH>(tmp,(unsigned int)n));
 			if ((w.type() == World::TYPE_MOON)&&(w.id() == id)) {
 				addWorld(tPtr,w,true);
 				return;
 			}
-		}
-	} catch ( ... ) {}
+		} catch ( ... ) {}
+	}
 
 	if (seed) {
 		Mutex::Lock _l(_upstreams_m);
@@ -381,9 +369,7 @@ void Topology::removeMoon(void *tPtr,const uint64_t id)
 		if (m->id() != id) {
 			nm.push_back(*m);
 		} else {
-			char savePath[64];
-			Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id);
-			RR->node->dataStoreDelete(tPtr,savePath);
+			RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,id);
 		}
 	}
 	_moons.swap(nm);
@@ -425,12 +411,12 @@ void Topology::clean(uint64_t now)
 
 Identity Topology::_getIdentity(void *tPtr,const Address &zta)
 {
-	char p[128];
-	Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)zta.toInt());
-	std::string ids(RR->node->dataStoreGet(tPtr,p));
-	if (ids.length() > 0) {
+	char tmp[512];
+	int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,zta.toInt(),tmp,sizeof(tmp) - 1);
+	if (n > 0) {
+		tmp[n] = (char)0;
 		try {
-			return Identity(ids);
+			return Identity(tmp);
 		} catch ( ... ) {} // ignore invalid IDs
 	}
 	return Identity();