mirror
/
devilutionX-ZeroTierOne
mirror of https://github.com/diasurgical/ZeroTierOne.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Refactor controller to use split-out DB for better performance and less ugly.

pull/1/head
Adam Ierymenko 9 years ago
parent
ab2ccb094a
commit
b03c7b2f30
6 changed files with 223 additions and 172 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 265
      controller/EmbeddedNetworkController.cpp
  2. 14
      controller/EmbeddedNetworkController.hpp
  3. 58
      controller/JSONDB.cpp
  4. 36
      controller/JSONDB.hpp
  5. 16
      osdep/OSUtils.cpp
  6. 6
      osdep/OSUtils.hpp

265
controller/EmbeddedNetworkController.cpp
Unescape View File

@ -64,6 +64,7 @@ using json = nlohmann::json;
namespace ZeroTier { namespace ZeroTier {
// JSON blob I/O // JSON blob I/O
/*
static json _readJson(const std::string &path) static json _readJson(const std::string &path)
{ {
std::string buf; std::string buf;
@ -78,6 +79,7 @@ static bool _writeJson(const std::string &path,const json &obj)
{ {
return OSUtils::writeFile(path.c_str(),obj.dump(2)); return OSUtils::writeFile(path.c_str(),obj.dump(2));
} }
*/
// Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible // Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible
static uint64_t _jI(const json &jv,const uint64_t dfl) static uint64_t _jI(const json &jv,const uint64_t dfl)
@ -475,55 +477,17 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule)
} }
EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) :
_node(node), _db(dbPath),
_path(dbPath), _node(node)
_daemonRun(true)
{ {
OSUtils::mkdir(dbPath); OSUtils::mkdir(dbPath);
OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions
_daemon = Thread::start(this);
} }
EmbeddedNetworkController::~EmbeddedNetworkController() EmbeddedNetworkController::~EmbeddedNetworkController()
{ {
} }
void EmbeddedNetworkController::threadMain()
throw()
{
uint64_t lastUpdatedNetworkMemberCache = 0;
while (_daemonRun) {
// Every 60 seconds we rescan the filesystem for network members and rebuild our cache
if ((OSUtils::now() - lastUpdatedNetworkMemberCache) >= 60000) {
const std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str()));
for(auto n=networks.begin();n!=networks.end();++n) {
if (n->length() == 16) {
const std::vector<std::string> members(OSUtils::listSubdirectories((*n + ZT_PATH_SEPARATOR_S + "member").c_str()));
std::map<Address,nlohmann::json> newCache;
for(auto m=members.begin();m!=members.end();++m) {
if (m->length() == ZT_ADDRESS_LENGTH_HEX) {
const Address maddr(*m);
try {
const json mj(_readJson((_path + ZT_PATH_SEPARATOR_S + "network" + ZT_PATH_SEPARATOR_S + *n + ZT_PATH_SEPARATOR_S + "member" + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json")));
if ((mj.is_object())&&(mj.size() > 0)) {
newCache[maddr] = mj;
}
} catch ( ... ) {}
}
}
{
Mutex::Lock _l(_networkMemberCache_m);
_networkMemberCache[Utils::hexStrToU64(n->c_str())] = newCache;
}
}
}
lastUpdatedNetworkMemberCache = OSUtils::now();
}
Thread::sleep(25);
}
}
NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData,NetworkConfig &nc) NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData,NetworkConfig &nc)
{ {
if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) {
@ -541,12 +505,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
lrt = now; lrt = now;
} }
json network(_readJson(_networkJP(nwid,false))); char nwids[24];
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid);
json network;
json member;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
member = _db.get("network",nwids,"member",identity.address().toString(),0);
}
if (!network.size()) if (!network.size())
return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND;
const std::string memberJP(_memberJP(nwid,identity.address(),true));
json member(_readJson(memberJP));
_initMember(member); _initMember(member);
{ {
@ -673,7 +642,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
// If they are not authorized, STOP! // If they are not authorized, STOP!
if (!authorizedBy) { if (!authorizedBy) {
_writeJson(memberJP,member); Mutex::Lock _l(_db_m);
_db.put("network",nwids,"member",identity.address().toString(),member);
return NetworkController::NETCONF_QUERY_ACCESS_DENIED; return NetworkController::NETCONF_QUERY_ACCESS_DENIED;
} }
@ -966,7 +936,10 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
return NETCONF_QUERY_INTERNAL_SERVER_ERROR; return NETCONF_QUERY_INTERNAL_SERVER_ERROR;
} }
_writeJson(memberJP,member); {
Mutex::Lock _l(_db_m);
_db.put("network",nwids,"member",identity.address().toString(),member);
}
return NetworkController::NETCONF_QUERY_OK; return NetworkController::NETCONF_QUERY_OK;
} }
@ -985,7 +958,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
char nwids[24]; char nwids[24];
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid);
json network(_readJson(_networkJP(nwid,false))); json network;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
}
if (!network.size()) if (!network.size())
return 404; return 404;
@ -996,7 +973,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
if (path.size() >= 4) { if (path.size() >= 4) {
const uint64_t address = Utils::hexStrToU64(path[3].c_str()); const uint64_t address = Utils::hexStrToU64(path[3].c_str());
json member(_readJson(_memberJP(nwid,Address(address),false))); json member;
{
Mutex::Lock _l(_db_m);
member = _db.get("network",nwids,"member",Address(address).toString(),0);
}
if (!member.size()) if (!member.size())
return 404; return 404;
@ -1007,19 +988,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
return 200; return 200;
} else { } else {
Mutex::Lock _l(_db_m);
responseBody = "{"; responseBody = "{";
std::vector<std::string> members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); std::string pfx(std::string("network/") + nwids + "member/");
for(std::vector<std::string>::iterator i(members.begin());i!=members.end();++i) { _db.filter(pfx,120000,[&responseBody](const std::string &n,const json &member) {
if (i->length() == ZT_ADDRESS_LENGTH_HEX) { if (member.size() > 0) {
json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); responseBody.append((responseBody.length() == 1) ? "\"" : ",\"");
if (member.size()) { responseBody.append(_jS(member["id"],""));
responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append("\":");
responseBody.append(*i); responseBody.append(_jS(member["revision"],"0"));
responseBody.append("\":");
responseBody.append(_jS(member["revision"],"0"));
}
} }
} return true; // never delete
});
responseBody.push_back('}'); responseBody.push_back('}');
responseContentType = "application/json"; responseContentType = "application/json";
@ -1056,7 +1037,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
} }
} else if (path.size() == 1) { } else if (path.size() == 1) {
responseBody = "["; /*
std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str()));
for(auto i(networks.begin());i!=networks.end();++i) { for(auto i(networks.begin());i!=networks.end();++i) {
if (i->length() == 16) { if (i->length() == 16) {
@ -1065,6 +1046,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
responseBody.append("\""); responseBody.append("\"");
} }
} }
*/
responseBody.push_back(']'); responseBody.push_back(']');
responseContentType = "application/json"; responseContentType = "application/json";
return 200; return 200;
@ -1122,7 +1104,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid);
if (path.size() >= 3) { if (path.size() >= 3) {
json network(_readJson(_networkJP(nwid,false))); json network;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
}
if (!network.size()) if (!network.size())
return 404; return 404;
@ -1131,7 +1117,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
char addrs[24]; char addrs[24];
Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address);
json member(_readJson(_memberJP(nwid,Address(address),true))); json member;
{
Mutex::Lock _l(_db_m);
member = _db.get("network",nwids,"member",Address(address).toString(),0);
}
if (!member.size())
return 404;
_initMember(member); _initMember(member);
try { try {
@ -1154,7 +1146,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} }
if (b.count("ipAssignments")) { if (b.count("ipAssignments")) {
auto ipa = b["ipAssignments"]; json &ipa = b["ipAssignments"];
if (ipa.is_array()) { if (ipa.is_array()) {
json mipa(json::array()); json mipa(json::array());
for(unsigned long i=0;i<ipa.size();++i) { for(unsigned long i=0;i<ipa.size();++i) {
@ -1169,7 +1161,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} }
if (b.count("tags")) { if (b.count("tags")) {
auto tags = b["tags"]; json &tags = b["tags"];
if (tags.is_array()) { if (tags.is_array()) {
std::map<uint64_t,uint64_t> mtags; std::map<uint64_t,uint64_t> mtags;
for(unsigned long i=0;i<tags.size();++i) { for(unsigned long i=0;i<tags.size();++i) {
@ -1189,7 +1181,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} }
if (b.count("capabilities")) { if (b.count("capabilities")) {
auto capabilities = b["capabilities"]; json &capabilities = b["capabilities"];
if (capabilities.is_array()) { if (capabilities.is_array()) {
json mcaps = json::array(); json mcaps = json::array();
for(unsigned long i=0;i<capabilities.size();++i) { for(unsigned long i=0;i<capabilities.size();++i) {
@ -1210,14 +1202,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
member["address"] = addrs; // legacy member["address"] = addrs; // legacy
member["nwid"] = nwids; member["nwid"] = nwids;
member["lastModified"] = now; member["lastModified"] = now;
auto revj = member["revision"]; json &revj = member["revision"];
member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
_writeJson(_memberJP(nwid,Address(address),true).c_str(),member);
{ {
Mutex::Lock _l(_networkMemberCache_m); Mutex::Lock _l(_db_m);
_networkMemberCache[nwid][Address(address)] = member; _db.put("network",nwids,"member",Address(address).toString(),member);
} }
// Add non-persisted fields // Add non-persisted fields
@ -1281,26 +1271,33 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} else { } else {
// POST to network ID // POST to network ID
// Magic ID ending with ______ picks a random unused network ID json network;
if (path[1].substr(10) == "______") { {
nwid = 0; Mutex::Lock _l(_db_m);
uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL;
uint64_t nwidPostfix = 0; // Magic ID ending with ______ picks a random unused network ID
for(unsigned long k=0;k<100000;++k) { // sanity limit on trials if (path[1].substr(10) == "______") {
Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); nwid = 0;
uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL;
if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; uint64_t nwidPostfix = 0;
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); for(unsigned long k=0;k<100000;++k) { // sanity limit on trials
if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) { Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix));
nwid = tryNwid; uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
break; if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid);
if (_db.get("network",nwids,120000).size() <= 0) {
nwid = tryNwid;
break;
}
} }
if (!nwid)
return 503;
} }
if (!nwid)
return 503;
}
json network(_readJson(_networkJP(nwid,true))); network = _db.get("network",nwids,0);
if (!network.size())
return 404;
}
_initNetwork(network); _initNetwork(network);
try { try {
@ -1481,11 +1478,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
network["id"] = nwids; network["id"] = nwids;
network["nwid"] = nwids; // legacy network["nwid"] = nwids; // legacy
auto rev = network["revision"]; json &rev = network["revision"];
network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL); network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL);
network["lastModified"] = now; network["lastModified"] = now;
_writeJson(_networkJP(nwid,true),network); {
Mutex::Lock _l(_db_m);
_db.put("network",nwids,network);
}
_NetworkMemberInfo nmi; _NetworkMemberInfo nmi;
_getNetworkMemberInfo(now,nwid,nmi); _getNetworkMemberInfo(now,nwid,nmi);
@ -1518,7 +1518,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
if ((path.size() >= 2)&&(path[1].length() == 16)) { if ((path.size() >= 2)&&(path[1].length() == 16)) {
const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); const uint64_t nwid = Utils::hexStrToU64(path[1].c_str());
json network(_readJson(_networkJP(nwid,false))); char nwids[24];
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid);
json network;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
}
if (!network.size()) if (!network.size())
return 404; return 404;
@ -1526,22 +1532,23 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) {
const uint64_t address = Utils::hexStrToU64(path[3].c_str()); const uint64_t address = Utils::hexStrToU64(path[3].c_str());
json member(_readJson(_memberJP(nwid,Address(address),false))); Mutex::Lock _l(_db_m);
json member = _db.get("network",nwids,"member",Address(address).toString(),0);
if (!member.size()) if (!member.size())
return 404; return 404;
_db.erase("network",nwids,"member",Address(address).toString());
OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str());
responseBody = member.dump(2); responseBody = member.dump(2);
responseContentType = "application/json"; responseContentType = "application/json";
return 200; return 200;
} }
} else { } else {
OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); Mutex::Lock _l(_db_m);
{ std::string pfx("network/"); pfx.append(nwids);
Mutex::Lock _l(_networkMemberCache_m); _db.filter(pfx,120000,[](const std::string &n,const json &obj) {
_networkMemberCache.erase(nwid); return false; // delete
} });
responseBody = network.dump(2); responseBody = network.dump(2);
responseContentType = "application/json"; responseContentType = "application/json";
return 200; return 200;
@ -1618,42 +1625,46 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes
void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi)
{ {
Mutex::Lock _mcl(_networkMemberCache_m); char pfx[256];
std::map< Address,nlohmann::json > &memberCacheEntry = _networkMemberCache[nwid]; Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid);
nmi.totalMemberCount = memberCacheEntry.size();
for(std::map< Address,nlohmann::json >::iterator nm(memberCacheEntry.begin());nm!=memberCacheEntry.end();++nm) { Mutex::Lock _l(_db_m);
if (_jB(nm->second["authorized"],false)) { _db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) {
++nmi.authorizedMemberCount; try {
if (_jB(member["authorized"],false)) {
if (nm->second.count("recentLog")) { ++nmi.authorizedMemberCount;
json &mlog = nm->second["recentLog"];
if ((mlog.is_array())&&(mlog.size() > 0)) { if (member.count("recentLog")) {
json &mlog1 = mlog[0]; const json &mlog = member["recentLog"];
if (mlog1.is_object()) { if ((mlog.is_array())&&(mlog.size() > 0)) {
if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) const json &mlog1 = mlog[0];
++nmi.activeMemberCount; if (mlog1.is_object()) {
if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD)
++nmi.activeMemberCount;
}
} }
} }
}
if (_jB(nm->second["activeBridge"],false)) { if (_jB(member["activeBridge"],false)) {
nmi.activeBridges.insert(nm->first); nmi.activeBridges.insert(_jS(member["id"],"0000000000"));
} }
if (nm->second.count("ipAssignments")) { if (member.count("ipAssignments")) {
json &mips = nm->second["ipAssignments"]; const json &mips = member["ipAssignments"];
if (mips.is_array()) { if (mips.is_array()) {
for(unsigned long i=0;i<mips.size();++i) { for(unsigned long i=0;i<mips.size();++i) {
InetAddress mip(_jS(mips[i],"")); InetAddress mip(_jS(mips[i],""));
if ((mip.ss_family == AF_INET)||(mip.ss_family == AF_INET6)) if ((mip.ss_family == AF_INET)||(mip.ss_family == AF_INET6))
nmi.allocatedIps.insert(mip); nmi.allocatedIps.insert(mip);
}
} }
} }
} else {
nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(member["lastDeauthorizedTime"],0ULL));
} }
} else { } catch ( ... ) {}
nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(nm->second["lastDeauthorizedTime"],0ULL)); return true;
} });
}
} }
} // namespace ZeroTier } // namespace ZeroTier

14
controller/EmbeddedNetworkController.hpp
Unescape View File

@ -40,6 +40,8 @@
#include "../ext/json/json.hpp" #include "../ext/json/json.hpp"
#include "JSONDB.hpp"
namespace ZeroTier { namespace ZeroTier {
class Node; class Node;
@ -50,10 +52,6 @@ public:
EmbeddedNetworkController(Node *node,const char *dbPath); EmbeddedNetworkController(Node *node,const char *dbPath);
virtual ~EmbeddedNetworkController(); virtual ~EmbeddedNetworkController();
// Thread main method -- do not call directly
void threadMain()
throw();
virtual NetworkController::ResultCode doNetworkConfigRequest( virtual NetworkController::ResultCode doNetworkConfigRequest(
const InetAddress &fromAddr, const InetAddress &fromAddr,
const Identity &signingId, const Identity &signingId,
@ -88,6 +86,7 @@ private:
static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report);
// Network base path and network JSON path // Network base path and network JSON path
/*
inline std::string _networkBP(const uint64_t nwid,bool create) inline std::string _networkBP(const uint64_t nwid,bool create)
{ {
char tmp[64]; char tmp[64];
@ -124,6 +123,10 @@ private:
// In-memory cache of network members // In-memory cache of network members
std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache; std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache;
Mutex _networkMemberCache_m; Mutex _networkMemberCache_m;
*/
JSONDB _db;
Mutex _db_m;
// Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places
// This does lock _networkMemberCache_m // This does lock _networkMemberCache_m
@ -203,9 +206,6 @@ private:
// Last request time by address, for rate limitation // Last request time by address, for rate limitation
std::map< std::pair<uint64_t,uint64_t>,uint64_t > _lastRequestTime; std::map< std::pair<uint64_t,uint64_t>,uint64_t > _lastRequestTime;
Mutex _lastRequestTime_m; Mutex _lastRequestTime_m;
Thread _daemon;
volatile bool _daemonRun;
}; };
} // namespace ZeroTier } // namespace ZeroTier

58
controller/JSONDB.cpp
Unescape View File

@ -27,7 +27,7 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj)
if (!_isValidObjectName(n)) if (!_isValidObjectName(n))
return false; return false;
std::string path(_genPath(n,false)); std::string path(_genPath(n,true));
if (!path.length()) if (!path.length())
return false; return false;
@ -36,10 +36,9 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj)
return false; return false;
_E &e = _db[n]; _E &e = _db[n];
e.obj = obj;
e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str());
e.lastCheck = OSUtils::now(); e.lastCheck = OSUtils::now();
e.obj = obj;
return true; return true;
} }
@ -56,7 +55,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
if (e != _db.end()) { if (e != _db.end()) {
if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck) if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck)
return e->second.obj; return e->second.obj;
std::string path(_genPath(n,false)); std::string path(_genPath(n,false));
if (!path.length()) // sanity check if (!path.length()) // sanity check
return _EMPTY_JSON; return _EMPTY_JSON;
@ -64,15 +62,13 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
// We are somewhat tolerant to momentary disk failures here. This may // We are somewhat tolerant to momentary disk failures here. This may
// occur over e.g. EC2's elastic filesystem (NFS). // occur over e.g. EC2's elastic filesystem (NFS).
const uint64_t lm = OSUtils::getLastModified(path.c_str()); const uint64_t lm = OSUtils::getLastModified(path.c_str());
if ((lm)&&(e->second.lastModifiedOnDisk != lm)) { if (e->second.lastModifiedOnDisk != lm) {
if (OSUtils::readFile(path.c_str(),buf)) { if (OSUtils::readFile(path.c_str(),buf)) {
try { try {
e->second.lastModifiedOnDisk = lm;
e->second.lastCheck = now;
e->second.obj = nlohmann::json::parse(buf); e->second.obj = nlohmann::json::parse(buf);
} catch ( ... ) { e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP
e->second.obj = _EMPTY_JSON; e->second.lastCheck = now;
} } catch ( ... ) {} // parse errors result in "holding pattern" behavior
} }
} }
@ -86,22 +82,50 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
return _EMPTY_JSON; return _EMPTY_JSON;
const uint64_t lm = OSUtils::getLastModified(path.c_str()); const uint64_t lm = OSUtils::getLastModified(path.c_str());
if (!lm)
return _EMPTY_JSON;
_E &e2 = _db[n]; _E &e2 = _db[n];
e2.lastModifiedOnDisk = lm;
e2.lastCheck = now;
try { try {
e2.obj = nlohmann::json::parse(buf); e2.obj = nlohmann::json::parse(buf);
} catch ( ... ) { } catch ( ... ) {
e2.obj = _EMPTY_JSON; e2.obj = _EMPTY_JSON;
} }
e2.lastModifiedOnDisk = lm;
e2.lastCheck = now;
return e2.obj; return e2.obj;
} }
} }
void JSONDB::erase(const std::string &n)
{
if (!_isValidObjectName(n))
return;
std::string path(_genPath(n,true));
if (!path.length())
return;
OSUtils::rm(path.c_str());
_db.erase(n);
}
void JSONDB::_reload(const std::string &p)
{
std::map<std::string,char> l(OSUtils::listDirectoryFull(p.c_str()));
for(std::map<std::string,char>::iterator li(l.begin());li!=l.end();++li) {
if (li->second == 'f') {
// assume p starts with _basePath, which it always does -- will throw otherwise
std::string n(p.substr(_basePath.length()));
while ((n.length() > 0)&&(n[0] == ZT_PATH_SEPARATOR)) n = n.substr(1);
if (ZT_PATH_SEPARATOR != '/') std::replace(n.begin(),n.end(),ZT_PATH_SEPARATOR,'/');
if ((n.length() > 0)&&(n[n.length() - 1] != '/')) n.push_back('/');
n.append(li->first);
this->get(n,0); // causes load and cache or update
} else if (li->second == 'd') {
this->_reload(p + ZT_PATH_SEPARATOR + li->first);
}
}
}
bool JSONDB::_isValidObjectName(const std::string &n) bool JSONDB::_isValidObjectName(const std::string &n)
{ {
if (n.length() == 0) if (n.length() == 0)
@ -139,8 +163,4 @@ std::string JSONDB::_genPath(const std::string &n,bool create)
return p; return p;
} }
void JSONDB::_reloadAll(const std::string &path)
{
}
} // namespace ZeroTier } // namespace ZeroTier

36
controller/JSONDB.hpp
Unescape View File

@ -41,13 +41,21 @@ public:
JSONDB(const std::string &basePath) : JSONDB(const std::string &basePath) :
_basePath(basePath) _basePath(basePath)
{ {
this->_reloadAll(_basePath); _reload(_basePath);
}
inline void reload()
{
_db.clear();
_reload(_basePath);
} }
bool put(const std::string &n,const nlohmann::json &obj); bool put(const std::string &n,const nlohmann::json &obj);
inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); }
inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); } inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); }
inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); }
inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); }
const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0); const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0);
@ -56,27 +64,37 @@ public:
inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); } inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); }
inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); } inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); }
void erase(const std::string &n);
inline void erase(const std::string &n1,const std::string &n2) { this->erase(n1 + "/" + n2); }
inline void erase(const std::string &n1,const std::string &n2,const std::string &n3) { this->erase(n1 + "/" + n2 + "/" + n3); }
inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4); }
inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5); }
template<typename F> template<typename F>
inline void each(F func,unsigned long maxSinceCheck = 0) inline void filter(const std::string &prefix,unsigned long maxSinceCheck,F func)
{ {
const uint64_t now = OSUtils::now(); for(std::map<std::string,_E>::iterator i(_db.lower_bound(prefix));i!=_db.end();) {
for(std::map<std::string,_E>::const_iterator i(_db.begin());i!=_db.end();++i) { if (i->first.substr(0,prefix.length()) == prefix) {
if ((now - i->second.lastCheck) > (uint64_t)maxSinceCheck) if (!func(i->first,get(i->second.obj,maxSinceCheck))) {
this->get(i->first); std::map<std::string,_E>::iterator i2(i); ++i2;
func(i->first,i->second.obj); this->erase(i->first);
i = i2;
} else ++i;
} else break;
} }
} }
private: private:
void _reload(const std::string &p);
bool _isValidObjectName(const std::string &n); bool _isValidObjectName(const std::string &n);
std::string _genPath(const std::string &n,bool create); std::string _genPath(const std::string &n,bool create);
void _reloadAll(const std::string &path);
struct _E struct _E
{ {
nlohmann::json obj;
uint64_t lastModifiedOnDisk; uint64_t lastModifiedOnDisk;
uint64_t lastCheck; uint64_t lastCheck;
nlohmann::json obj;
}; };
std::string _basePath; std::string _basePath;

16
osdep/OSUtils.cpp
Unescape View File

@ -107,17 +107,18 @@ std::vector<std::string> OSUtils::listDirectory(const char *path)
return r; return r;
} }
std::vector<std::string> OSUtils::listSubdirectories(const char *path) std::map<std::string,char> OSUtils::listDirectoryFull(const char *path)
{ {
std::vector<std::string> r; std::map<std::string,char> r;
#ifdef __WINDOWS__ #ifdef __WINDOWS__
HANDLE hFind; HANDLE hFind;
WIN32_FIND_DATAA ffd; WIN32_FIND_DATAA ffd;
if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) {
do { do {
if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))&&((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)) if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))) {
r.push_back(std::string(ffd.cFileName)); r[ffd.cFileName] = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) ? 'd' : 'f';
}
} while (FindNextFileA(hFind,&ffd)); } while (FindNextFileA(hFind,&ffd));
FindClose(hFind); FindClose(hFind);
} }
@ -132,8 +133,9 @@ std::vector<std::string> OSUtils::listSubdirectories(const char *path)
if (readdir_r(d,&de,&dptr)) if (readdir_r(d,&de,&dptr))
break; break;
if (dptr) { if (dptr) {
if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_DIR)) if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))) {
r.push_back(std::string(dptr->d_name)); r[dptr->d_name] = (dptr->d_type == DT_DIR) ? 'd' : 'f';
}
} else break; } else break;
} }
closedir(d); closedir(d);
@ -178,7 +180,7 @@ bool OSUtils::rmDashRf(const char *path)
std::string p(path); std::string p(path);
p.push_back(ZT_PATH_SEPARATOR); p.push_back(ZT_PATH_SEPARATOR);
p.append(dptr->d_name); p.append(dptr->d_name);
if (unlink(p.c_str()) != 0) { if (unlink(p.c_str()) != 0) { // unlink first will remove symlinks instead of recursing them
if (!rmDashRf(p.c_str())) if (!rmDashRf(p.c_str()))
return false; return false;
} }

6
osdep/OSUtils.hpp
Unescape View File

@ -110,12 +110,12 @@ public:
static std::vector<std::string> listDirectory(const char *path); static std::vector<std::string> listDirectory(const char *path);
/** /**
* List a directory's subdirectories * List all contents in a directory
* *
* @param path Path to list * @param path Path to list
* @return Names of subdirectories (without path prepended) * @return Names of things and types, currently just 'f' and 'd'
*/ */
static std::vector<std::string> listSubdirectories(const char *path); static std::map<std::string,char> listDirectoryFull(const char *path);
/** /**
* Delete a directory and all its files and subdirectories recursively * Delete a directory and all its files and subdirectories recursively

Write Preview
Loading…
Cancel
Save