if((sqlite3_prepare_v2(_db,"SELECT v FROM Config WHERE k = 'schemaVersion';",-1,&s,(constchar**)0)==SQLITE_OK)&&(s)){
intschemaVersion=-1234;
if(sqlite3_step(s)==SQLITE_ROW){
schemaVersion=sqlite3_column_int(s,0);
}
sqlite3_finalize(s);
if(schemaVersion==-1234){
sqlite3_close(_db);
throwstd::runtime_error("SqliteNetworkController schemaVersion not found in Config table (init failure?)");
}
if(schemaVersion<2){
// Create NodeHistory table to upgrade from version 1 to version 2
if(sqlite3_exec(_db,
"CREATE TABLE NodeHistory (\n"
" nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n"
" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"
" networkVisitCounter INTEGER NOT NULL DEFAULT(0),\n"
" networkRequestAuthorized INTEGER NOT NULL DEFAULT(0),\n"
" requestTime INTEGER NOT NULL DEFAULT(0),\n"
" clientMajorVersion INTEGER NOT NULL DEFAULT(0),\n"
" clientMinorVersion INTEGER NOT NULL DEFAULT(0),\n"
" clientRevision INTEGER NOT NULL DEFAULT(0),\n"
" networkRequestMetaData VARCHAR(1024),\n"
" fromAddress VARCHAR(128)\n"
");\n"
"CREATE INDEX NodeHistory_nodeId ON NodeHistory (nodeId);\n"
"CREATE INDEX NodeHistory_networkId ON NodeHistory (networkId);\n"
"CREATE INDEX NodeHistory_requestTime ON NodeHistory (requestTime);\n"
"UPDATE \"Config\" SET \"v\" = 2 WHERE \"k\" = 'schemaVersion';\n"
,0,0,0)!=SQLITE_OK){
charerr[1024];
Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 2: %s",sqlite3_errmsg(_db));
sqlite3_close(_db);
throwstd::runtime_error(err);
}else{
schemaVersion=2;
}
}
if(schemaVersion<3){
// Create Route table to upgrade from version 2 to version 3 and migrate old
// data. Also delete obsolete Gateway table that was never actually used, and
// migrate Network flags to a bitwise flags field instead of ASCII cruft.
if(sqlite3_exec(_db,
"DROP TABLE Gateway;\n"
"CREATE TABLE Route (\n"
" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"
" target blob(16) NOT NULL,\n"
" via blob(16),\n"
" targetNetmaskBits integer NOT NULL,\n"
" ipVersion integer NOT NULL,\n"
" flags integer NOT NULL,\n"
" metric integer NOT NULL\n"
");\n"
"CREATE INDEX Route_networkId ON Route (networkId);\n"
"INSERT INTO Route SELECT DISTINCT networkId,\"ip\" AS \"target\",NULL AS \"via\",ipNetmaskBits AS targetNetmaskBits,ipVersion,0 AS \"flags\",0 AS \"metric\" FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n"
"ALTER TABLE Network ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n"
"UPDATE Network SET \"flags\" = (\"flags\" | 1) WHERE v4AssignMode = 'zt';\n"
"UPDATE Network SET \"flags\" = (\"flags\" | 2) WHERE v6AssignMode = 'rfc4193';\n"
"UPDATE Network SET \"flags\" = (\"flags\" | 4) WHERE v6AssignMode = '6plane';\n"
"ALTER TABLE Member ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n"
"DELETE FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n"
"UPDATE \"Config\" SET \"v\" = 3 WHERE \"k\" = 'schemaVersion';\n"
,0,0,0)!=SQLITE_OK){
charerr[1024];
Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db));
sqlite3_close(_db);
throwstd::runtime_error(err);
}else{
schemaVersion=3;
}
}
if(schemaVersion<4){
// Turns out this was overkill and a huge performance drag. Will be revisiting this
// more later but for now a brief snapshot of recent history stored in Member is fine.
// Also prepare for implementation of proof of work requests.
if(sqlite3_exec(_db,
"DROP TABLE NodeHistory;\n"
"ALTER TABLE Member ADD COLUMN lastRequestTime integer NOT NULL DEFAULT(0);\n"
"ALTER TABLE Member ADD COLUMN lastPowDifficulty integer NOT NULL DEFAULT(0);\n"
"ALTER TABLE Member ADD COLUMN lastPowTime integer NOT NULL DEFAULT(0);\n"
"ALTER TABLE Member ADD COLUMN recentHistory blob;\n"
"CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n"
"UPDATE \"Config\" SET \"v\" = 4 WHERE \"k\" = 'schemaVersion';\n"
,0,0,0)!=SQLITE_OK){
charerr[1024];
Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db));
sqlite3_close(_db);
throwstd::runtime_error(err);
}else{
schemaVersion=4;
}
}
if(schemaVersion<5){
// Upgrade old rough draft Rule table to new release format
if(sqlite3_exec(_db,
"DROP TABLE Relay;\n"
"DROP INDEX Rule_networkId_ruleNo;\n"
"ALTER TABLE \"Rule\" RENAME TO RuleOld;\n"
"CREATE TABLE Rule (\n"
" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"
" capId integer,\n"
" ruleNo integer NOT NULL,\n"
" ruleType integer NOT NULL DEFAULT(0),\n"
"\"addr\" blob(16),\n"
"\"int1\" integer,\n"
"\"int2\" integer,\n"
"\"int3\" integer,\n"
"\"int4\" integer\n"
");\n"
"INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n"
"INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n"
"DROP TABLE RuleOld;\n"
"CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n"
"CREATE TABLE MemberTC (\n"
" networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"
" nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n"
" tagId integer,\n"
" tagValue integer,\n"
" capId integer,\n"
" capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n"
");\n"
"CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n"
"UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n"
,0,0,0)!=SQLITE_OK){
charerr[1024];
Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db));
(sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT INTO Network (id,name,creationTime,revision) VALUES (?,?,?,1)",-1,&_sCreateNetwork,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"DELETE FROM Network WHERE id = ?",-1,&_sDeleteNetwork,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT nodeId FROM Member WHERE networkId = ? AND activeBridge > 0 AND authorized > 0",-1,&_sGetActiveBridges,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT m.nodeId,m.memberRevision FROM Member AS m WHERE m.networkId = ? ORDER BY m.nodeId ASC",-1,&_sListNetworkMembers,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"UPDATE Member SET authorized = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberAuthorized,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"UPDATE Member SET activeBridge = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberActiveBridge,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"UPDATE Member SET \"lastRequestTime\" = ?, \"recentHistory\" = ? WHERE rowid = ?",-1,&_sUpdateMemberHistory,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sDeleteMember,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(constchar**)0)!=SQLITE_OK)
||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(constchar**)0)!=SQLITE_OK)
){
std::stringerr(std::string("SqliteNetworkController unable to initialize one or more prepared statements: ")+sqlite3_errmsg(_db));
if((b.count("identity"))&&(!member.count("identity")))member["identity"]=_jS(b["identity"],"");// allow identity to be populated only if not already known
if(b.count("ipAssignments")){
@ -1184,19 +944,47 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
@ -11,8 +11,6 @@ Data is stored in JSON format under `controller.d` in the ZeroTier working direc
Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons.
Since this implementation uses a JSON store in the filesystem we recommend running it on SSD-backed hosts. Slow disks will become a speed bottleneck under heavy load. For really huge and busy controllers you could consider linking `controller.d/` to a folder under `/dev/shm` (Linux RAM disk) and then setting up an out-of-band periodic snapshot cron job or background process to persist the data and a script to populate `/dev/shm` on boot before the controller starts. This is beyond the scope of this guide but is not particularly hard.
Since ZeroTier nodes are mobile and do not need static IPs, implementing high availability fail-over for controllers is easy. Just replicate their working directories from master to backup and have something automatically fire up the backup if the master goes down. Many modern orchestration tools have built-in support for this. It would also be possible in theory to run controllers on a replicated or distributed filesystem, but we haven't tested this yet.
### Dockerizing Controllers
@ -67,15 +65,15 @@ When POSTing new networks take care that their IDs are not in use, otherwise you
| name | string | A short name for this network | YES |
| private | boolean | Is access control enabled? | YES |