Added new unit tests and time-sync code for subtests, recvfrom() fixes

FAQ.md

@@ -0,0 +1,10 @@
+## Frequently Asked Questions
+***
+
+### Can I use this in my commercial or closed-source application or service?
+
+Yes! - Just let us know, and we will work out a licensing scheme. You will need to build the library with the `lwIP` network stack. In the case that you're a non-profit, or are developing open source software, you can use this entirely for free and may choose either `picoTCP` or `lwIP` as your network stack.
+
+### Application or service won't fully come online
+
+Sometimes it can take a substatial amount of time for libzt to come online and become reachable. In cases where it never seems to progress beyond this stage you should check to make sure there are no rogue processes on the machines using the same ZeroTier identity files, or no other instances on your ZeroTier network using said identity files. If this doesn't help. Contact us.

include/libzt.h

@@ -85,17 +85,27 @@ struct zts_ifreq {
 #endif
 
 /****************************************************************************/
-/* LWIP                                                                     */
+/* lwIP                                                                     */
 /****************************************************************************/
 
+#if defined(STACK_LWIP)
 #define LWIP_APPLICATION_POLL_FREQ         2
 #define LWIP_TCP_TIMER_INTERVAL            50
 #define LWIP_STATUS_TMR_INTERVAL           500 // How often we check VirtualSocket statuses (in ms)
+#endif
+
+/****************************************************************************/
+/* picoTCP                                                                  */
+/****************************************************************************/
+
+#if defined(STACK_PICO)
+#endif
 
 /****************************************************************************/
 /* Defines                                                                  */
 /****************************************************************************/
 
+#define ZT_MAX_SOCKETS                     1024
 #define ZT_SDK_MTU                         ZT_MAX_MTU
 #define ZT_LEN_SZ                          4
 #define ZT_ADDR_SZ                         128
@@ -206,7 +216,6 @@ struct zts_ifreq {
 
 /****************************************************************************/
 /* SDK Socket API (ZeroTier Service Controls)                               */
-/* Implemented in libzt.cpp                                                 */ 
 /****************************************************************************/
 
 #ifdef __cplusplus
@@ -341,9 +350,8 @@ void zts_disable_http_control_plane();
 
 /****************************************************************************/
 /* SDK Socket API (Socket User Controls)                                    */
-/* - These functions are designed to work just like regular socket calls    */
+/* - These functions are designed to work just like ordinary socket calls   */
 /*   but are provisioned and handled by ZeroTier                            */
-/* Implemented in Socket.c                                                  */ 
 /****************************************************************************/
 
 /**

make-linux.mk

@@ -249,20 +249,26 @@ UNIT_TEST_OBJ_FILES := $(addprefix $(TEST_BUILD_DIR)/,$(notdir $(UNIT_TEST_SRC_F
 UNIT_TEST_INCLUDES  := -Iinclude
 UNIT_TEST_LIBS      := -L$(BUILD) -lzt $(COMMON_LIBS)
 
-$(TEST_BUILD_DIR)/%: $(UNIT_TEST_SRC_DIR)/%.cpp
-	@mkdir -p $(TEST_BUILD_DIR)
-	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) -o $@ $< $(UNIT_TEST_LIBS)
-	@./check.sh $@
+#$(TEST_BUILD_DIR)/%: $(UNIT_TEST_SRC_DIR)/%.cpp
+#	@mkdir -p $(TEST_BUILD_DIR)
+#	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) -o $@ $< $(UNIT_TEST_LIBS)
+#	@./check.sh $@
 
-tests: $(UNIT_TEST_OBJ_FILES)
+tests: selftest nativetest ztproxy intercept
 
 intercept:
 	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) examples/intercept/intercept.cpp -D_GNU_SOURCE -shared -o $(BUILD)/intercept.so $< $(UNIT_TEST_LIBS) -ldl
 	@./check.sh $(BUILD)/intercept.so
-
 ztproxy:
 	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) examples/ztproxy/ztproxy.cpp -o $(BUILD)/ztproxy $< $(UNIT_TEST_LIBS) -ldl
 	@./check.sh $(BUILD)/ztproxy
+selftest:
+	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) test/selftest.cpp -D__SELFTEST__ -o $(BUILD)/selftest $(UNIT_TEST_LIBS)
+	@./check.sh $(BUILD)/selftest
+nativetest:
+	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) test/selftest.cpp -D__NATIVETEST__ -o $(BUILD)/nativetest
+	@./check.sh $(BUILD)/nativetest
+
 	
 ##############################################################################
 ## Misc                                                                     ##

make-mac.mk

@@ -261,20 +261,25 @@ UNIT_TEST_OBJ_FILES := $(addprefix $(TEST_BUILD_DIR)/,$(notdir $(UNIT_TEST_SRC_F
 UNIT_TEST_INCLUDES  := -Iinclude
 UNIT_TEST_LIBS      := -L$(BUILD) -lzt
 
-$(TEST_BUILD_DIR)/%: $(UNIT_TEST_SRC_DIR)/%.cpp
-	@mkdir -p $(TEST_BUILD_DIR)
-	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) -o $@ $< $(UNIT_TEST_LIBS)
-	@./check.sh $@
+#$(TEST_BUILD_DIR)/%: $(UNIT_TEST_SRC_DIR)/%.cpp
+#	@mkdir -p $(TEST_BUILD_DIR)
+#	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) -o $@ $< $(UNIT_TEST_LIBS)
+#	@./check.sh $@
 
-tests: $(UNIT_TEST_OBJ_FILES)
+tests: selftest nativetest ztproxy intercept
 
 intercept:
-	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) examples/intercept/intercept.cpp -D_GNU_SOURCE -shared -o $(BUILD)/intercept.so $< $(UNIT_TEST_LIBS) -ldl
+	@$(CXX) $(CXXFLAGS) -fPIE $(UNIT_TEST_INCLUDES) examples/intercept/intercept.cpp -D_GNU_SOURCE -shared -o $(BUILD)/intercept.so $< $(UNIT_TEST_LIBS) -ldl
 	@./check.sh $(BUILD)/intercept.so
-
 ztproxy:
 	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) examples/ztproxy/ztproxy.cpp -o $(BUILD)/ztproxy $< $(UNIT_TEST_LIBS) -ldl
 	@./check.sh $(BUILD)/ztproxy
+selftest:
+	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) test/selftest.cpp -D__SELFTEST__ -o $(BUILD)/selftest $(UNIT_TEST_LIBS)
+	@./check.sh $(BUILD)/selftest
+nativetest:
+	@$(CXX) $(CXXFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) test/selftest.cpp -D__NATIVETEST__ -o $(BUILD)/nativetest
+	@./check.sh $(BUILD)/nativetest
 
 ##############################################################################
 ## Misc                                                                     ##

src/libzt.cpp

@@ -500,9 +500,8 @@ Linux:
 
 int zts_connect(ZT_CONNECT_SIG) {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		DEBUG_ERROR("EBADF");
 		return -1;
 	}
 	if(!ZeroTier::zt1Service) {
@@ -516,6 +515,19 @@ int zts_connect(ZT_CONNECT_SIG) {
 		errno = EBADF;
 		return -1;
 	}
+	if(!addr) {
+		DEBUG_ERROR("invalid address for fd=%d", fd);
+		errno = EINVAL;
+		return -1;
+	}
+	if(addrlen <= 0) {
+		DEBUG_ERROR("invalid address length for fd=%d", fd);
+		errno = EINVAL;
+		return -1;
+	}
+	// TODO: Handle bad address lengths, right now this call will still 
+	// succeed with a complete connect despite a bad address length. 
+
 	// DEBUG_EXTRA("fd = %d, %s : %d", fd, ipstr, ntohs(port));
 	ZeroTier::InetAddress inet;
 	sockaddr2inet(vs->socket_family, addr, &inet);
@@ -618,7 +630,7 @@ Darwin:
 */
 int zts_bind(ZT_BIND_SIG) {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
 		return -1;
 	}
@@ -707,7 +719,7 @@ Linux:
 */
 int zts_listen(ZT_LISTEN_SIG) {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
 		return -1;
 	}
@@ -754,7 +766,7 @@ Darwin:
 int zts_accept(ZT_ACCEPT_SIG) {
 	int err = errno = 0;
 	//DEBUG_EXTRA("fd=%d", fd);
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
 		return -1;
 	}
@@ -857,7 +869,7 @@ EPERM Firewall rules forbid VirtualSocket.
 	{
 		errno = 0;
 		//DEBUG_INFO("fd=%d", fd);
-		if(fd < 0) {
+		if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 			errno = EBADF;
 			return -1;
 		}
@@ -888,9 +900,9 @@ int zts_setsockopt(ZT_SETSOCKOPT_SIG)
 	int err = errno = 0;
 #if defined(STACK_PICO)
 	//DEBUG_INFO("fd=%d", fd);
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	// Disable Nagle's algorithm
 	struct pico_socket *p = NULL;
@@ -926,9 +938,9 @@ int zts_setsockopt(ZT_SETSOCKOPT_SIG)
 int zts_getsockopt(ZT_GETSOCKOPT_SIG)
 {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	err = getsockopt(fd, level, optname, optval, optlen);
 	return err;
@@ -945,9 +957,9 @@ int zts_getsockopt(ZT_GETSOCKOPT_SIG)
 int zts_getsockname(ZT_GETSOCKNAME_SIG)
 {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	// TODO
 	return err;
@@ -968,9 +980,9 @@ Linux:
 int zts_getpeername(ZT_GETPEERNAME_SIG)
 {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	ZeroTier::VirtualSocket *vs = get_virtual_socket(fd);
 	if(!vs) {
@@ -1037,8 +1049,7 @@ Linux / Darwin:
 int zts_close(ZT_CLOSE_SIG)
 {
 	int err = errno = 0;
-	//DEBUG_EXTRA("fd=%d", fd);
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
 		return -1;
 	}
@@ -1096,9 +1107,9 @@ int zts_select(ZT_SELECT_SIG)
 int zts_fcntl(ZT_FCNTL_SIG)
 {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	else {
 		err = fcntl(fd, cmd, flags);
@@ -1116,9 +1127,9 @@ int zts_fcntl(ZT_FCNTL_SIG)
 int zts_ioctl(ZT_IOCTL_SIG)
 {
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	else {
 #if defined(__linux__)
@@ -1190,15 +1201,30 @@ Linux:
 	[  ] [EPIPE]                 	The local end has been shut down on a VirtualSocket oriented socket. 
 									In this case the process will also receive a SIGPIPE unless MSG_NOSIGNAL is set.
 
+	ZT_SENDTO_SIG int fd, const void *buf, size_t len, int flags, const struct sockaddr *addr, socklen_t addrlen
 */
+
 ssize_t zts_sendto(ZT_SENDTO_SIG)
 {
-	DEBUG_TRANS("fd=%d", fd);
+	//DEBUG_TRANS("fd=%d", fd);
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
 		return -1;
 	}
+	if(len == 0) {
+		return 0;
+	}
+	if(len > ZT_SOCKET_MSG_BUF_SZ) {
+		DEBUG_ERROR("msg is too long to be sent atomically (len=%d)", len);
+		errno = EMSGSIZE;
+		return -1;
+	}
+	if(!buf) {
+		DEBUG_ERROR("msg buf is null");
+		errno = EINVAL;
+		return -1;
+	}
 	ZeroTier::VirtualSocket *vs = get_virtual_socket(fd);
 	if(!vs) {
 		DEBUG_ERROR("no vs found for fd=%x", fd);
@@ -1308,11 +1334,26 @@ ssize_t zts_sendto(ZT_SENDTO_SIG)
 									socket.  In this case, the process will also receive a SIGPIPE
 									unless MSG_NOSIGNAL is set.
 
+	ZT_SEND_SIG int fd, const void *buf, size_t len, int flags
 */
+
 ssize_t zts_send(ZT_SEND_SIG)
 {
-	DEBUG_TRANS("fd=%d", fd);
+	// DEBUG_TRANS("fd=%d", fd);
 	int err = errno = 0;
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
+		errno = EBADF;
+		return -1;
+	}
+	if(len == 0) {
+		return 0;
+	}
+	if(len > ZT_SOCKET_MSG_BUF_SZ) {
+		DEBUG_ERROR("msg is too long to be sent atomically (len=%d)", len);
+		errno = EMSGSIZE;
+		return -1;
+	}
+
 	ZeroTier::VirtualSocket *vs = get_virtual_socket(fd);
 	if(!vs) {
 		DEBUG_ERROR("invalid vs for fd=%d", fd);
@@ -1381,9 +1422,9 @@ ssize_t zts_sendmsg(ZT_SENDMSG_SIG)
 {
 	DEBUG_TRANS("fd=%d", fd);	
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	else {
 		err = sendmsg(fd, msg, flags);
@@ -1414,7 +1455,10 @@ ssize_t zts_sendmsg(ZT_SENDMSG_SIG)
 	[  ] ENOMEM 					Could not allocate memory for recvmsg().
 	[  ] ENOTCONN 					The socket is associated with a connection-oriented protocol and has not been connected (see connect(2) and accept(2)).
 	[  ] ENOTSOCK 					The file descriptor sockfd does not refer to a socket.
+
+	ZT_RECV_SIG int fd, void *buf, size_t len, int flags
 */
+
 ssize_t zts_recv(ZT_RECV_SIG)
 {
 	DEBUG_TRANS("fd=%d", fd);
@@ -1502,47 +1546,85 @@ ssize_t zts_recv(ZT_RECV_SIG)
 	[  ] [ENOTCONN] 				The socket is associated with a connection-oriented protocol 
 									and has not been connected (see connect(2) and accept(2)).
 	[  ] [ENOTSOCK] 				The argument sockfd does not refer to a socket.
+
+	ZT_RECVFROM_SIG int fd, void *buf, size_t len, int flags, struct sockaddr *addr, socklen_t *addrlen
 */
+
 ssize_t zts_recvfrom(ZT_RECVFROM_SIG)
 {
-	DEBUG_TRANS("fd=%d", fd);
+	//DEBUG_TRANS("fd=%d", fd);
 	int32_t r = 0;
 	errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
 		return -1;
 	}
+	if(len == 0) {
+		return 0;
+	}
+	if(!buf) {
+		DEBUG_ERROR("buf is null");
+		errno = EINVAL;
+		return -1;
+	}
 	char udp_msg_buf[ZT_SOCKET_MSG_BUF_SZ];
 	char *msg_ptr = udp_msg_buf;
-	r = read(fd, msg_ptr, sizeof(udp_msg_buf));
-	if(r > 0) {
-		*addrlen = sizeof(struct sockaddr_storage);
+	memset(msg_ptr, 0, sizeof(int32_t)); // zero only len portion
 
-		// get message length
-		int32_t udp_msg_len = 0;
-		memcpy(&udp_msg_len, msg_ptr, sizeof(udp_msg_len));
-		msg_ptr+=sizeof(int32_t);
+	int32_t udp_msg_len = 0;
+	
+	// PEEK at the buffer and see if we can read a length, if not, err out
+	r = recv(fd, msg_ptr, sizeof(int32_t), MSG_PEEK);
+	if(r != sizeof(int32_t)){
+		//DEBUG_ERROR("invalid datagram, PEEK, r=%d", r);
+		errno = EIO; // TODO: test for this
+		return -1;
+	}
+	// read of sizeof(int32_t) for the length of the datagram (including address)
+	r = read(fd, msg_ptr, sizeof(int32_t));
+	// copy to length variable
+	memcpy(&udp_msg_len, msg_ptr, sizeof(int32_t));
+	msg_ptr+=sizeof(int32_t);
 
-		// get address
+	if(udp_msg_len <= 0) {
+		DEBUG_ERROR("invalid datagram");
+		errno = EIO; // TODO: test for this
+		return -1;
+	}
+	// there is a datagram to read, so let's read it
+	// zero remainder of buffer
+	memset(msg_ptr, 0, ZT_SOCKET_MSG_BUF_SZ- sizeof(int32_t));
+	if((r = read(fd, msg_ptr, udp_msg_len)) < 0) {
+		DEBUG_ERROR("invalid datagram");
+		errno = EIO; // TODO: test for this
+		return -1;
+	}
+	// get address
+	if(addr) {
+		if(*addrlen < sizeof(struct sockaddr_storage)) {
+			DEBUG_ERROR("invalid address length provided");
+			errno = EINVAL;
+			return -1;
+		}
+		*addrlen = sizeof(struct sockaddr_storage);
 		memcpy(addr, msg_ptr, *addrlen);
-		msg_ptr+=*addrlen;
-		
-		// get payload
-		int32_t payload_sz = udp_msg_len - *addrlen;
-		memcpy(buf, msg_ptr, payload_sz);
-		r = payload_sz;
 	}
-	return r;
+	msg_ptr+=sizeof(struct sockaddr_storage);
+	// get payload
+	int32_t payload_sz = udp_msg_len - *addrlen;
+	int32_t write_sz = len < payload_sz ? len : payload_sz;
+	memcpy(buf, msg_ptr, write_sz);
+	return write_sz;
 }
 
 // TODO
 ssize_t zts_recvmsg(ZT_RECVMSG_SIG)
 {
-	DEBUG_TRANS("fd=%d", fd);
+	//DEBUG_TRANS("fd=%d", fd);
 	int err = errno = 0;
-	if(fd < 0) {
+	if(fd < 0 || fd >= ZT_MAX_SOCKETS) {
 		errno = EBADF;
-		err = -1;
+		return -1;
 	}
 	else {
 		err = recvmsg(fd, msg, flags);
@@ -1551,12 +1633,12 @@ ssize_t zts_recvmsg(ZT_RECVMSG_SIG)
 }
 
 int zts_read(ZT_READ_SIG) {
-	DEBUG_TRANS("fd=%d", fd);
+	//DEBUG_TRANS("fd=%d", fd);
 	return read(fd, buf, len);
 }
 
 int zts_write(ZT_WRITE_SIG) {
-	DEBUG_TRANS("fd=%d", fd);
+	//DEBUG_TRANS("fd=%d", fd);
 	return write(fd, buf, len);
 }
 

src/lwIP.cpp

@@ -51,7 +51,6 @@ err_t tapif_init(struct netif *netif)
 
 err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 {
-	DEBUG_INFO();
 	struct pbuf *q;
 	char buf[ZT_MAX_MTU+32];
 	char *bufptr;
@@ -76,6 +75,19 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 
 	tap->_handler(tap->_arg,NULL,tap->_nwid,src_mac,dest_mac,
 		ZeroTier::Utils::ntoh((uint16_t)ethhdr->type),0,buf + sizeof(struct eth_hdr),totalLength - sizeof(struct eth_hdr));
+
+	if(ZT_DEBUG_LEVEL >= ZT_MSG_TRANSFER) {
+		char flagbuf[32];
+		memset(&flagbuf, 0, 32);
+		char macBuf[ZT_MAC_ADDRSTRLEN], nodeBuf[ZT_ID_LEN];
+		mac2str(macBuf, ZT_MAC_ADDRSTRLEN, ethhdr->dest.addr);
+		ZeroTier::MAC mac;
+		mac.setTo(ethhdr->dest.addr, 6);
+		mac.toAddress(tap->_nwid).toString(nodeBuf);
+		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(), 
+			ZeroTier::Utils::ntoh(ethhdr->type), beautify_eth_proto_nums(ZeroTier::Utils::ntoh(ethhdr->type)), flagbuf);
+	}
+
 	return ERR_OK;
 }
 
@@ -89,7 +101,7 @@ namespace ZeroTier
 		if (std::find(tap->_ips.begin(),tap->_ips.end(),ip) == tap->_ips.end()) {
 			tap->_ips.push_back(ip);
 			std::sort(tap->_ips.begin(),tap->_ips.end());
-			char ipbuf[64], nmbuf[64];
+			char ipbuf[INET6_ADDRSTRLEN], nmbuf[INET6_ADDRSTRLEN];
 #if defined(LIBZT_IPV4)
 			if (ip.isV4()) {
 				// Set IP
@@ -140,12 +152,22 @@ namespace ZeroTier
 		}
 	}
 
+	int lwIP::lwip_add_dns_nameserver(struct sockaddr *addr)
+	{
+		return -1;
+	}
+
+	int lwIP::lwip_del_dns_nameserver(struct sockaddr *addr)
+	{
+		return -1;
+	}
+
 	void lwIP::lwip_loop(VirtualTap *tap)
 	{
-	   DEBUG_INFO();
-	   uint64_t prev_tcp_time = 0, prev_discovery_time = 0;
-	   while(tap->_run)
-	   {
+		// DEBUG_INFO();
+		uint64_t prev_tcp_time = 0, prev_discovery_time = 0;
+		while(tap->_run)
+		{
 			uint64_t now = OSUtils::now();
 			uint64_t since_tcp = now - prev_tcp_time;
 			uint64_t since_discovery = now - prev_discovery_time;
@@ -183,7 +205,6 @@ namespace ZeroTier
 
 	void lwIP::lwip_eth_rx(VirtualTap *tap, const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len)
 	{
-		DEBUG_INFO("etherType=%x, len=%d", etherType, len);
 		struct pbuf *p,*q;
 		if (!tap->_enabled)
 			return;
@@ -211,6 +232,19 @@ namespace ZeroTier
 				dataptr += q->len;
 			}
 		} 
+
+		if(ZT_DEBUG_LEVEL >= ZT_MSG_TRANSFER) {
+			char flagbuf[32];
+			memset(&flagbuf, 0, 32);
+			char macBuf[ZT_MAC_ADDRSTRLEN], nodeBuf[ZT_ID_LEN];
+			mac2str(macBuf, ZT_MAC_ADDRSTRLEN, ethhdr.dest.addr);
+			ZeroTier::MAC mac;
+			mac.setTo(ethhdr.dest.addr, 6);
+			mac.toAddress(tap->_nwid).toString(nodeBuf);
+			DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] proto=0x%04x %s %s", len, macBuf, nodeBuf, tap->nodeId().c_str(), 
+				ZeroTier::Utils::ntoh(ethhdr.type), beautify_eth_proto_nums(ZeroTier::Utils::ntoh(ethhdr.type)), flagbuf);
+		}
+
 		else {
 			DEBUG_ERROR("dropped packet: no pbufs available");
 			return;
@@ -260,7 +294,7 @@ namespace ZeroTier
 			struct sockaddr_in *in4 = (struct sockaddr_in *)addr;
 			if(addr->sa_family == AF_INET) {
 				inet_ntop(AF_INET, &(in4->sin_addr), addrstr, INET_ADDRSTRLEN); 
-				DEBUG_INFO("%s:%d", addrstr, lwip_ntohs(in4->sin_port));
+				DEBUG_EXTRA("connecting to %s : %d", addrstr, lwip_ntohs(in4->sin_port));
 			}
 			ba = convert_ip(in4); 
 			port = lwip_ntohs(in4->sin_port);
@@ -271,36 +305,33 @@ namespace ZeroTier
 			if(addr->sa_family == AF_INET6) {        
 				struct sockaddr_in6 *vsaddr6 = (struct sockaddr_in6 *)addr;
 				inet_ntop(AF_INET6, &(in6->sin6_addr), addrstr, INET6_ADDRSTRLEN);
-				DEBUG_INFO("%s:%d", addrstr, lwip_ntohs(in6->sin6_port));
+				DEBUG_EXTRA("connecting to %s : %d", addrstr, lwip_ntohs(in6->sin6_port));
 			}
 #endif
-
-		DEBUG_INFO("addr=%s", addrstr);
-
 		if(vs->socket_type == SOCK_DGRAM) {
 			// Generates no network traffic
 			if((err = udp_connect((struct udp_pcb*)vs->pcb,(ip_addr_t *)&ba,port)) < 0) {
 				DEBUG_ERROR("error while connecting to with UDP");
 			}
-			udp_recv((struct udp_pcb*)vs->pcb, nc_udp_recved, vs);
+			udp_recv((struct udp_pcb*)vs->pcb, lwip_cb_udp_recved, vs);
 			return ERR_OK;
 		}
 
 		if(vs->socket_type == SOCK_STREAM) {
 			struct tcp_pcb *tpcb = (struct tcp_pcb*)vs->pcb;
-			tcp_sent(tpcb, nc_sent);
-			tcp_recv(tpcb, nc_recved);
-			tcp_err(tpcb, nc_err);
-			tcp_poll(tpcb, nc_poll, LWIP_APPLICATION_POLL_FREQ);
+			tcp_sent(tpcb, lwip_cb_sent);
+			tcp_recv(tpcb, lwip_cb_tcp_recved);
+			tcp_err(tpcb, lwip_cb_err);
+			tcp_poll(tpcb, lwip_cb_poll, LWIP_APPLICATION_POLL_FREQ);
 			tcp_arg(tpcb, vs);
 				
 			//DEBUG_EXTRA(" pcb->state=%x", vs->TCP_pcb->state);
 			//if(vs->TCP_pcb->state != CLOSED) {
-			//	DEBUG_INFO(" cannot connect using this PCB, PCB!=CLOSED");
-			//	tap->sendReturnValue(tap->_phy.getDescriptor(rpcSock), -1, EAGAIN);
-			//	return;
+			//  DEBUG_INFO(" cannot connect using this PCB, PCB!=CLOSED");
+			//  tap->sendReturnValue(tap->_phy.getDescriptor(rpcSock), -1, EAGAIN);
+			//  return;
 			//}
-			if((err = tcp_connect(tpcb,&ba,port,nc_connected)) < 0)
+			if((err = tcp_connect(tpcb,&ba,port,lwip_cb_connected)) < 0)
 			{
 				if(err == ERR_ISCONN) {
 					// Already in connected state
@@ -334,7 +365,7 @@ namespace ZeroTier
 				//   that the SYN packet was enqueued onto the stack properly,
 				//   that's it!
 				// - Most instances of a retval for a connect() should happen
-				//   in the nc_connect() and nc_err() callbacks!
+				//   in the nc_connect() and lwip_cb_err() callbacks!
 				DEBUG_ERROR("unable to connect");
 				errno = EAGAIN;
 				return -1;
@@ -354,7 +385,7 @@ namespace ZeroTier
 			struct sockaddr_in *in4 = (struct sockaddr_in *)addr;
 			if(addr->sa_family == AF_INET) {
 				inet_ntop(AF_INET, &(in4->sin_addr), addrstr, INET_ADDRSTRLEN); 
-				DEBUG_INFO("%s:%d", addrstr, lwip_ntohs(in4->sin_port));
+				DEBUG_EXTRA("binding to %s : %d", addrstr, lwip_ntohs(in4->sin_port));
 			}
 			ba = convert_ip(in4); 
 			port = lwip_ntohs(in4->sin_port);
@@ -365,7 +396,7 @@ namespace ZeroTier
 			if(addr->sa_family == AF_INET6) {        
 				struct sockaddr_in6 *vsaddr6 = (struct sockaddr_in6 *)addr;
 				inet_ntop(AF_INET6, &(in6->sin6_addr), addrstr, INET6_ADDRSTRLEN);
-				DEBUG_INFO("%s:%d", addrstr, lwip_ntohs(in6->sin6_port));
+				DEBUG_EXTRA("binding to %s : %d", addrstr, lwip_ntohs(in6->sin6_port));
 			}
 #endif
 		if(vs->socket_type == SOCK_DGRAM) {
@@ -376,7 +407,7 @@ namespace ZeroTier
 			}
 			else {
 				// set the recv callback
-				udp_recv((struct udp_pcb*)vs->pcb, nc_udp_recved, new VirtualBindingPair(tap, vs));
+				udp_recv((struct udp_pcb*)vs->pcb, lwip_cb_udp_recved, new VirtualBindingPair(tap, vs));
 				err = ERR_OK; 
 				errno = ERR_OK; // success
 			}
@@ -417,7 +448,7 @@ namespace ZeroTier
 #endif
 		if(listeningPCB != NULL) {
 			vs->pcb = listeningPCB;
-			tcp_accept(listeningPCB, nc_accept); // set callback
+			tcp_accept(listeningPCB, lwip_cb_accept); // set callback
 			tcp_arg(listeningPCB, vs);
 			//fcntl(tap->_phy.getDescriptor(vs->sock), F_SETFL, O_NONBLOCK);
 		}
@@ -446,7 +477,7 @@ namespace ZeroTier
 		DEBUG_EXTRA("vs=%p", vs);
 		int err = 0;
 		if(!vs) {
-			DEBUG_ERROR("no VirtualSocket");
+			DEBUG_ERROR("no virtual socket");
 			return -1;
 		}
 		if(!lwip_invoked) {
@@ -487,14 +518,14 @@ namespace ZeroTier
 		DEBUG_EXTRA("vs=%p, len=%d", (void*)&vs, len);
 		int err = 0;
 		if(!vs) {
-			DEBUG_ERROR("no VirtualSocket");
+			DEBUG_ERROR("no virtual socket");
 			return -1;
 		}
 		if(vs->socket_type == SOCK_DGRAM) 
 		{
 			DEBUG_ERROR("socket_type==SOCK_DGRAM");
 			// TODO: Packet re-assembly hasn't yet been tested with lwIP so UDP packets are limited to MTU-sized chunks
-            int udp_trans_len = std::min((ssize_t)vs->TXbuf->count(), (ssize_t)ZT_MAX_MTU);
+			int udp_trans_len = std::min((ssize_t)vs->TXbuf->count(), (ssize_t)ZT_MAX_MTU);
 			DEBUG_EXTRA("allocating pbuf chain of size=%d for UDP packet, txsz=%d", udp_trans_len, vs->TXbuf->count());
 			struct pbuf * pb = pbuf_alloc(PBUF_TRANSPORT, udp_trans_len, PBUF_POOL);
 			if(!pb){
@@ -524,7 +555,7 @@ namespace ZeroTier
 			int err, r;
 			if(!sndbuf) {
 				// PCB send buffer is full, turn off readability notifications for the
-				// corresponding PhySocket until nc_sent() is called and confirms that there is
+				// corresponding PhySocket until lwip_cb_sent() is called and confirms that there is
 				// now space on the buffer
 				DEBUG_ERROR("lwIP stack is full, sndbuf == 0");
 				vs->tap->_phy.setNotifyReadable(vs->sock, false);
@@ -571,7 +602,7 @@ namespace ZeroTier
 		// FIXME: check if already closed? vs->TCP_pcb->state != CLOSED
 		if(vs->pcb) {
 			//DEBUG_EXTRA("vs=%p, sock=%p, PCB->state = %d", 
-			//	(void*)&conn, (void*)&sock, vs->TCP_pcb->state);
+			//  (void*)&conn, (void*)&sock, vs->TCP_pcb->state);
 			if(((struct tcp_pcb*)vs->pcb)->state == SYN_SENT /*|| vs->TCP_pcb->state == CLOSE_WAIT*/) {
 				DEBUG_EXTRA("ignoring close request. invalid PCB state for this operation. sock=%p", vs->sock);
 				return -1;
@@ -596,7 +627,7 @@ namespace ZeroTier
 	/* Callbacks from lwIP stack                                                */
 	/****************************************************************************/
 
-	err_t lwIP::nc_recved(void *arg, struct tcp_pcb *PCB, struct pbuf *p, err_t err)
+	err_t lwIP::lwip_cb_tcp_recved(void *arg, struct tcp_pcb *PCB, struct pbuf *p, err_t err)
 	{
 		DEBUG_INFO();
 		VirtualSocket *vs = (VirtualSocket *)arg;
@@ -649,12 +680,11 @@ namespace ZeroTier
 		return ERR_OK;
 	}
 
-	err_t lwIP::nc_accept(void *arg, struct tcp_pcb *newPCB, err_t err)
+	err_t lwIP::lwip_cb_accept(void *arg, struct tcp_pcb *newPCB, err_t err)
 	{
 		VirtualSocket *vs = (VirtualSocket*)arg;
 		DEBUG_INFO("vs=%p", vs);
 		//Mutex::Lock _l(vs->tap->_tcpconns_m);
-		// create and populate new VirtualSocket object
 		VirtualSocket *new_vs = new VirtualSocket();
 		new_vs->socket_type = SOCK_STREAM;
 		new_vs->pcb = newPCB;
@@ -666,22 +696,22 @@ namespace ZeroTier
 		vs->_AcceptedConnections.push(new_vs);
 		// set callbacks
 		tcp_arg(newPCB, new_vs);
-		tcp_recv(newPCB, nc_recved);
-		tcp_err(newPCB, nc_err);
-		tcp_sent(newPCB, nc_sent);
-		tcp_poll(newPCB, nc_poll, 1);
+		tcp_recv(newPCB, lwip_cb_tcp_recved);
+		tcp_err(newPCB, lwip_cb_err);
+		tcp_sent(newPCB, lwip_cb_sent);
+		tcp_poll(newPCB, lwip_cb_poll, 1);
 		// let lwIP know that it can queue additional incoming VirtualSockets
 		tcp_accepted((struct tcp_pcb*)vs->pcb); 
 		return 0;
 	}
 		
-	void lwIP::nc_udp_recved(void * arg, struct udp_pcb * upcb, struct pbuf * p, const ip_addr_t * addr, u16_t port)
+	void lwIP::lwip_cb_udp_recved(void * arg, struct udp_pcb * upcb, struct pbuf * p, const ip_addr_t * addr, u16_t port)
 	{
 		DEBUG_INFO();
 		// to be implemented
 	}
 
-	err_t lwIP::nc_sent(void* arg, struct tcp_pcb *PCB, u16_t len)
+	err_t lwIP::lwip_cb_sent(void* arg, struct tcp_pcb *PCB, u16_t len)
 	{
 		DEBUG_EXTRA("pcb=%p", (void*)&PCB);
 		VirtualSocket *vs = (VirtualSocket *)arg;
@@ -696,7 +726,7 @@ namespace ZeroTier
 		return ERR_OK;
 	}
 
-	err_t lwIP::nc_connected(void *arg, struct tcp_pcb *PCB, err_t err)
+	err_t lwIP::lwip_cb_connected(void *arg, struct tcp_pcb *PCB, err_t err)
 	{
 		DEBUG_ATTN("pcb=%p", (void*)&PCB);
 		VirtualSocket *vs = (VirtualSocket *)arg;
@@ -706,12 +736,12 @@ namespace ZeroTier
 		// FIXME: check stack for expected return values
 	}
 
-	err_t lwIP::nc_poll(void* arg, struct tcp_pcb *PCB)
+	err_t lwIP::lwip_cb_poll(void* arg, struct tcp_pcb *PCB)
 	{
 		return ERR_OK;
 	}
 
-	void lwIP::nc_err(void *arg, err_t err)
+	void lwIP::lwip_cb_err(void *arg, err_t err)
 	{
 		DEBUG_ERROR("err=%d", err);
 		VirtualSocket *vs = (VirtualSocket *)arg;

src/lwIP.hpp

@@ -174,6 +174,16 @@ namespace ZeroTier {
 		 */
 		void lwip_init_interface(VirtualTap *tap, const InetAddress &ip);
 
+		/*
+		 * Registers a DNS nameserver with the network stack
+		 */
+		int lwip_add_dns_nameserver(struct sockaddr *addr);
+
+		/*
+		 * Un-registers a DNS nameserver from the network stack
+		 */
+		int lwip_del_dns_nameserver(struct sockaddr *addr);
+
 		/*
 		 * Main stack loop
 		 */
@@ -184,22 +194,84 @@ namespace ZeroTier {
 		 */
 		void lwip_eth_rx(VirtualTap *tap, const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
 		
+		/*
+		 * Creates a stack-specific "socket" or "VirtualSocket object"
+		 */
 		int lwip_Socket(void **pcb, int socket_family, int socket_type, int protocol);
+
+		/*
+		 * Connect to remote host via userspace network stack interface - Called from VirtualTap
+		 */
 		int lwip_Connect(VirtualSocket *vs, const struct sockaddr *addr, socklen_t addrlen);
+
+		/*
+		 * Bind to a userspace network stack interface - Called from VirtualTap
+		 */
 		int lwip_Bind(VirtualTap *tap, VirtualSocket *vs, const struct sockaddr *addr, socklen_t addrlen);
+
+		/*
+		 * Listen for incoming VirtualSockets - Called from VirtualTap
+		 */
 		int lwip_Listen(VirtualSocket *vs, int backlog);
+
+		/*
+		 * Accept an incoming VirtualSocket - Called from VirtualTap
+		 */
 		VirtualSocket* lwip_Accept(VirtualSocket *vs);
+
+		/*
+		 * Read from RX buffer to application - Called from VirtualTap
+		 */
 		int lwip_Read(VirtualSocket *vs, bool lwip_invoked);
+
+		/*
+		 * Write to userspace network stack - Called from VirtualTap
+		 */
 		int lwip_Write(VirtualSocket *vs, void *data, ssize_t len);
+
+		/*
+		 * Close a VirtualSocket - Called from VirtualTap
+		 */
 		int lwip_Close(VirtualSocket *vs);
 
-		static err_t nc_recved(void *arg, struct tcp_pcb *PCB, struct pbuf *p, err_t err);
-		static err_t nc_accept(void *arg, struct tcp_pcb *newPCB, err_t err);
-		static void nc_udp_recved(void * arg, struct udp_pcb * upcb, struct pbuf * p, const ip_addr_t * addr, u16_t port);
-		static void nc_err(void *arg, err_t err);
-		static err_t nc_poll(void* arg, struct tcp_pcb *PCB);
-		static err_t nc_sent(void *arg, struct tcp_pcb *PCB, u16_t len);
-		static err_t nc_connected(void *arg, struct tcp_pcb *PCB, err_t err);
+
+		// --- Callbacks from network stack ---
+
+
+		/*
+		 * Callback for handling received UDP packets (already processed by network stack)
+		 */
+		static err_t lwip_cb_tcp_recved(void *arg, struct tcp_pcb *PCB, struct pbuf *p, err_t err);
+
+		/*
+		 * Callback for handling accepted connection
+		 */
+		static err_t lwip_cb_accept(void *arg, struct tcp_pcb *newPCB, err_t err);
+
+		/*
+		 * Callback for handling received TCP packets (already processed by stack)
+		 */
+		static void lwip_cb_udp_recved(void * arg, struct udp_pcb * upcb, struct pbuf * p, const ip_addr_t * addr, u16_t port);
+
+		/*
+		 * Callback for handling errors from within the network stack
+		 */
+		static void lwip_cb_err(void *arg, err_t err);
+
+		/*
+		 * Callback for handling periodic background tasks
+		 */
+		static err_t lwip_cb_poll(void* arg, struct tcp_pcb *PCB);
+
+		/*
+		 * Callback for handling confirmation of sent packets
+		 */
+		static err_t lwip_cb_sent(void *arg, struct tcp_pcb *PCB, u16_t len);
+
+		/*
+		 * Callback for handling successful connections
+		 */
+		static err_t lwip_cb_connected(void *arg, struct tcp_pcb *PCB, err_t err);
 	};
 } 
 

src/picoTCP.cpp

@@ -391,7 +391,7 @@ namespace ZeroTier {
 		int txsz = vs->TXbuf->count();
 		if(txsz <= 0)
 			return;
-		//DEBUG_INFO("TXbuf->count() = %d", vs->TXbuf->count());
+		//DEBUG_INFO("TXbuf->count()=%d", vs->TXbuf->count());
 
 		int r, max_write_len = std::min(std::min(txsz, ZT_SDK_MTU),ZT_STACK_SOCKET_WR_MAX);
 		if((r = pico_socket_write(vs->picosock, vs->TXbuf->get_buf(), max_write_len)) < 0) {
@@ -447,7 +447,7 @@ namespace ZeroTier {
 				DEBUG_ERROR("pico_socket_close()=%d, pico_err=%d, %s", err, pico_err, beautify_pico_error(pico_err));
 			}
 			DEBUG_EXTRA("PICO_SOCK_EV_CLOSE (socket closure) err=%d (%s), picosock=%p", pico_err, beautify_pico_error(pico_err), s);
-			//DEBUG_EXTRA("PICO_SOCK_EV_CLOSE (socket closure) err = %d, picosock=%p, vs=%p, app_fd=%d, sdk_fd=%d", err, s, vs, vs->app_fd, vs->sdk_fd);            
+			//DEBUG_EXTRA("PICO_SOCK_EV_CLOSE (socket closure) err=%d, picosock=%p, vs=%p, app_fd=%d, sdk_fd=%d", err, s, vs, vs->app_fd, vs->sdk_fd);            
 			//vs->closure_ts = std::time(nullptr);	
 			return;
 		}
@@ -591,17 +591,9 @@ namespace ZeroTier {
 			{
 				tcp_hdr_ptr = &buf + PICO_SIZE_ETHHDR + PICO_SIZE_IP4HDR;
 				hdr = (struct pico_tcp_hdr *)tcp_hdr_ptr;
-
-				/*
-					ext/picotcp/build/include/pico_tcp.h:#define PICO_TCP_SYNACK    (PICO_TCP_SYN | PICO_TCP_ACK)
-					ext/picotcp/build/include/pico_tcp.h:#define PICO_TCP_PSHACK    (PICO_TCP_PSH | PICO_TCP_ACK)
-					ext/picotcp/build/include/pico_tcp.h:#define PICO_TCP_FINACK    (PICO_TCP_FIN | PICO_TCP_ACK)
-					ext/picotcp/build/include/pico_tcp.h:#define PICO_TCP_FINPSHACK (PICO_TCP_FIN | PICO_TCP_PSH | PICO_TCP_ACK)
-					ext/picotcp/build/include/pico_tcp.h:#define PICO_TCP_RSTACK    (PICO_TCP_RST | PICO_TCP_ACK)
-				*/
 				
 				if(hdr) {
-	 				char *flag_ptr = flagbuf;
+					char *flag_ptr = flagbuf;
 
 					if (hdr->flags & PICO_TCP_PSH) {
 						sprintf(flag_ptr, "PSH ");
@@ -991,10 +983,10 @@ namespace ZeroTier {
 				vs->TXbuf->consume(r);
 			}
 			if(vs->socket_type == SOCK_STREAM) {
-				DEBUG_TRANS("len=%5d, [app(buf) --> network_stack(vs=%p)] proto=0x%04x (TCP)", r, vs, PICO_PROTO_TCP);
+				DEBUG_TRANS("len=%5d [app(buf) --> network_stack(vs=%p)] proto=0x%04x (TCP)", r, vs, PICO_PROTO_TCP);
 			}
 			if(vs->socket_type == SOCK_DGRAM) {
-				DEBUG_TRANS("len=%5d, [app(buf) --> network_stack(vs=%p)] proto=0x%04x (TCP)", r, vs, PICO_PROTO_UDP);
+				DEBUG_TRANS("len=%5d [app(buf) --> network_stack(vs=%p)] proto=0x%04x (TCP)", r, vs, PICO_PROTO_UDP);
 			}
 		}
 		return err;
@@ -1016,8 +1008,8 @@ namespace ZeroTier {
 			return ZT_ERR_OK;
 		if((err = pico_socket_close(vs->picosock)) < 0) {
 			errno = pico_err;
-			DEBUG_ERROR("error closing pico_socket(%p), err=%d, pico_err=%s, %s", 
-				(void*)(vs->picosock), err, pico_err, beautify_pico_error(pico_err));
+			DEBUG_ERROR("error closing pico_socket, err=%d, pico_err=%s, %s", 
+				err, pico_err, beautify_pico_error(pico_err));
 		}
 		return err;
 	}

test/generate_test_env.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# create dirs to house the identities/config
-mkdir alice
-mkdir bob
-mkdir ted 
-mkdir carol
-
-# generate identities
-zerotier-one alice -d
-echo $! >> "zto.alice"
-zerotier-one bob -d
-echo $! >> "zto.bob"
-zerotier-one ted -d
-echo $! >> "zto.ted"
-zerotier-one carol -d
-echo $! >> "zto.carol"
-
-# should be done by now
-sleep(30)
-
-# kill daemons
-echo "killing daemons"
-
-pid=$(cat alice/zto.alice)
-kill -9 $pid
-pid=$(cat bob/zto.bob)
-kill -9 $pid
-pid=$(cat ted/zto.ted)
-kill -9 $pid
-pid=$(cat carol/zto.carol)
-kill -9 $pid

test/stop.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-if [[ $1 == "" ]]
-then
-	echo "Usage: stop <alice|bob>."
-	exit
-fi
-
-echo "killing $1"
-
-for f in "test/*.$1";
-do
-	pid=$(cat $f)
-	kill -9 $pid
-	rm -rf $f
-done