Performance tweaks to lwipopts.h (TX=12MB/s)

docker-test/throughput-test/Dockerfile

@@ -2,18 +2,14 @@
 FROM fedora:22
 MAINTAINER https://www.zerotier.com/
 
-# Install Apache
+# Install apps
-RUN yum -y update && yum clean all
+RUN yum -y update
-RUN yum -y install httpd && yum clean all
+RUN yum -y install httpd
+RUN yum -y install nano
+RUN yum clean all
 
 EXPOSE 9993/udp 9992/udp 9991/udp 9990/udp
 
-# Add ZT files
-RUN mkdir -p /var/lib/zerotier-one/networks.d
-RUN touch /var/lib/zerotier-one/networks.d/e5cd7a9e1c5311ab.conf
-ADD zerotier-one /
-ADD zerotier-cli /
-
 # Install sys-call intercept library
 ADD intercept /
 ADD libintercept.so.1.0 /
@@ -22,6 +18,12 @@ RUN cp libintercept.so.1.0 /lib/libintercept.so.1.0
 RUN ln -sf /lib/libintercept.so.1.0 /lib/libintercept
 RUN /usr/bin/install -c intercept /usr/bin
 
+# Add ZT files
+RUN mkdir -p /var/lib/zerotier-one/networks.d
+RUN touch /var/lib/zerotier-one/networks.d/e5cd7a9e1c5311ab.conf
+ADD zerotier-one /
+ADD zerotier-cli /
+
 # Install test-setup scripts
 ADD generate_file.sh /generate_file.sh
 RUN chmod -v +x /generate_file.sh

docker-test/throughput-test/build.sh

@@ -1,4 +1,5 @@
 cd ../../
+make clean
 make
 cd netcon
 make -f make-intercept.mk lib

docker-test/zerotier-cli

@@ -1 +0,0 @@
-zerotier-one

ext/lwipopts.h

@@ -44,7 +44,10 @@
  */
 #include "lwip/debug.h"
 
-#define TCP_MSS 1400
+#define LWIP_CHKSUM_ALGORITHM 2
+
+
+#define TCP_MSS 1460
 
 /*
 The TCP window size can be adjusted by changing the define TCP_WND. However,
@@ -119,7 +122,11 @@ happening sooner than they should.
 ------------------------------------------------------------------------------*/
 
 
-#define LWIP_CHKSUM_ALGORITHM 2
+
+/* Misc */
+#define MEM_LIBC_MALLOC 1
+#define MEMP_MEM_MALLOC 1
+
 
 
 /**
@@ -137,7 +144,28 @@ happening sooner than they should.
 #define TCP_SND_BUF                     1024 * 63
 //#define TCP_OVERSIZE                    TCP_MSS
 
-#define TCP_SND_QUEUELEN                (2 * TCP_SND_BUF/TCP_MSS)
+#define TCP_SND_QUEUELEN                1024
+
+/*------------------------------------------------------------------------------
+-------------------------------- Pbuf Options ----------------------------------
+------------------------------------------------------------------------------*/
+
+/**
+ * PBUF_LINK_HLEN: the number of bytes that should be allocated for a
+ * link level header. The default is 14, the standard value for
+ * Ethernet.
+ */
+#define PBUF_LINK_HLEN                  16
+
+/**
+ * PBUF_POOL_BUFSIZE: the size of each pbuf in the pbuf pool. The default is
+ * designed to accomodate single full size TCP frame in one pbuf, including
+ * TCP_MSS, IP header, and link header.
+*
+ */
+#define PBUF_POOL_BUFSIZE               LWIP_MEM_ALIGN_SIZE(TCP_MSS+40+PBUF_LINK_HLEN)
+
+
 /*------------------------------------------------------------------------------
 -------------------------- Internal Memory Pool Sizes --------------------------
 ------------------------------------------------------------------------------*/
@@ -397,26 +425,6 @@ happening sooner than they should.
 #define LWIP_LISTEN_BACKLOG             0
 
 
-/*------------------------------------------------------------------------------
--------------------------------- Pbuf Options ----------------------------------
-------------------------------------------------------------------------------*/
-
-/**
- * PBUF_LINK_HLEN: the number of bytes that should be allocated for a
- * link level header. The default is 14, the standard value for
- * Ethernet.
- */
-#define PBUF_LINK_HLEN                  16
-
-/**
- * PBUF_POOL_BUFSIZE: the size of each pbuf in the pbuf pool. The default is
- * designed to accomodate single full size TCP frame in one pbuf, including
- * TCP_MSS, IP header, and link header.
-*
- */
-#define PBUF_POOL_BUFSIZE               LWIP_MEM_ALIGN_SIZE(TCP_MSS+40+PBUF_LINK_HLEN)
-
-
 /*------------------------------------------------------------------------------
 --------------------------------- LOOPIF Options -------------------------------
 ------------------------------------------------------------------------------*/
@@ -453,7 +461,7 @@ happening sooner than they should.
 /**
  * LWIP_STATS==1: Enable statistics collection in lwip_stats.
  */
-#define LWIP_STATS                      1
+#define LWIP_STATS                      0
 
 /*------------------------------------------------------------------------------
 --------------------------------- PPP Options ----------------------------------
@@ -464,10 +472,4 @@ happening sooner than they should.
  */
 #define PPP_SUPPORT                     0
 
-
-/* Misc */
-
-
-#define MEM_LIBC_MALLOC 1
-#define MEMP_MEM_MALLOC 0
 #endif /* __LWIPOPTS_H__ */

netcon/NetconEthernetTap.cpp

@@ -50,7 +50,7 @@
 #include "NetconUtilities.hpp"
 
 #define APPLICATION_POLL_FREQ 				20
-#define ZT_LWIP_TCP_TIMER_INTERVAL 		50
+#define ZT_LWIP_TCP_TIMER_INTERVAL 		5
 #define STATUS_TMR_INTERVAL						2000 // How often we check connection statuses
 #define DEBUG_LEVEL 									3
 
@@ -418,22 +418,26 @@ void NetconEthernetTap::threadMain()
 
 
 	fprintf(stderr, "- MEM_SIZE = %dM\n", MEM_SIZE / (1024*1024));
-	fprintf(stderr, "- TCP_SND_BUF = %dK\n", TCP_SND_BUF / 1024);
+	fprintf(stderr, "- PBUF_POOL_SIZE = %d\n", PBUF_POOL_SIZE);
+	fprintf(stderr, "- PBUF_POOL_BUFSIZE  = %d\n", PBUF_POOL_BUFSIZE);
 	fprintf(stderr, "- MEMP_NUM_PBUF = %d\n", MEMP_NUM_PBUF);
 	fprintf(stderr, "- MEMP_NUM_TCP_PCB = %d\n", MEMP_NUM_TCP_PCB);
 	fprintf(stderr, "- MEMP_NUM_TCP_PCB_LISTEN = %d\n", MEMP_NUM_TCP_PCB_LISTEN);
-	fprintf(stderr, "- MEMP_NUM_TCP_SEG = %d\n", MEMP_NUM_TCP_SEG);
+	fprintf(stderr, "- MEMP_NUM_TCP_SEG = %d\n\n", MEMP_NUM_TCP_SEG);
-	fprintf(stderr, "- PBUF_POOL_SIZE = %d\n", PBUF_POOL_SIZE);
+
-	fprintf(stderr, "- TCP_SND_QUEUELEN = %d\n", TCP_SND_QUEUELEN);
+	fprintf(stderr, "- TCP_SND_BUF = %dK\n", TCP_SND_BUF / 1024);
-	fprintf(stderr, "- TCP_MAXRTX = %d\n", TCP_MAXRTX);
+	fprintf(stderr, "- TCP_SND_QUEUELEN = %d\n\n", TCP_SND_QUEUELEN);
-	fprintf(stderr, "- IP_REASSEMBLY = %d\n", IP_REASSEMBLY);
+
 	fprintf(stderr, "- TCP_WND = %d\n", TCP_WND);
 	fprintf(stderr, "- TCP_MSS = %d\n", TCP_MSS);
+	fprintf(stderr, "- TCP_MAXRTX = %d\n", TCP_MAXRTX);
+	fprintf(stderr, "- IP_REASSEMBLY = %d\n\n", IP_REASSEMBLY);
 	fprintf(stderr, "- ARP_TMR_INTERVAL = %d\n", ARP_TMR_INTERVAL);
 	fprintf(stderr, "- TCP_TMR_INTERVAL = %d\n", TCP_TMR_INTERVAL);
 	fprintf(stderr, "- IP_TMR_INTERVAL  = %d\n", IP_TMR_INTERVAL);
 
 
+
 	// Main timer loop
 	while (_run) {
 		uint64_t now = OSUtils::now();
@@ -929,6 +933,7 @@ err_t NetconEthernetTap::nc_sent(void* arg, struct tcp_pcb *tpcb, u16_t len)
 	if(len) {
 		l->conn->acked+=len;
 		//dwr("W = %d, A = %d\n", l->conn->written, l->conn->acked);
+		dwr("ACK = %d\n", len);
 		l->tap->_phy.setNotifyReadable(l->conn->dataSock, true);
 		l->tap->_phy.whack();
 	}

netcon/README.md

@@ -163,3 +163,11 @@ To Test:
  - sshd will enter an infinite accept loop maxing out CPU if service dies
 
 
+### Speed Notes
+
+http://lwip.100.n7.nabble.com/Performance-question-td4545.html
+ - 120-140Mbps, erroneous reports of 300-400 Mbps, claims linux stack is capable of 180Mbps
+
+
+ 
+