FIX Double Buffer issue, improved FileBuffer indexing

TermTk/TTkCore/canvas.py

@@ -85,10 +85,14 @@ class TTkCanvas:
         for i in range(0,h):
             self._data[i] = [' ']*w
             self._colors[i] = [TTkColor.RST]*w
-        self._width  = w
-        self._height = h
         if self._doubleBuffer:
-            self._bufferedData, self._bufferedColors = self.copy()
+            self._bufferedData = [[]]*h
+            self._bufferedColors = [[]]*h
+            for i in range(0,h):
+                self._bufferedData[i] = ['']*w
+                self._bufferedColors[i] = [TTkColor.RST]*w
+        self._height = h
+        self._width  = w
 
     def resize(self, w, h):
         self._newWidth = w
@@ -550,7 +554,7 @@ class TTkCanvas:
                 color = self._colors[y][x]
                 if empty:
                     ansi = color+lbt.Mv.t(y+1,x+1)
-                    lastcolor = color
+                    #lastcolor = color
                     empty = False
                 if color != lastcolor:
                     ansi += color-lastcolor

TermTk/TTkCore/filebuffer.py

@@ -69,7 +69,7 @@ class TTkFileBuffer():
         self._window = window
         self._numW = numWindows
         self._filename = filename
-        self._indexes = []
+        self._indexes = [0]
         self._indexesMutex = threading.Lock()
         self._width=0
         self._buffer = [None]*self._numW
@@ -132,27 +132,24 @@ class TTkFileBuffer():
         lines = 0
         offset = 0
         fileSize = os.stat(self._filename).st_size
-        linesToRead = 100000
+        chunkSize = 0x1000000 # ~16M
         with open(self._filename,'r') as infile:
-            for line in infile:
-                lines += 1
-                indexes.append(offset)
-                offset += len(line)
-                if len(indexes) == linesToRead:
-                    self._indexesMutex.acquire()
-                    self._indexes += indexes
-                    self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages)))
-                    self._indexesMutex.release()
-                    indexes = []
-                    # TTkLog.debug(f"{self._filename}: Indexed {int(100*offset/fileSize)}% {len(self._indexes)} ...")
-                    self.indexUpdated.emit(offset/fileSize)
-        self._indexesMutex.acquire()
-        self._indexes += indexes
-        self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages)))
-        self._indexesMutex.release()
-        # TTkLog.debug(f"{self._filename}: Indexed {len(self._indexes)} END")
+            while (chunk:=infile.read(chunkSize)):
+                start = 0
+                while (index:=chunk.find('\n',start))!=-1:
+                    indexes.append(index+offset+1)
+                    start = index+1
+                self._indexesMutex.acquire()
+                self._indexes += indexes
+                self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages)))
+                self._indexesMutex.release()
+                indexes = []
+                offset+=len(chunk)
+                self.indexUpdated.emit(offset/fileSize)
+                # TTkLog.debug(f"{self._filename} {offset/fileSize} ...")
         self.indexUpdated.emit(1.0)
         self.indexed.emit()
+        # TTkLog.debug(f"{self._filename} {offset/fileSize} END")
 
     def searchRe(self, regex):
         indexes = []