diff --git a/TermTk/TTkCore/filebuffer.py b/TermTk/TTkCore/filebuffer.py new file mode 100644 index 00000000..e9b864e3 --- /dev/null +++ b/TermTk/TTkCore/filebuffer.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +# MIT License +# +# Copyright (c) 2021 Eugenio Parodi +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import threading, os +from TermTk.TTkCore.log import TTkLog +from TermTk.TTkCore.signal import pyTTkSignal, pyTTkSlot + +''' + w1 w3 w2 w5 + Buffer |----|----|----|----| + | \ / \ + | x \ + | / \ \ + File |----|----|----|----|----|----| + w1 w2 w3 w4 w5 w6 +''' +class TTkFileBuffer(): + class _Page: + __slots__ = ('_page', '_size', '_buffer') + def __init__(self, page, size): + self._page = page + self._size = size + self._buffer = [""]*self._size + #TTkLog.debug(f"{self._buffer}") + @property + def buffer(self): + return self._buffer + @property + def page(self): + return self._page + + __slots__ = ( + '_indexes', '_indexesMutex', + '_filename', '_fd', + '_pages', '_buffer', + '_window', '_numW', + '_width', + #Signals + 'indexUpdated', 'indexed') + def __init__(self, filename, window, numWindows): + # Signals + self.indexUpdated = pyTTkSignal(float) + self.indexed = pyTTkSignal() + + self._window = window + self._numW = numWindows + self._filename = filename + self._indexes = [] + self._indexesMutex = threading.Lock() + self._width=0 + self._buffer = [None]*self._numW + self._pages = [None] + self._fd = open(self._filename,'r') + threading.Thread(target=self.createIndex).start() + + def __del__(self): + self._fd.close() + + def getLen(self): + return len(self._indexes) + + def getWidth(self, indexes=None): + return self._width + + def getLineDirect(self, line): + if line >= self.getLen(): + return "" + self._indexesMutex.acquire() + self._fd.seek(self._indexes[line]) + self._indexesMutex.release() + return self._fd.readline() + + def getLine(self, line): + if line >= self.getLen(): + return "" + page = line//self._window + offset = line%self._window + if self._pages[page] == None: + # Dispose of the pages to the bottom + dispose = self._buffer.pop(0) + if dispose is not None: + self._pages[dispose.page] = None + self._pages[page] = self._Page(page, self._window) + self._buffer.append(self._pages[page]) + self._indexesMutex.acquire() + self._fd.seek(self._indexes[line]) + self._indexesMutex.release() + buffer = self._pages[page].buffer + for i in range(self._window): + buffer[i] = self._fd.readline() + self._width = max(self._width,len(buffer[i])) + else: + # Push the page to the top of the buffer + i = self._buffer.index(self._pages[page]) + p = self._buffer.pop(i) + self._buffer.append(p) + return self._pages[page].buffer[offset] + + + def createIndex(self): + TTkLog.debug(f"Start Indexing {self._filename}") + indexes = [] + lines = 0 + offset = 0 + fileSize = os.stat(self._filename).st_size + linesToRead = 100000 + with open(self._filename,'r') as infile: + for line in infile: + lines += 1 + indexes.append(offset) + offset += len(line) + if len(indexes) == linesToRead: + self._indexesMutex.acquire() + self._indexes += indexes + self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages))) + self._indexesMutex.release() + indexes = [] + # TTkLog.debug(f"{self._filename}: Indexed {int(100*offset/fileSize)}% {len(self._indexes)} ...") + self.indexUpdated.emit(offset/fileSize) + self._indexesMutex.acquire() + self._indexes += indexes + self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages))) + self._indexesMutex.release() + # TTkLog.debug(f"{self._filename}: Indexed {len(self._indexes)} END") + self.indexUpdated.emit(1.0) + self.indexed.emit() + + def searchRe(self, regex): + indexes = [] + id = 0 + rr = re.compile(regex) + with open(self._filename,'r') as infile: + for line in infile: + ma = rr.match(line) + if ma: + indexes.append(id) + id += 1 + return indexes + + def search(self, txt): + indexes = [] + id = 0 + with open(self._filename,'r') as infile: + for line in infile: + if txt in line: + indexes.append(id) + id += 1 + return indexes diff --git a/demo/tlogg.py b/demo/tlogg.py index 220f0851..bb25f2a3 100755 --- a/demo/tlogg.py +++ b/demo/tlogg.py @@ -35,6 +35,7 @@ from TermTk.TTkCore.constant import TTkK from TermTk.TTkCore.log import TTkLog from TermTk.TTkCore.color import TTkColor from TermTk.TTkCore.ttk import TTk +from TermTk.TTkCore.filebuffer import TTkFileBuffer from TermTk.TTkWidgets.frame import TTkFrame from TermTk.TTkTemplates.color import TColor from TermTk.TTkTemplates.text import TText @@ -42,122 +43,13 @@ from TermTk.TTkCore.signal import pyTTkSlot, pyTTkSignal from TermTk.TTkAbstract.abstractscrollarea import TTkAbstractScrollArea from TermTk.TTkAbstract.abstractscrollview import TTkAbstractScrollView - - -''' - w1 w3 w2 w5 - Buffer |----|----|----|----| - | \ / \ - | x \ - | / \ \ - File |----|----|----|----|----|----| - w1 w2 w3 w4 w5 w6 -''' -class _FileBuffer(): - class _Page: - __slots__ = ('_page', '_size', '_buffer') - def __init__(self, page, size): - self._page = page - self._size = size - self._buffer = [""]*self._size - #TTkLog.debug(f"{self._buffer}") - @property - def buffer(self): - return self._buffer - @property - def page(self): - return self._page - - __slots__ = ('_filename', '_indexes', '_fd', '_lastline', '_pages', '_buffer', '_window', '_numW', '_width') - def __init__(self, filename, window, numWindows): - self._window = window - self._numW = numWindows - self._filename = filename - self._indexes = [] - self._width=0 - self.createIndex() - self._buffer = [None]*self._numW - self._pages = [None]*(1+self.getLen()//window) - self._fd = open(self._filename,'r') - - def __del__(self): - self._fd.close() - - def getLen(self): - return len(self._indexes) - - def getWidth(self, indexes=None): - return self._width - - def getLineDirect(self, line): - if line >= self.getLen(): - return "" - self._fd.seek(self._indexes[line]) - return self._fd.readline() - - def getLine(self, line): - if line >= self.getLen(): - return "" - page = line//self._window - offset = line%self._window - if self._pages[page] == None: - # Dispose of the pages to the bottom - dispose = self._buffer.pop(0) - if dispose is not None: - self._pages[dispose.page] = None - self._pages[page] = self._Page(page, self._window) - self._buffer.append(self._pages[page]) - self._fd.seek(self._indexes[line]) - buffer = self._pages[page].buffer - for i in range(self._window): - buffer[i] = self._fd.readline() - self._width = max(self._width,len(buffer[i])) - else: - # Push the page to the top of the buffer - i = self._buffer.index(self._pages[page]) - p = self._buffer.pop(i) - self._buffer.append(p) - return self._pages[page].buffer[offset] - - - def createIndex(self): - self._indexes = [] - lines = 0 - offset = 0 - with open(self._filename,'r') as infile: - for line in infile: - lines += 1 - self._indexes.append(offset) - offset += len(line) - - def searchRe(self, regex): - indexes = [] - id = 0 - rr = re.compile(regex) - with open(self._filename,'r') as infile: - for line in infile: - ma = rr.match(line) - if ma: - indexes.append(id) - id += 1 - return indexes - - def search(self, txt): - indexes = [] - id = 0 - with open(self._filename,'r') as infile: - for line in infile: - if txt in line: - indexes.append(id) - id += 1 - return indexes - class _FileViewer(TTkAbstractScrollView): - __slots__ = ('_fileBuffer', '_indexes', '_indexesMark', '_indexexSearched', '_selected') + __slots__ = ('_fileBuffer', '_indexes', '_indexesMark', '_indexexSearched', '_selected', '_indexing') def __init__(self, *args, **kwargs): self._indexes = None self._indexesMark = [] self._indexesSearched = [] + self._indexing = None self._selected = None TTkAbstractScrollView.__init__(self, *args, **kwargs) self._name = kwargs.get('name' , '_FileViewer' ) @@ -168,6 +60,16 @@ class _FileViewer(TTkAbstractScrollView): def _viewChangedHandler(self): self.update() + @pyTTkSlot(float) + def fileIndexing(self, percentage): + self._indexing = percentage + self.viewChanged.emit() + + @pyTTkSlot() + def fileIndexed(self): + self._indexing = None + self.viewChanged.emit() + def showIndexes(self, indexes): self._indexes = indexes self.viewChanged.emit() @@ -217,7 +119,8 @@ class _FileViewer(TTkAbstractScrollView): self.getCanvas().drawText( pos=(2,i), text=self._fileBuffer.getLineDirect(self._indexes[i+oy]).replace('\t',' ').replace('\n','')[ox:] ) - + if self._indexing is not None: + self.getCanvas().drawText(pos=(0,0), text=f" [ Indexed: {int(100*self._indexing)}% ] ") class FileViewer(TTkAbstractScrollArea): @@ -269,8 +172,10 @@ def main(): bottomFrame.layout().addItem(bottomLayoutSearch) # Define the main file Viewer - fileBuffer = _FileBuffer(file, 0x1000, 0x10) + fileBuffer = TTkFileBuffer(file, 0x1000, 0x10) topViewer = FileViewer(parent=topFrame, filebuffer=fileBuffer) + fileBuffer.indexUpdated.connect(topViewer.viewport().fileIndexing) + fileBuffer.indexed.connect(topViewer.viewport().fileIndexed) # Define the Search Viewer bottomViewer = FileViewer(parent=bottomFrame, filebuffer=fileBuffer) bottomViewport = bottomViewer.viewport() diff --git a/tools/create_log.py b/tools/create_log.py new file mode 100755 index 00000000..358b2ff8 --- /dev/null +++ b/tools/create_log.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +# MIT License +# +# Copyright (c) 2021 Eugenio Parodi +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sys +import random + +words = ["Lorem", "ipsum", "dolor", "sit", "amet,", "consectetur", "adipiscing", "elit,", "sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore", "magna", "aliqua.", "Ut", "enim", "ad", "minim", "veniam,", "quis", "nostrud", "exercitation", "ullamco", "laboris", "nisi", "ut", "aliquip", "ex", "ea", "commodo", "consequat.", "Duis", "aute", "irure", "dolor", "in", "reprehenderit", "in", "voluptate", "velit", "esse", "cillum", "dolore", "eu", "fugiat", "nulla", "pariatur.", "Excepteur", "sint", "occaecat", "cupidatat", "non", "proident,", "sunt", "in", "culpa", "qui", "officia", "deserunt", "mollit", "anim", "id", "est", "laborum."] +def getWord(): + return random.choice(words) +def getSentence(a,b): + return " ".join([getWord() for i in range(0,random.randint(a,b))]) + + +if len(sys.argv) != 3 : + print ("Missing filename") + print ("use %s " % sys.argv[0]) + exit(1) + +filename = sys.argv[1] +lines = int(sys.argv[2]) +print ("Lines=%d" % lines) + +with open(filename, 'a') as out: + for i in range(0,lines): + seconds = 1000 + i + m, s = divmod(seconds, 60) + h, m = divmod(m, 60) + if (random.random() < 0.9): + out.write( "TEST;%d:%02d:%02d;COL1\tCOL2 COL3 c:COL4;LIN=%05X\tRND=%f %s %s\n" % (h, m, s, i, random.random(), getSentence(3,20), " Fill" * random.randint(1,5)) ) + else: + out.write( "TEST;%d:%02d:%02d;COL1 --- (BROKEN LINE) --- LIN=%05X\tRND=%f %s %s\n" % (h, m, s, i, random.random(), getSentence(3,20), " Fill" * random.randint(1,5)) )