mirror
/
pyTermTk
mirror of https://github.com/ceccopierangiolieugenio/pyTermTk.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

FileBuffer Async indexing and moved to TTkCore, added log generator

pull/5/head
Eugenio Parodi 5 years ago
parent
fcc26b20c3
commit
6693af9e5e
3 changed files with 239 additions and 113 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 169
      TermTk/TTkCore/filebuffer.py
  2. 131
      demo/tlogg.py
  3. 52
      tools/create_log.py

169
TermTk/TTkCore/filebuffer.py
Unescape View File

@ -0,0 +1,169 @@
#!/usr/bin/env python3
# MIT License
#
# Copyright (c) 2021 Eugenio Parodi <ceccopierangiolieugenio AT googlemail DOT com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import threading, os
from TermTk.TTkCore.log import TTkLog
from TermTk.TTkCore.signal import pyTTkSignal, pyTTkSlot
'''
w1 w3 w2 w5
Buffer |----|----|----|----|
| \ / \
| x \
| / \ \
File |----|----|----|----|----|----|
w1 w2 w3 w4 w5 w6
'''
class TTkFileBuffer():
class _Page:
__slots__ = ('_page', '_size', '_buffer')
def __init__(self, page, size):
self._page = page
self._size = size
self._buffer = [""]*self._size
#TTkLog.debug(f"{self._buffer}")
@property
def buffer(self):
return self._buffer
@property
def page(self):
return self._page
__slots__ = (
'_indexes', '_indexesMutex',
'_filename', '_fd',
'_pages', '_buffer',
'_window', '_numW',
'_width',
#Signals
'indexUpdated', 'indexed')
def __init__(self, filename, window, numWindows):
# Signals
self.indexUpdated = pyTTkSignal(float)
self.indexed = pyTTkSignal()
self._window = window
self._numW = numWindows
self._filename = filename
self._indexes = []
self._indexesMutex = threading.Lock()
self._width=0
self._buffer = [None]*self._numW
self._pages = [None]
self._fd = open(self._filename,'r')
threading.Thread(target=self.createIndex).start()
def __del__(self):
self._fd.close()
def getLen(self):
return len(self._indexes)
def getWidth(self, indexes=None):
return self._width
def getLineDirect(self, line):
if line >= self.getLen():
return ""
self._indexesMutex.acquire()
self._fd.seek(self._indexes[line])
self._indexesMutex.release()
return self._fd.readline()
def getLine(self, line):
if line >= self.getLen():
return ""
page = line//self._window
offset = line%self._window
if self._pages[page] == None:
# Dispose of the pages to the bottom
dispose = self._buffer.pop(0)
if dispose is not None:
self._pages[dispose.page] = None
self._pages[page] = self._Page(page, self._window)
self._buffer.append(self._pages[page])
self._indexesMutex.acquire()
self._fd.seek(self._indexes[line])
self._indexesMutex.release()
buffer = self._pages[page].buffer
for i in range(self._window):
buffer[i] = self._fd.readline()
self._width = max(self._width,len(buffer[i]))
else:
# Push the page to the top of the buffer
i = self._buffer.index(self._pages[page])
p = self._buffer.pop(i)
self._buffer.append(p)
return self._pages[page].buffer[offset]
def createIndex(self):
TTkLog.debug(f"Start Indexing {self._filename}")
indexes = []
lines = 0
offset = 0
fileSize = os.stat(self._filename).st_size
linesToRead = 100000
with open(self._filename,'r') as infile:
for line in infile:
lines += 1
indexes.append(offset)
offset += len(line)
if len(indexes) == linesToRead:
self._indexesMutex.acquire()
self._indexes += indexes
self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages)))
self._indexesMutex.release()
indexes = []
# TTkLog.debug(f"{self._filename}: Indexed {int(100*offset/fileSize)}% {len(self._indexes)} ...")
self.indexUpdated.emit(offset/fileSize)
self._indexesMutex.acquire()
self._indexes += indexes
self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages)))
self._indexesMutex.release()
# TTkLog.debug(f"{self._filename}: Indexed {len(self._indexes)} END")
self.indexUpdated.emit(1.0)
self.indexed.emit()
def searchRe(self, regex):
indexes = []
id = 0
rr = re.compile(regex)
with open(self._filename,'r') as infile:
for line in infile:
ma = rr.match(line)
if ma:
indexes.append(id)
id += 1
return indexes
def search(self, txt):
indexes = []
id = 0
with open(self._filename,'r') as infile:
for line in infile:
if txt in line:
indexes.append(id)
id += 1
return indexes

131
demo/tlogg.py
Unescape View File

@ -35,6 +35,7 @@ from TermTk.TTkCore.constant import TTkK
from TermTk.TTkCore.log import TTkLog
from TermTk.TTkCore.color import TTkColor
from TermTk.TTkCore.ttk import TTk
from TermTk.TTkCore.filebuffer import TTkFileBuffer
from TermTk.TTkWidgets.frame import TTkFrame
from TermTk.TTkTemplates.color import TColor
from TermTk.TTkTemplates.text import TText
@ -42,122 +43,13 @@ from TermTk.TTkCore.signal import pyTTkSlot, pyTTkSignal
from TermTk.TTkAbstract.abstractscrollarea import TTkAbstractScrollArea
from TermTk.TTkAbstract.abstractscrollview import TTkAbstractScrollView
'''
w1 w3 w2 w5
Buffer |----|----|----|----|
| \ / \
| x \
| / \ \
File |----|----|----|----|----|----|
w1 w2 w3 w4 w5 w6
'''
class _FileBuffer():
class _Page:
__slots__ = ('_page', '_size', '_buffer')
def __init__(self, page, size):
self._page = page
self._size = size
self._buffer = [""]*self._size
#TTkLog.debug(f"{self._buffer}")
@property
def buffer(self):
return self._buffer
@property
def page(self):
return self._page
__slots__ = ('_filename', '_indexes', '_fd', '_lastline', '_pages', '_buffer', '_window', '_numW', '_width')
def __init__(self, filename, window, numWindows):
self._window = window
self._numW = numWindows
self._filename = filename
self._indexes = []
self._width=0
self.createIndex()
self._buffer = [None]*self._numW
self._pages = [None]*(1+self.getLen()//window)
self._fd = open(self._filename,'r')
def __del__(self):
self._fd.close()
def getLen(self):
return len(self._indexes)
def getWidth(self, indexes=None):
return self._width
def getLineDirect(self, line):
if line >= self.getLen():
return ""
self._fd.seek(self._indexes[line])
return self._fd.readline()
def getLine(self, line):
if line >= self.getLen():
return ""
page = line//self._window
offset = line%self._window
if self._pages[page] == None:
# Dispose of the pages to the bottom
dispose = self._buffer.pop(0)
if dispose is not None:
self._pages[dispose.page] = None
self._pages[page] = self._Page(page, self._window)
self._buffer.append(self._pages[page])
self._fd.seek(self._indexes[line])
buffer = self._pages[page].buffer
for i in range(self._window):
buffer[i] = self._fd.readline()
self._width = max(self._width,len(buffer[i]))
else:
# Push the page to the top of the buffer
i = self._buffer.index(self._pages[page])
p = self._buffer.pop(i)
self._buffer.append(p)
return self._pages[page].buffer[offset]
def createIndex(self):
self._indexes = []
lines = 0
offset = 0
with open(self._filename,'r') as infile:
for line in infile:
lines += 1
self._indexes.append(offset)
offset += len(line)
def searchRe(self, regex):
indexes = []
id = 0
rr = re.compile(regex)
with open(self._filename,'r') as infile:
for line in infile:
ma = rr.match(line)
if ma:
indexes.append(id)
id += 1
return indexes
def search(self, txt):
indexes = []
id = 0
with open(self._filename,'r') as infile:
for line in infile:
if txt in line:
indexes.append(id)
id += 1
return indexes
class _FileViewer(TTkAbstractScrollView):
__slots__ = ('_fileBuffer', '_indexes', '_indexesMark', '_indexexSearched', '_selected')
__slots__ = ('_fileBuffer', '_indexes', '_indexesMark', '_indexexSearched', '_selected', '_indexing')
def __init__(self, *args, **kwargs):
self._indexes = None
self._indexesMark = []
self._indexesSearched = []
self._indexing = None
self._selected = None
TTkAbstractScrollView.__init__(self, *args, **kwargs)
self._name = kwargs.get('name' , '_FileViewer' )
@ -168,6 +60,16 @@ class _FileViewer(TTkAbstractScrollView):
def _viewChangedHandler(self):
self.update()
@pyTTkSlot(float)
def fileIndexing(self, percentage):
self._indexing = percentage
self.viewChanged.emit()
@pyTTkSlot()
def fileIndexed(self):
self._indexing = None
self.viewChanged.emit()
def showIndexes(self, indexes):
self._indexes = indexes
self.viewChanged.emit()
@ -217,7 +119,8 @@ class _FileViewer(TTkAbstractScrollView):
self.getCanvas().drawText(
pos=(2,i),
text=self._fileBuffer.getLineDirect(self._indexes[i+oy]).replace('\t',' ').replace('\n','')[ox:] )
if self._indexing is not None:
self.getCanvas().drawText(pos=(0,0), text=f" [ Indexed: {int(100*self._indexing)}% ] ")
class FileViewer(TTkAbstractScrollArea):
@ -269,8 +172,10 @@ def main():
bottomFrame.layout().addItem(bottomLayoutSearch)
# Define the main file Viewer
fileBuffer = _FileBuffer(file, 0x1000, 0x10)
fileBuffer = TTkFileBuffer(file, 0x1000, 0x10)
topViewer = FileViewer(parent=topFrame, filebuffer=fileBuffer)
fileBuffer.indexUpdated.connect(topViewer.viewport().fileIndexing)
fileBuffer.indexed.connect(topViewer.viewport().fileIndexed)
# Define the Search Viewer
bottomViewer = FileViewer(parent=bottomFrame, filebuffer=fileBuffer)
bottomViewport = bottomViewer.viewport()

52
tools/create_log.py
Unescape View File

@ -0,0 +1,52 @@
#!/usr/bin/env python3
# MIT License
#
# Copyright (c) 2021 Eugenio Parodi <ceccopierangiolieugenio AT googlemail DOT com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import sys
import random
words = ["Lorem", "ipsum", "dolor", "sit", "amet,", "consectetur", "adipiscing", "elit,", "sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore", "magna", "aliqua.", "Ut", "enim", "ad", "minim", "veniam,", "quis", "nostrud", "exercitation", "ullamco", "laboris", "nisi", "ut", "aliquip", "ex", "ea", "commodo", "consequat.", "Duis", "aute", "irure", "dolor", "in", "reprehenderit", "in", "voluptate", "velit", "esse", "cillum", "dolore", "eu", "fugiat", "nulla", "pariatur.", "Excepteur", "sint", "occaecat", "cupidatat", "non", "proident,", "sunt", "in", "culpa", "qui", "officia", "deserunt", "mollit", "anim", "id", "est", "laborum."]
def getWord():
return random.choice(words)
def getSentence(a,b):
return " ".join([getWord() for i in range(0,random.randint(a,b))])
if len(sys.argv) != 3 :
print ("Missing filename")
print ("use %s <FILENAME> <LINES>" % sys.argv[0])
exit(1)
filename = sys.argv[1]
lines = int(sys.argv[2])
print ("Lines=%d" % lines)
with open(filename, 'a') as out:
for i in range(0,lines):
seconds = 1000 + i
m, s = divmod(seconds, 60)
h, m = divmod(m, 60)
if (random.random() < 0.9):
out.write( "TEST;%d:%02d:%02d;COL1\tCOL2 COL3 c:COL4;LIN=%05X\tRND=%f %s %s\n" % (h, m, s, i, random.random(), getSentence(3,20), " Fill" * random.randint(1,5)) )
else:
out.write( "TEST;%d:%02d:%02d;COL1 --- (BROKEN LINE) --- LIN=%05X\tRND=%f %s %s\n" % (h, m, s, i, random.random(), getSentence(3,20), " Fill" * random.randint(1,5)) )
Write Preview
Loading…
Cancel
Save