Module TermTk.TTkCore.filebuffer
Expand source code
#!/usr/bin/env python3
# MIT License
#
# Copyright (c) 2021 Eugenio Parodi <ceccopierangiolieugenio AT googlemail DOT com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import os
import re
import threading
from TermTk.TTkCore.log import TTkLog
from TermTk.TTkCore.signal import pyTTkSignal, pyTTkSlot
'''
w1 w3 w2 w5
Buffer |----|----|----|----| cache buffer
| \ / \
| x \
| / \ \
Pages | 0 | 2 | 1 |None| 3 |None| index to buffer
File |----|----|----|----|----|----| view as list of windows
w1 w2 w3 w4 w5 w6
'''
class TTkFileBuffer():
class _Page:
__slots__ = ('_page', '_size', '_buffer')
def __init__(self, page, size):
self._page = page
self._size = size
self._buffer = [""]*self._size
#TTkLog.debug(f"{self._buffer}")
@property
def buffer(self):
return self._buffer
@property
def page(self):
return self._page
__slots__ = (
'_indexes', '_indexesMutex',
'_filename', '_fd',
'_pages', '_buffer',
'_window', '_numW',
'_width',
#Signals
'indexUpdated', 'indexed')
def __init__(self, filename, window, numWindows):
# Signals
self.indexUpdated = pyTTkSignal(float)
self.indexed = pyTTkSignal()
self._window = window
self._numW = numWindows
self._filename = filename
self._indexes = [0]
self._indexesMutex = threading.Lock()
self._width=0
self._buffer = [None]*self._numW
self._pages = [None]
self._fd = open(self._filename,'r')
threading.Thread(target=self.createIndex).start()
def __del__(self):
self._fd.close()
def getLen(self):
return len(self._indexes)
def getWidth(self, indexes=None):
return self._width
def getLineDirect(self, line):
if line >= self.getLen():
return ""
self._indexesMutex.acquire()
self._fd.seek(self._indexes[line])
self._indexesMutex.release()
return self._fd.readline()
def getLine(self, line):
if line >= self.getLen():
return ""
page = line//self._window
offset = line%self._window
if self._pages[page] == None:
# Dispose of the pages to the bottom
dispose = self._buffer.pop(0)
if dispose is not None:
self._pages[dispose.page] = None
self._pages[page] = self._Page(page, self._window)
self._buffer.append(self._pages[page])
self._indexesMutex.acquire()
self._fd.seek(self._indexes[line-offset])
self._indexesMutex.release()
buffer = self._pages[page].buffer
for i in range(self._window):
buffer[i] = self._fd.readline()
#self._width = max(self._width,len(buffer[i]))
else:
# Push the page to the top of the buffer
i = self._buffer.index(self._pages[page])
p = self._buffer.pop(i)
self._buffer.append(p)
return self._pages[page].buffer[offset]
def getSlice(self, line, length):
ret = []
for i in range(line, line+length):
ret.append(self.getLine(i))
return ret
def createIndex(self):
# TTkLog.debug(f"Start Indexing {self._filename}")
indexes = []
lines = 0
offset = 0
width = 0
fileSize = os.stat(self._filename).st_size
chunkSize = 0x1000000 # ~16M
with open(self._filename,'r') as infile:
while (chunk:=infile.read(chunkSize)):
start = 0
while (index:=chunk.find('\n',start))!=-1:
indexes.append(index+offset+1)
start = index+1
self._indexesMutex.acquire()
self._indexes += indexes
self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages)))
self._indexesMutex.release()
indexes = []
offset+=len(chunk)
self.indexUpdated.emit(offset/fileSize)
# TTkLog.debug(f"{self._filename} {offset/fileSize} ...")
self._width = max([ (self._indexes[i+1]-self._indexes[i]) for i in range(len(self._indexes)-1) ])
self.indexUpdated.emit(1.0)
self.indexed.emit()
# TTkLog.debug(f"{self._filename} {offset/fileSize} END")
def searchRe(self, regex):
indexes = []
id = 0
rr = re.compile(regex)
with open(self._filename,'r') as infile:
for line in infile:
ma = rr.match(line)
if ma:
indexes.append(id)
id += 1
return indexes
def search(self, txt):
indexes = []
id = 0
with open(self._filename,'r') as infile:
for line in infile:
if txt in line:
indexes.append(id)
id += 1
return indexesClasses
class TTkFileBuffer (filename, window, numWindows)-
Expand source code
class TTkFileBuffer(): class _Page: __slots__ = ('_page', '_size', '_buffer') def __init__(self, page, size): self._page = page self._size = size self._buffer = [""]*self._size #TTkLog.debug(f"{self._buffer}") @property def buffer(self): return self._buffer @property def page(self): return self._page __slots__ = ( '_indexes', '_indexesMutex', '_filename', '_fd', '_pages', '_buffer', '_window', '_numW', '_width', #Signals 'indexUpdated', 'indexed') def __init__(self, filename, window, numWindows): # Signals self.indexUpdated = pyTTkSignal(float) self.indexed = pyTTkSignal() self._window = window self._numW = numWindows self._filename = filename self._indexes = [0] self._indexesMutex = threading.Lock() self._width=0 self._buffer = [None]*self._numW self._pages = [None] self._fd = open(self._filename,'r') threading.Thread(target=self.createIndex).start() def __del__(self): self._fd.close() def getLen(self): return len(self._indexes) def getWidth(self, indexes=None): return self._width def getLineDirect(self, line): if line >= self.getLen(): return "" self._indexesMutex.acquire() self._fd.seek(self._indexes[line]) self._indexesMutex.release() return self._fd.readline() def getLine(self, line): if line >= self.getLen(): return "" page = line//self._window offset = line%self._window if self._pages[page] == None: # Dispose of the pages to the bottom dispose = self._buffer.pop(0) if dispose is not None: self._pages[dispose.page] = None self._pages[page] = self._Page(page, self._window) self._buffer.append(self._pages[page]) self._indexesMutex.acquire() self._fd.seek(self._indexes[line-offset]) self._indexesMutex.release() buffer = self._pages[page].buffer for i in range(self._window): buffer[i] = self._fd.readline() #self._width = max(self._width,len(buffer[i])) else: # Push the page to the top of the buffer i = self._buffer.index(self._pages[page]) p = self._buffer.pop(i) self._buffer.append(p) return self._pages[page].buffer[offset] def getSlice(self, line, length): ret = [] for i in range(line, line+length): ret.append(self.getLine(i)) return ret def createIndex(self): # TTkLog.debug(f"Start Indexing {self._filename}") indexes = [] lines = 0 offset = 0 width = 0 fileSize = os.stat(self._filename).st_size chunkSize = 0x1000000 # ~16M with open(self._filename,'r') as infile: while (chunk:=infile.read(chunkSize)): start = 0 while (index:=chunk.find('\n',start))!=-1: indexes.append(index+offset+1) start = index+1 self._indexesMutex.acquire() self._indexes += indexes self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages))) self._indexesMutex.release() indexes = [] offset+=len(chunk) self.indexUpdated.emit(offset/fileSize) # TTkLog.debug(f"{self._filename} {offset/fileSize} ...") self._width = max([ (self._indexes[i+1]-self._indexes[i]) for i in range(len(self._indexes)-1) ]) self.indexUpdated.emit(1.0) self.indexed.emit() # TTkLog.debug(f"{self._filename} {offset/fileSize} END") def searchRe(self, regex): indexes = [] id = 0 rr = re.compile(regex) with open(self._filename,'r') as infile: for line in infile: ma = rr.match(line) if ma: indexes.append(id) id += 1 return indexes def search(self, txt): indexes = [] id = 0 with open(self._filename,'r') as infile: for line in infile: if txt in line: indexes.append(id) id += 1 return indexesInstance variables
var indexUpdated-
Return an attribute of instance, which is of type owner.
var indexed-
Return an attribute of instance, which is of type owner.
Methods
def createIndex(self)-
Expand source code
def createIndex(self): # TTkLog.debug(f"Start Indexing {self._filename}") indexes = [] lines = 0 offset = 0 width = 0 fileSize = os.stat(self._filename).st_size chunkSize = 0x1000000 # ~16M with open(self._filename,'r') as infile: while (chunk:=infile.read(chunkSize)): start = 0 while (index:=chunk.find('\n',start))!=-1: indexes.append(index+offset+1) start = index+1 self._indexesMutex.acquire() self._indexes += indexes self._pages += [None]*(1+(self.getLen()//self._window)-(len(self._pages))) self._indexesMutex.release() indexes = [] offset+=len(chunk) self.indexUpdated.emit(offset/fileSize) # TTkLog.debug(f"{self._filename} {offset/fileSize} ...") self._width = max([ (self._indexes[i+1]-self._indexes[i]) for i in range(len(self._indexes)-1) ]) self.indexUpdated.emit(1.0) self.indexed.emit() def getLen(self)-
Expand source code
def getLen(self): return len(self._indexes) def getLine(self, line)-
Expand source code
def getLine(self, line): if line >= self.getLen(): return "" page = line//self._window offset = line%self._window if self._pages[page] == None: # Dispose of the pages to the bottom dispose = self._buffer.pop(0) if dispose is not None: self._pages[dispose.page] = None self._pages[page] = self._Page(page, self._window) self._buffer.append(self._pages[page]) self._indexesMutex.acquire() self._fd.seek(self._indexes[line-offset]) self._indexesMutex.release() buffer = self._pages[page].buffer for i in range(self._window): buffer[i] = self._fd.readline() #self._width = max(self._width,len(buffer[i])) else: # Push the page to the top of the buffer i = self._buffer.index(self._pages[page]) p = self._buffer.pop(i) self._buffer.append(p) return self._pages[page].buffer[offset] def getLineDirect(self, line)-
Expand source code
def getLineDirect(self, line): if line >= self.getLen(): return "" self._indexesMutex.acquire() self._fd.seek(self._indexes[line]) self._indexesMutex.release() return self._fd.readline() def getSlice(self, line, length)-
Expand source code
def getSlice(self, line, length): ret = [] for i in range(line, line+length): ret.append(self.getLine(i)) return ret def getWidth(self, indexes=None)-
Expand source code
def getWidth(self, indexes=None): return self._width def search(self, txt)-
Expand source code
def search(self, txt): indexes = [] id = 0 with open(self._filename,'r') as infile: for line in infile: if txt in line: indexes.append(id) id += 1 return indexes def searchRe(self, regex)-
Expand source code
def searchRe(self, regex): indexes = [] id = 0 rr = re.compile(regex) with open(self._filename,'r') as infile: for line in infile: ma = rr.match(line) if ma: indexes.append(id) id += 1 return indexes