dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Remove threading for CSV to fix context injection

Browse source
This commit is contained in:
Dazed 2023-04-23 11:06:16 -05:00
parent 58e8299700
commit 1265cfc270
1 changed files with 12 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/csvtl.py
View file

@ -91,6 +91,7 @@ def getResultString(translatedData, translationTime, filename):
def parseCSV(readFile, writeFile, filename): def parseCSV(readFile, writeFile, filename):
totalTokens = 0 totalTokens = 0
totalLines = 0 totalLines = 0
textHistory = []
global LOCK global LOCK
# Get total for progress bar # Get total for progress bar
@ -103,20 +104,17 @@ def parseCSV(readFile, writeFile, filename):
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc=filename
pbar.total=totalLines pbar.total=totalLines
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(translateCSV, row, pbar, writer) for row in reader]
for future in as_completed(futures): for row in reader:
try: try:
totalTokens += future.result() totalTokens += translateCSV(row, pbar, writer, textHistory)
except Exception as e: except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno)
return [reader, totalTokens, e, tracebackLineNo] return [reader, totalTokens, e, tracebackLineNo]
return [reader, totalTokens, None] return [reader, totalTokens, None]
def translateCSV(row, pbar, writer): def translateCSV(row, pbar, writer, textHistory):
translatedText = '' translatedText = ''
textHistory = []
maxHistory = MAXHISTORY maxHistory = MAXHISTORY
tokens = 0 tokens = 0
global LOCK, ESTIMATE global LOCK, ESTIMATE
@ -154,13 +152,11 @@ def translateCSV(row, pbar, writer):
row[1] = translatedText row[1] = translatedText
# Keep textHistory list at length maxHistory # Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory:
textHistory.pop(0)
if not ESTIMATE:
writer.writerow(row)
with LOCK: with LOCK:
if len(textHistory) > maxHistory:
textHistory.pop(0)
if not ESTIMATE:
writer.writerow(row)
pbar.update(1) pbar.update(1)
except Exception as e: except Exception as e: