92 lines
No EOL
2.7 KiB
Python
92 lines
No EOL
2.7 KiB
Python
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
import textwrap
|
|
import threading
|
|
import time
|
|
import traceback
|
|
import tiktoken
|
|
|
|
from colorama import Fore
|
|
from dotenv import load_dotenv
|
|
import openai
|
|
from retry import retry
|
|
from tqdm import tqdm
|
|
|
|
#Globals
|
|
load_dotenv()
|
|
openai.organization = os.getenv('org')
|
|
openai.api_key = os.getenv('key')
|
|
|
|
APICOST = .002 # Depends on the model https://openai.com/pricing
|
|
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
|
|
THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread.
|
|
LOCK = threading.Lock()
|
|
WIDTH = 60
|
|
LISTWIDTH = 60
|
|
MAXHISTORY = 10
|
|
ESTIMATE = ''
|
|
TOTALCOST = 0
|
|
TOKENS = 0
|
|
TOTALTOKENS = 0
|
|
NAMESLIST = []
|
|
|
|
#tqdm Globals
|
|
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
|
|
POSITION=0
|
|
LEAVE=False
|
|
|
|
def handleAllText(filename, estimate):
|
|
global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
|
|
ESTIMATE = estimate
|
|
|
|
with open('translated/' + filename, 'w+t', newline='', encoding='utf-16-le') as writeFile:
|
|
start = time.time()
|
|
translatedData = openFiles(filename, writeFile)
|
|
|
|
if estimate:
|
|
# Print Result
|
|
end = time.time()
|
|
tqdm.write(getResultString(['', TOKENS, None], end - start, filename))
|
|
TOTALCOST += TOKENS * .001 * APICOST
|
|
TOTALTOKENS += TOKENS
|
|
TOKENS = 0
|
|
os.remove('translated/' + filename)
|
|
|
|
else:
|
|
# Print Result
|
|
end = time.time()
|
|
tqdm.write(getResultString(translatedData, end - start, filename))
|
|
TOTALCOST += translatedData[1] * .001 * APICOST
|
|
TOTALTOKENS += translatedData[1]
|
|
|
|
return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
|
|
|
|
def openFiles(filename, writeFile):
|
|
with open('files/' + filename, 'r', encoding='utf-16-le') as readFile, writeFile:
|
|
translatedData = parseText(readFile, writeFile, filename)
|
|
|
|
return translatedData
|
|
|
|
def getResultString(translatedData, translationTime, filename):
|
|
# File Print String
|
|
tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
|
|
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
|
|
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
|
|
|
|
if translatedData[2] == None:
|
|
# Success
|
|
return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
|
|
|
|
else:
|
|
# Fail
|
|
try:
|
|
raise translatedData[2]
|
|
except Exception as e:
|
|
errorString = str(e) + '|' + translatedData[3] + Fore.RED
|
|
return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
|
|
errorString + Fore.RESET
|
|
|