dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Create a cost estimation feature (WIP)

Browse source
This commit is contained in:
Dazed 2023-04-22 10:25:41 -05:00
parent 8142001b8a
commit b02c79dc16
3 changed files with 102 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
main.py
View file

@ -14,44 +14,49 @@ twice. You can simply copy the file generated in /translations back over to /fil
start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n')
def main():
estimate = ''
while estimate == '':
estimate = input('Select Translation or Cost Estimation:\n\n1. Translate\n2. Estimate\n')
match estimate:
case '1': estimate = False
case '2': estimate = True
case _: estimate = ''
version = input('Select the RPGMaker Version:\n\n1. MV/MZ\n2. ACE\n')
totalCost = 0
match version:
case '1':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for filename in os.listdir("files"):
if filename.endswith('json'):
future = executor.submit(handleMVMZ, filename)
future = executor.submit(handleMVMZ, filename, estimate)
try:
future.result()
totalCost = future.result()
except Exception as e:
print(Fore.RED + str(e))
# This is to encourage people to grab what's in /translated instead
deleteFolderFiles('files')
# Prevent immediately closing of CLI
input('Done! Press Enter to close.')
case '2':
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for filename in os.listdir("files"):
if filename.endswith('json'):
future = executor.submit(handleACE, filename)
future = executor.submit(handleACE, filename, estimate)
try:
future.result()
totalCost = future.result()
except Exception as e:
print(Fore.RED + str(e))
# This is to encourage people to grab what's in /translated instead
deleteFolderFiles('files')
if estimate == False:
# This is to encourage people to grab what's in /translated instead
deleteFolderFiles('files')
# Prevent immediately closing of CLI
input('Done! Press Enter to close.')
# Prevent immediately closing of CLI
print(totalCost)
input('Done! Press Enter to close.')
def deleteFolderFiles(folderPath):
for filename in os.listdir(folderPath):

54
rpgmakerace.py
View file

@ -20,12 +20,16 @@ load_dotenv()
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
COST = .002 # Depends on the model https://openai.com/pricing
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
LOCK = threading.Lock()
WIDTH = 60
MAXHISTORY = 10
ESTIMATE = ''
CHARACTERS = 0
TOTALCOST = 0
TOTALTOKENS = 0
#tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
@ -40,7 +44,11 @@ CODE101 = False
CODE355655 = False
CODE357 = False
def handleACE(filename):
def handleACE(filename, estimate):
global ESTIMATE
ESTIMATE = estimate
totalStart = time.time()
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
with open('files/' + filename, 'r', encoding='UTF-8') as f:
data = json.load(f)
@ -97,24 +105,45 @@ def handleACE(filename):
end = time.time()
json.dump(translatedData[0], outFile, ensure_ascii=False)
printString(translatedData, end - start, f)
def printString(translatedData, translationTime, f):
# Strings
# Print Result
if estimate:
global CHARACTERS
print('CHARACTERS Total: ' + str(CHARACTERS))
printString(['', round(CHARACTERS/.325, 1), None], end - start, f.name)
# Reset CHARACTERS*
CHARACTERS = 0
else:
printString(translatedData, end - start, f.name)
# Final Output
totalEnd = time.time()
printString(['', round(TOTALTOKENS, 1), None], totalEnd - totalStart, 'TOTAL')
def printString(translatedData, translationTime, filename):
global TOTALCOST, TOTALTOKENS
# Cost Estimation
cost = translatedData[1] * .001 * APICOST
TOTALCOST += cost
TOTALTOKENS += translatedData[1]
# File Print String
tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * COST) + ']'
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
if translatedData[2] == None:
# Success
tqdm.write(f.name + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET)
tqdm.write(filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET)
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
errorString = str(e) + Fore.RED
tqdm.write(f.name + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
tqdm.write(filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
errorString + Fore.RESET)
def parseMap(data, filename):
@ -641,6 +670,12 @@ def searchSystem(data, pbar):
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
global CHARACTERS
CHARACTERS += len(t) + len(history)
return (t, 0)
# If there isn't any Japanese in the text just skip
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+', t):
return(t, 0)
@ -657,5 +692,4 @@ def translateGPT(t, history):
request_timeout=30,
)
return [response.choices[0].message.content, response.usage.total_tokens]
return [response.choices[0].message.content, response.usage.total_tokens]

48
rpgmakermvmz.py
View file

@ -8,6 +8,7 @@ import textwrap
import threading
import time
import traceback
import tiktoken
from colorama import Fore
from dotenv import load_dotenv
@ -20,12 +21,16 @@ load_dotenv()
openai.organization = os.getenv('org')
openai.api_key = os.getenv('key')
COST = .002 # Depends on the model https://openai.com/pricing
APICOST = .002 # Depends on the model https://openai.com/pricing
PROMPT = Path('prompt.txt').read_text(encoding='utf-8')
THREADS = 20
LOCK = threading.Lock()
WIDTH = 60
MAXHISTORY = 10
ESTIMATE = ''
TOTALCOST = 0
TOKENS = 0
TOTALTOKENS = 0
#tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
@ -40,7 +45,10 @@ CODE101 = False
CODE355655 = False
CODE357 = False
def handleMVMZ(filename):
def handleMVMZ(filename, estimate):
global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
ESTIMATE = estimate
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
with open('files/' + filename, 'r', encoding='UTF-8') as f:
data = json.load(f)
@ -97,25 +105,40 @@ def handleMVMZ(filename):
end = time.time()
json.dump(translatedData[0], outFile, ensure_ascii=False)
printString(translatedData, end - start, f)
def printString(translatedData, translationTime, f):
# Strings
# Print Result
if estimate:
tqdm.write(getResultString(['', TOKENS, None], end - start, f.name))
TOTALCOST += TOKENS * .001 * APICOST
TOTALTOKENS += TOKENS
TOKENS = 0
else:
tqdm.write(getResultString(translatedData, end - start, f.name))
TOTALCOST += translatedData[1] * .001 * APICOST
TOTALTOKENS += translatedData[1]
return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename):
# File Print String
tokenString = Fore.YELLOW + '[' + str(translatedData[1]) + \
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * COST) + ']'
' Tokens/${:,.4f}'.format(translatedData[1] * .001 * APICOST) + ']'
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]'
if translatedData[2] == None:
# Success
tqdm.write(f.name + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET)
return filename + ': ' + tokenString + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET
else:
# Fail
try:
raise translatedData[2]
except Exception as e:
errorString = str(e) + Fore.RED
tqdm.write(f.name + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
errorString + Fore.RESET)
return filename + ': ' + tokenString + timeString + Fore.RED + u' \u2717 ' +\
errorString + Fore.RESET
def parseMap(data, filename):
totalTokens = 0
@ -637,6 +660,13 @@ def searchSystem(data, pbar):
@retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history):
# If ESTIMATE is True just count this as an execution and return.
if ESTIMATE:
global TOKENS
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
TOKENS += len(enc.encode(t) + enc.encode(history) + enc.encode(PROMPT))
return (t, 0)
# If there isn't any Japanese in the text just skip
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+', t):
return(t, 0)