dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adjust csvtl.py for game

Browse source
This commit is contained in:
Dazed 2023-08-16 17:30:45 -05:00
parent 858f8373e3
commit ee1b180434
2 changed files with 90 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
modules/csvtl.py
View file

@ -64,7 +64,7 @@ def handleCSV(filename, estimate):
return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL')
def openFiles(filename, writeFile): def openFiles(filename, writeFile):
with open('files/' + filename, 'r', encoding='UTF-8') as readFile, writeFile: with open('files/' + filename, 'r', encoding='shift-jis') as readFile, writeFile:
translatedData = parseCSV(readFile, writeFile, filename) translatedData = parseCSV(readFile, writeFile, filename)
return translatedData return translatedData
@ -108,7 +108,7 @@ def parseCSV(readFile, writeFile, filename):
readFile.seek(0) readFile.seek(0)
reader = csv.reader(readFile, delimiter=',',) reader = csv.reader(readFile, delimiter=',',)
writer = csv.writer(writeFile, delimiter=',', doublequote=None) writer = csv.writer(writeFile, delimiter=',', quotechar='\"')
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc=filename
@ -162,31 +162,56 @@ def translateCSV(row, pbar, writer, textHistory, format):
pbar.update(1) pbar.update(1)
# TextHistory is what we use to give GPT Context, so thats appended here. # TextHistory is what we use to give GPT Context, so thats appended here.
textHistory.append(': ' + translatedText) textHistory.append('\"' + translatedText + '\"')
# Translate Everything # Translate Everything
case '2': case '2':
for i in range(len(row)): for i in range(len(row)):
if i not in [2,3,4,5,6,7,8,9,10]: if i not in [1]:
continue continue
jaString = row[i] jaString = row[i]
if '_' in jaString:
continue
#Translate matchList = re.findall(r':name\[(.+?),.+?\](.+?[」）]+?)', jaString)
response = translateGPT(jaString, '', True)
translatedText = response[0]
tokens += response[1]
# Textwrap for match in matchList:
translatedText = textwrap.fill(translatedText, width=WIDTH) speaker = match[0]
text = match[1]
# Set Data # Translate Speaker
translatedText = translatedText.replace('\"', '') response = translateGPT (speaker, 'Reply with the English translation of the NPC name.', True)
translatedText = translatedText.replace('\'', '') translatedSpeaker = response[0]
translatedText = translatedText.replace(',', '') tokens += response[1]
row[i] = translatedText
writer.writerow(row) # Translate Line
response = translateGPT(translatedSpeaker + ': ' + text, 'Previous Translated Text: ' + '|'.join(textHistory), True)
translatedText = response[0]
tokens += response[1]
# TextHistory is what we use to give GPT Context, so thats appended here.
textHistory.append(translatedText)
# Remove Speaker from translated text
translatedText = re.sub(r'.+?: ', '', translatedText)
# Set Data
translatedText = translatedText.replace('\"', '')
translatedText = translatedText.replace('「', '')
translatedText = translatedText.replace('」', '')
translatedText = translatedText.replace('\n', '')
# Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = '「' + translatedText + '」\n'
row[i] = re.sub(rf'\b{re.escape(speaker)}\b', translatedSpeaker, row[i])
row[i] = row[i].replace(text, translatedText)
# Keep textHistory list at length maxHistory
with LOCK:
if len(textHistory) > maxHistory:
textHistory.pop(0)
if not ESTIMATE:
writer.writerow(row)
pbar.update(1) pbar.update(1)
except Exception as e: except Exception as e:
@ -197,13 +222,13 @@ def translateCSV(row, pbar, writer, textHistory, format):
def subVars(jaString): def subVars(jaString):
varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\.<>a-zA-Z]+' varRegex = r'\\+[a-zA-Z]+\[.+?\]|[\\]+[#|]+|\\+[\\\[\]\.<>a-zA-Z0-9]+'
count = 0 count = 0
varList = re.findall(varRegex, jaString) varList = re.findall(varRegex, jaString)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[v' + str(count) + ']') jaString = jaString.replace(var, '<x' + str(count) + '>')
count += 1 count += 1
return [jaString, varList] return [jaString, varList]
@ -213,7 +238,7 @@ def resubVars(translatedText, varList):
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
translatedText = translatedText.replace('[v' + str(count) + ']', var) translatedText = translatedText.replace('<x' + str(count) + '>', var)
count += 1 count += 1
# Remove Color Variables Spaces # Remove Color Variables Spaces
@ -241,13 +266,13 @@ def translateGPT(t, history, fullPromptFlag):
return(t, 0) return(t, 0)
"""Translate text using GPT""" """Translate text using GPT"""
context = 'Character Context: クレア == Clea | Female, ノラ == Nora | Female, リルム == Relm | Female, ソフィア == Sophia | Female, セリス == Celis | Female, ピステ == Piste | Female, イクト == Ect | Female, カロン == Caron | Female, エモニ == Emoni | Female, アミナ == Amina | Female, アルマダ == Armada | Female, フォニ == Phoni | Female, エレオ == Eleo | Female, ペルノ == Perno | Female' context = 'Eroge Names Context: ミカエル == Mikael | Female, ミカ == Mika | Female, ベルゼビュート == Beelzebuth | Female, ベル == Bel | Female, アズラエル == Azriel | Female, アズ == Az | Female, フレイア == Freya | Female'
if fullPromptFlag: if fullPromptFlag:
system = PROMPT system = PROMPT
user = 'Text to Translate: ' + subbedT user = 'Current Text to Translate: ' + subbedT
else: else:
system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.'
user = 'Text to Translate: ' + subbedT user = 'Current Text to Translate: ' + subbedT
response = openai.ChatCompletion.create( response = openai.ChatCompletion.create(
temperature=0, temperature=0,
model="gpt-3.5-turbo", model="gpt-3.5-turbo",
@ -270,10 +295,10 @@ def translateGPT(t, history, fullPromptFlag):
# Remove Placeholder Text # Remove Placeholder Text
translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('English Translation: ', '')
translatedText = translatedText.replace('Translation: ', '') translatedText = translatedText.replace('Translation: ', '')
translatedText = translatedText.replace('Text to Translate: ', '') translatedText = translatedText.replace('Current Text to Translate: ', '')
translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('English Translation:', '')
translatedText = translatedText.replace('Translation:', '') translatedText = translatedText.replace('Translation:', '')
translatedText = translatedText.replace('Text to Translate:', '') translatedText = translatedText.replace('Current Text to Translate:', '')
# Return Translation # Return Translation
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText:

70
modules/rpgmakermvmz.py
View file

@ -32,6 +32,7 @@ ESTIMATE = ''
TOTALCOST = 0 TOTALCOST = 0
TOKENS = 0 TOKENS = 0
TOTALTOKENS = 0 TOTALTOKENS = 0
NAMESLIST = []
#tqdm Globals #tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}'
@ -48,11 +49,12 @@ CODE355655 = False
CODE357 = False CODE357 = False
CODE657 = False CODE657 = False
CODE356 = False CODE356 = False
CODE320 = False CODE320 = True
CODE324 = False CODE324 = False
CODE111 = False CODE111 = False
CODE408 = False CODE408 = False
CODE108 = False CODE108 = False
NAMES = True # Output a list of all the character names found
def handleMVMZ(filename, estimate): def handleMVMZ(filename, estimate):
global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST
@ -65,6 +67,7 @@ def handleMVMZ(filename, estimate):
# Print Result # Print Result
end = time.time() end = time.time()
tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) tqdm.write(getResultString(['', TOKENS, None], end - start, filename))
tqdm.write(str(NAMESLIST))
with LOCK: with LOCK:
TOTALCOST += TOKENS * .001 * APICOST TOTALCOST += TOKENS * .001 * APICOST
TOTALTOKENS += TOKENS TOTALTOKENS += TOKENS
@ -452,6 +455,7 @@ def searchCodes(page, pbar):
speakerVar = '' speakerVar = ''
match = [] match = []
global LOCK global LOCK
global NAMESLIST
try: try:
for i in range(len(page['list'])): for i in range(len(page['list'])):
@ -494,14 +498,20 @@ def searchCodes(page, pbar):
# Check for speaker # Check for speaker
if '\\N' in finalJAString or '\\n' in finalJAString: if '\\N' in finalJAString or '\\n' in finalJAString:
match = re.findall(r'([\\]+[Nn]<([一-龠ぁ-ゔァ-ヴー]+)>)', finalJAString) match = re.findall(r'([\\]+[Nn]<(.+)>)', finalJAString)
if len(match) != 0: if len(match) != 0:
response = translateGPT(match[0][1], 'Reply with only the english translation of the NPC name', True) response = translateGPT(match[0][1], 'Reply with only the english translation of the NPC name', True)
tokens += response[1] tokens += response[1]
speaker = response[0].strip('.') speaker = response[0].strip('.')
speakerVar = match[0][0].replace(match[0][1], response[0]) speakerVar = match[0][0].replace(match[0][1], response[0])
finalJAString = finalJAString.replace(match[0][0], '') finalJAString = finalJAString.replace(match[0][0], '')
# Put names in list
if NAMES == True and speaker not in NAMESLIST:
with LOCK:
NAMESLIST.append(speaker)
# Need to remove outside code and put it back later # Need to remove outside code and put it back later
startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', finalJAString) startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', finalJAString)
finalJAString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', '', finalJAString) finalJAString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', '', finalJAString)
@ -521,12 +531,12 @@ def searchCodes(page, pbar):
# Translate # Translate
if speaker == '' and finalJAString != '': if speaker == '' and finalJAString != '':
response = translateGPT(finalJAString, 'Previous Dialogue: ' + '|'.join(textHistory), True) response = translateGPT(finalJAString, 'Previous Translated Text: ' + '|'.join(textHistory), True)
tokens += response[1] tokens += response[1]
translatedText = response[0] translatedText = response[0]
textHistory.append('\"' + translatedText + '\"') textHistory.append('\"' + translatedText + '\"')
elif finalJAString != '': elif finalJAString != '':
response = translateGPT(speaker + ': ' + finalJAString, 'Previous Dialogue: ' + '|'.join(textHistory), True) response = translateGPT(speaker + ': ' + finalJAString, 'Previous Translated Text: ' + '|'.join(textHistory), True)
tokens += response[1] tokens += response[1]
translatedText = response[0] translatedText = response[0]
textHistory.append('\"' + translatedText + '\"') textHistory.append('\"' + translatedText + '\"')
@ -774,8 +784,8 @@ def searchCodes(page, pbar):
continue continue
# Want to translate this script # Want to translate this script
if 'ans:' not in jaString: # if 'ans:' not in jaString:
continue # continue
# Need to remove outside code and put it back later # Need to remove outside code and put it back later
startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】]+', jaString) startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】]+', jaString)
@ -981,32 +991,30 @@ def searchCodes(page, pbar):
page['list'][i]['parameters'][j] = translatedText page['list'][i]['parameters'][j] = translatedText
### Event Code: 320 Set Variable ### Event Code: 320 Set Variable
if page['list'][i]['code'] == 320 and CODE320 == True or page['list'][i]['code'] == 324 and CODE324 == True: if page['list'][i]['code'] == 320 and CODE320 == True:
jaString = page['list'][i]['parameters'][1] jaString = page['list'][i]['parameters'][1]
if type(jaString) != str:
continue
# Definitely don't want to mess with files
if '■' in jaString or '_' in jaString:
continue
# Need to remove outside code and put it back later # If there isn't any Japanese in the text just skip
startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】a-zA-Z\\]+', jaString) if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString):
jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】a-zA-Z\\]+', '', jaString) continue
endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー【】。、…！？]+$', jaString)
jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー【】。、…！？]+$', '', jaString) response = translateGPT(jaString, 'Reply with the English translation of the NPC name.', True)
if startString is None: startString = ''
else: startString = startString.group()
if endString is None: endString = ''
else: endString = endString.group()
response = translateGPT(jaString, 'Reply with only the english translation of the npc nickname.', False)
translatedText = response[0] translatedText = response[0]
tokens += response[1]
# Remove characters that may break scripts # Remove characters that may break scripts
charList = ['\"'] charList = ['.', '\"', '\'', '\\n']
for char in charList: for char in charList:
translatedText = translatedText.replace(char, '') translatedText = translatedText.replace(char, '')
translatedText = translatedText.strip('.')
# Set Data # Set Data
tokens += response[1] page['list'][i]['parameters'][1] = translatedText
page['list'][i]['parameters'][1] = startString + translatedText + endString
except IndexError as e: except IndexError as e:
# This is part of the logic so we just pass it # This is part of the logic so we just pass it
@ -1162,13 +1170,13 @@ def searchSystem(data, pbar):
return tokens return tokens
def subVars(jaString): def subVars(jaString):
varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\.<>a-zA-Z]+' varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\\\[\]\.<>a-zA-Z0-9]+'
count = 0 count = 0
varList = re.findall(varRegex, jaString) varList = re.findall(varRegex, jaString)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[v' + str(count) + ']') jaString = jaString.replace(var, '<x' + str(count) + '>')
count += 1 count += 1
return [jaString, varList] return [jaString, varList]
@ -1178,7 +1186,7 @@ def resubVars(translatedText, varList):
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
translatedText = translatedText.replace('[v' + str(count) + ']', var) translatedText = translatedText.replace('<x' + str(count) + '>', var)
count += 1 count += 1
# Remove Color Variables Spaces # Remove Color Variables Spaces
@ -1206,13 +1214,13 @@ def translateGPT(t, history, fullPromptFlag):
return(t, 0) return(t, 0)
"""Translate text using GPT""" """Translate text using GPT"""
context = 'Eroge Characters Context: クレア == Clea | Female, ノラ == Nora | Female, リルム == Relm | Female, ソフィア == Sophia | Female, セリス == Celis | Female, ピステ == Piste | Female, イクト == Ect | Female, カロン == Caron | Female, エモニ == Emoni | Female, アミナ == Amina | Female, アルマダ == Armada | Female, フォニ == Phoni | Female, エレオ == Eleo | Female, ペルノ == Perno | Female' context = 'Eroge Names Context: ミカエル == Mikael | Female, ミカ == Mika | Female, ベルゼビュート == Beelzebuth | Female, ベル == Bel | Female, アズラエル == Azriel | Female, アズ == Az | Female, フレイア == Freya | Female'
if fullPromptFlag: if fullPromptFlag:
system = PROMPT system = PROMPT
user = 'Text to Translate: ' + subbedT user = 'Current Text to Translate: ' + subbedT
else: else:
system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.'
user = 'Text to Translate: ' + subbedT user = 'Current Text to Translate: ' + subbedT
response = openai.ChatCompletion.create( response = openai.ChatCompletion.create(
temperature=0, temperature=0,
model="gpt-3.5-turbo", model="gpt-3.5-turbo",
@ -1235,10 +1243,10 @@ def translateGPT(t, history, fullPromptFlag):
# Remove Placeholder Text # Remove Placeholder Text
translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('English Translation: ', '')
translatedText = translatedText.replace('Translation: ', '') translatedText = translatedText.replace('Translation: ', '')
translatedText = translatedText.replace('Text to Translate: ', '') translatedText = translatedText.replace('Current Text to Translate: ', '')
translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('English Translation:', '')
translatedText = translatedText.replace('Translation:', '') translatedText = translatedText.replace('Translation:', '')
translatedText = translatedText.replace('Text to Translate:', '') translatedText = translatedText.replace('Current Text to Translate:', '')
# Return Translation # Return Translation
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: