From ee1b1804340e3a4760d93ba711db3682b19f177f Mon Sep 17 00:00:00 2001 From: Dazed Date: Wed, 16 Aug 2023 17:30:45 -0500 Subject: [PATCH] Adjust csvtl.py for game --- modules/csvtl.py | 77 +++++++++++++++++++++++++++-------------- modules/rpgmakermvmz.py | 70 ++++++++++++++++++++----------------- 2 files changed, 90 insertions(+), 57 deletions(-) diff --git a/modules/csvtl.py b/modules/csvtl.py index 7d4f272..07c09c7 100644 --- a/modules/csvtl.py +++ b/modules/csvtl.py @@ -64,7 +64,7 @@ def handleCSV(filename, estimate): return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') def openFiles(filename, writeFile): - with open('files/' + filename, 'r', encoding='UTF-8') as readFile, writeFile: + with open('files/' + filename, 'r', encoding='shift-jis') as readFile, writeFile: translatedData = parseCSV(readFile, writeFile, filename) return translatedData @@ -108,7 +108,7 @@ def parseCSV(readFile, writeFile, filename): readFile.seek(0) reader = csv.reader(readFile, delimiter=',',) - writer = csv.writer(writeFile, delimiter=',', doublequote=None) + writer = csv.writer(writeFile, delimiter=',', quotechar='\"') with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: pbar.desc=filename @@ -162,31 +162,56 @@ def translateCSV(row, pbar, writer, textHistory, format): pbar.update(1) # TextHistory is what we use to give GPT Context, so thats appended here. - textHistory.append(': ' + translatedText) + textHistory.append('\"' + translatedText + '\"') # Translate Everything case '2': for i in range(len(row)): - if i not in [2,3,4,5,6,7,8,9,10]: + if i not in [1]: continue jaString = row[i] - if '_' in jaString: - continue - #Translate - response = translateGPT(jaString, '', True) - translatedText = response[0] - tokens += response[1] + matchList = re.findall(r':name\[(.+?),.+?\](.+?[」）]+?)', jaString) - # Textwrap - translatedText = textwrap.fill(translatedText, width=WIDTH) + for match in matchList: + speaker = match[0] + text = match[1] - # Set Data - translatedText = translatedText.replace('\"', '') - translatedText = translatedText.replace('\'', '') - translatedText = translatedText.replace(',', '') - row[i] = translatedText - writer.writerow(row) + # Translate Speaker + response = translateGPT (speaker, 'Reply with the English translation of the NPC name.', True) + translatedSpeaker = response[0] + tokens += response[1] + + # Translate Line + response = translateGPT(translatedSpeaker + ': ' + text, 'Previous Translated Text: ' + '|'.join(textHistory), True) + translatedText = response[0] + tokens += response[1] + + # TextHistory is what we use to give GPT Context, so thats appended here. + textHistory.append(translatedText) + + # Remove Speaker from translated text + translatedText = re.sub(r'.+?: ', '', translatedText) + + # Set Data + translatedText = translatedText.replace('\"', '') + translatedText = translatedText.replace('「', '') + translatedText = translatedText.replace('」', '') + translatedText = translatedText.replace('\n', '') + + # Textwrap + translatedText = textwrap.fill(translatedText, width=WIDTH) + + translatedText = '「' + translatedText + '」\n' + row[i] = re.sub(rf'\b{re.escape(speaker)}\b', translatedSpeaker, row[i]) + row[i] = row[i].replace(text, translatedText) + + # Keep textHistory list at length maxHistory + with LOCK: + if len(textHistory) > maxHistory: + textHistory.pop(0) + if not ESTIMATE: + writer.writerow(row) pbar.update(1) except Exception as e: @@ -197,13 +222,13 @@ def translateCSV(row, pbar, writer, textHistory, format): def subVars(jaString): - varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\.<>a-zA-Z]+' + varRegex = r'\\+[a-zA-Z]+\[.+?\]|[\\]+[#|]+|\\+[\\\[\]\.<>a-zA-Z0-9]+' count = 0 varList = re.findall(varRegex, jaString) if len(varList) != 0: for var in varList: - jaString = jaString.replace(var, '[v' + str(count) + ']') + jaString = jaString.replace(var, '') count += 1 return [jaString, varList] @@ -213,7 +238,7 @@ def resubVars(translatedText, varList): if len(varList) != 0: for var in varList: - translatedText = translatedText.replace('[v' + str(count) + ']', var) + translatedText = translatedText.replace('', var) count += 1 # Remove Color Variables Spaces @@ -241,13 +266,13 @@ def translateGPT(t, history, fullPromptFlag): return(t, 0) """Translate text using GPT""" - context = 'Character Context: クレア == Clea | Female, ノラ == Nora | Female, リルム == Relm | Female, ソフィア == Sophia | Female, セリス == Celis | Female, ピステ == Piste | Female, イクト == Ect | Female, カロン == Caron | Female, エモニ == Emoni | Female, アミナ == Amina | Female, アルマダ == Armada | Female, フォニ == Phoni | Female, エレオ == Eleo | Female, ペルノ == Perno | Female' + context = 'Eroge Names Context: ミカエル == Mikael | Female, ミカ == Mika | Female, ベルゼビュート == Beelzebuth | Female, ベル == Bel | Female, アズラエル == Azriel | Female, アズ == Az | Female, フレイア == Freya | Female' if fullPromptFlag: system = PROMPT - user = 'Text to Translate: ' + subbedT + user = 'Current Text to Translate: ' + subbedT else: system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Text to Translate: ' + subbedT + user = 'Current Text to Translate: ' + subbedT response = openai.ChatCompletion.create( temperature=0, model="gpt-3.5-turbo", @@ -270,10 +295,10 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Text to Translate: ', '') + translatedText = translatedText.replace('Current Text to Translate: ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Text to Translate:', '') + translatedText = translatedText.replace('Current Text to Translate:', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index fa1ee65..9b869a4 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -32,6 +32,7 @@ ESTIMATE = '' TOTALCOST = 0 TOKENS = 0 TOTALTOKENS = 0 +NAMESLIST = [] #tqdm Globals BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' @@ -48,11 +49,12 @@ CODE355655 = False CODE357 = False CODE657 = False CODE356 = False -CODE320 = False +CODE320 = True CODE324 = False CODE111 = False CODE408 = False CODE108 = False +NAMES = True # Output a list of all the character names found def handleMVMZ(filename, estimate): global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST @@ -65,6 +67,7 @@ def handleMVMZ(filename, estimate): # Print Result end = time.time() tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) + tqdm.write(str(NAMESLIST)) with LOCK: TOTALCOST += TOKENS * .001 * APICOST TOTALTOKENS += TOKENS @@ -452,6 +455,7 @@ def searchCodes(page, pbar): speakerVar = '' match = [] global LOCK + global NAMESLIST try: for i in range(len(page['list'])): @@ -494,14 +498,20 @@ def searchCodes(page, pbar): # Check for speaker if '\\N' in finalJAString or '\\n' in finalJAString: - match = re.findall(r'([\\]+[Nn]<([一-龠ぁ-ゔァ-ヴー]+)>)', finalJAString) + match = re.findall(r'([\\]+[Nn]<(.+)>)', finalJAString) if len(match) != 0: response = translateGPT(match[0][1], 'Reply with only the english translation of the NPC name', True) tokens += response[1] speaker = response[0].strip('.') speakerVar = match[0][0].replace(match[0][1], response[0]) finalJAString = finalJAString.replace(match[0][0], '') + + # Put names in list + if NAMES == True and speaker not in NAMESLIST: + with LOCK: + NAMESLIST.append(speaker) + # Need to remove outside code and put it back later startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', finalJAString) finalJAString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', '', finalJAString) @@ -521,12 +531,12 @@ def searchCodes(page, pbar): # Translate if speaker == '' and finalJAString != '': - response = translateGPT(finalJAString, 'Previous Dialogue: ' + '|'.join(textHistory), True) + response = translateGPT(finalJAString, 'Previous Translated Text: ' + '|'.join(textHistory), True) tokens += response[1] translatedText = response[0] textHistory.append('\"' + translatedText + '\"') elif finalJAString != '': - response = translateGPT(speaker + ': ' + finalJAString, 'Previous Dialogue: ' + '|'.join(textHistory), True) + response = translateGPT(speaker + ': ' + finalJAString, 'Previous Translated Text: ' + '|'.join(textHistory), True) tokens += response[1] translatedText = response[0] textHistory.append('\"' + translatedText + '\"') @@ -774,8 +784,8 @@ def searchCodes(page, pbar): continue # Want to translate this script - if 'ans:' not in jaString: - continue + # if 'ans:' not in jaString: + # continue # Need to remove outside code and put it back later startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】]+', jaString) @@ -981,32 +991,30 @@ def searchCodes(page, pbar): page['list'][i]['parameters'][j] = translatedText ### Event Code: 320 Set Variable - if page['list'][i]['code'] == 320 and CODE320 == True or page['list'][i]['code'] == 324 and CODE324 == True: + if page['list'][i]['code'] == 320 and CODE320 == True: jaString = page['list'][i]['parameters'][1] + if type(jaString) != str: + continue + + # Definitely don't want to mess with files + if '■' in jaString or '_' in jaString: + continue - # Need to remove outside code and put it back later - startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】a-zA-Z\\]+', jaString) - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】a-zA-Z\\]+', '', jaString) - endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー【】。、…！？]+$', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー【】。、…！？]+$', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() - if endString is None: endString = '' - else: endString = endString.group() - - response = translateGPT(jaString, 'Reply with only the english translation of the npc nickname.', False) + # If there isn't any Japanese in the text just skip + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): + continue + + response = translateGPT(jaString, 'Reply with the English translation of the NPC name.', True) translatedText = response[0] + tokens += response[1] # Remove characters that may break scripts - charList = ['\"'] + charList = ['.', '\"', '\'', '\\n'] for char in charList: translatedText = translatedText.replace(char, '') - translatedText = translatedText.strip('.') - # Set Data - tokens += response[1] - page['list'][i]['parameters'][1] = startString + translatedText + endString + page['list'][i]['parameters'][1] = translatedText except IndexError as e: # This is part of the logic so we just pass it @@ -1162,13 +1170,13 @@ def searchSystem(data, pbar): return tokens def subVars(jaString): - varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\.<>a-zA-Z]+' + varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\\\[\]\.<>a-zA-Z0-9]+' count = 0 varList = re.findall(varRegex, jaString) if len(varList) != 0: for var in varList: - jaString = jaString.replace(var, '[v' + str(count) + ']') + jaString = jaString.replace(var, '') count += 1 return [jaString, varList] @@ -1178,7 +1186,7 @@ def resubVars(translatedText, varList): if len(varList) != 0: for var in varList: - translatedText = translatedText.replace('[v' + str(count) + ']', var) + translatedText = translatedText.replace('', var) count += 1 # Remove Color Variables Spaces @@ -1206,13 +1214,13 @@ def translateGPT(t, history, fullPromptFlag): return(t, 0) """Translate text using GPT""" - context = 'Eroge Characters Context: クレア == Clea | Female, ノラ == Nora | Female, リルム == Relm | Female, ソフィア == Sophia | Female, セリス == Celis | Female, ピステ == Piste | Female, イクト == Ect | Female, カロン == Caron | Female, エモニ == Emoni | Female, アミナ == Amina | Female, アルマダ == Armada | Female, フォニ == Phoni | Female, エレオ == Eleo | Female, ペルノ == Perno | Female' + context = 'Eroge Names Context: ミカエル == Mikael | Female, ミカ == Mika | Female, ベルゼビュート == Beelzebuth | Female, ベル == Bel | Female, アズラエル == Azriel | Female, アズ == Az | Female, フレイア == Freya | Female' if fullPromptFlag: system = PROMPT - user = 'Text to Translate: ' + subbedT + user = 'Current Text to Translate: ' + subbedT else: system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Text to Translate: ' + subbedT + user = 'Current Text to Translate: ' + subbedT response = openai.ChatCompletion.create( temperature=0, model="gpt-3.5-turbo", @@ -1235,10 +1243,10 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Text to Translate: ', '') + translatedText = translatedText.replace('Current Text to Translate: ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Text to Translate:', '') + translatedText = translatedText.replace('Current Text to Translate:', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: