diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index b702908..8a72292 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -25,7 +25,7 @@ APICOST = .002 # Depends on the model https://openai.com/pricing PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = 20 # For GPT4 rate limit will be hit if you have more than 1 thread. LOCK = threading.Lock() -WIDTH = 60 +WIDTH = 50 LISTWIDTH = 60 MAXHISTORY = 10 ESTIMATE = '' @@ -44,7 +44,7 @@ CODE401 = True CODE405 = False CODE102 = True CODE122 = False -CODE101 = False +CODE101 = True CODE355655 = False CODE357 = False CODE657 = False @@ -216,22 +216,19 @@ def translateNote(event, regex): match = re.findall(regex, jaString, re.DOTALL) if match: - jaString = match[0] - # Need to remove outside code - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】]+', '', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？]+$', '', jaString) - oldjaString = jaString - + oldJAString = match[0] # Remove any textwrap - jaString = re.sub(r'\n', ' ', jaString) - - response = translateGPT(jaString, '', True) + jaString = re.sub(r'\n', ' ', oldJAString) + + # Translate + response = translateGPT(jaString, 'Reply with the English translation of the NPC name.', True) translatedText = response[0] # Textwrap translatedText = textwrap.fill(translatedText, width=LISTWIDTH) - event['note'] = event['note'].replace(oldjaString, translatedText) + translatedText = translatedText.replace('\"', '') + event['note'] = event['note'].replace(oldJAString, translatedText) return response[1] return 0 @@ -434,10 +431,12 @@ def searchNames(name, pbar, context): responseList.append(translateGPT(name['description'], '', True)) else: responseList.append(['', 0]) + if 'hint' in name['note']: + tokens += translateNote(name, r']*)>') if 'Enemies' in context: - if 'desc1' in name['note']: - tokens += translateNote(name, r']*)>') + if 'taneoya' in name['note']: + tokens += translateNote(name, r']*)>') if 'desc2' in name['note']: tokens += translateNote(name, r']*)>') @@ -506,8 +505,6 @@ def searchCodes(page, pbar): # if jaString.startswith('　'): # jaString = jaString + ': ' - jaString = re.sub(r'([\u3000-\uffef])\1{3,}', r'\1\1\1', jaString) - # Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior) currentGroup.append(jaString) @@ -517,7 +514,6 @@ def searchCodes(page, pbar): i += 1 jaString = codeList[i]['parameters'][0] - jaString = re.sub(r'([\u3000-\uffef])\1{3,}', r'\1\1\1', jaString) currentGroup.append(jaString) # Join up 401 groups for better translation. @@ -532,7 +528,6 @@ def searchCodes(page, pbar): response = translateGPT(match[0][1], 'Reply with only the english translation of the NPC name', True) tokens += response[1] speaker = response[0].strip('.') - speakerVar = match[0][0].replace(match[0][1], response[0]) finalJAString = finalJAString.replace(match[0][0], '') # Put names in list @@ -571,7 +566,7 @@ def searchCodes(page, pbar): textHistory.append('\"' + translatedText + '\"') # Remove added speaker - translatedText = translatedText.replace(speaker + ': ', speakerVar) + translatedText = re.sub(r'^.+?:\s', '', translatedText) speaker = '' speakerVar = '' else: @@ -763,6 +758,9 @@ def searchCodes(page, pbar): # Set Data speaker = translatedText codeList[i]['parameters'][4] = translatedText + if NAMES == True and speaker not in NAMESLIST: + with LOCK: + NAMESLIST.append(speaker) ## Event Code: 355 or 655 Scripts [Optional] if (codeList[i]['code'] == 355) and CODE355655 == True: @@ -1089,16 +1087,16 @@ def searchSS(state, pbar): tokens = 0 # Name - nameResponse = translateGPT(state['name'], 'Reply with only the english translation of the RPG Skill name.', False) if 'name' in state else '' + nameResponse = translateGPT(state['name'], 'Reply with only the english translation of the RPG Skill name.', True) if 'name' in state else '' # Description - descriptionResponse = translateGPT(state['description'], 'Reply with only the english translation of the description.', False) if 'description' in state else '' + descriptionResponse = translateGPT(state['description'], 'Reply with only the english translation of the description.', True) if 'description' in state else '' # Messages - message1Response = translateGPT('Taro' + state['message1'], 'reply with only the gender neutral english translation of the text.', False) if 'message1' in state else '' - message2Response = translateGPT(state['message2'], 'reply with only the english translation of the text.', False) if 'message2' in state else '' - message3Response = translateGPT(state['message3'], 'reply with only the english translation of the text.', False) if 'message3' in state else '' - message4Response = translateGPT(state['message4'], 'reply with only the english translation of the text.', False) if 'message4' in state else '' + message1Response = translateGPT('Taro' + state['message1'], 'reply with only the gender neutral english translation of the action.', True) if 'message1' in state else '' + message2Response = translateGPT(state['message2'], 'reply with only the english translation of the text.', True) if 'message2' in state else '' + message3Response = translateGPT(state['message3'], 'reply with only the english translation of the text.', True) if 'message3' in state else '' + message4Response = translateGPT(state['message4'], 'reply with only the english translation of the text.', True) if 'message4' in state else '' # if 'note' in state: if 'DOBBY' in state['note']: @@ -1243,15 +1241,16 @@ def translateGPT(t, history, fullPromptFlag): return(t, 0) """Translate text using GPT""" - context = 'Eroge Names Context: ミカエル == Mikael | Female, ミカ == Mika | Female, ベルゼビュート == Beelzebuth | Female, ベル == Bel | Female, アズラエル == Azriel | Female, アズ == Az | Female, フレイア == Freya | Female' + context = 'Eroge Names Context: カレン == Karen | Female, エリス == Eris | Female, コレット == Colette | Female, テオ == Teo | Male, メイヴィス == Mavis | Female, ' if fullPromptFlag: system = PROMPT - user = 'Current Text to Translate: ' + subbedT + user = 'Line to Translate: ' + subbedT else: system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Current Text to Translate: ' + subbedT + user = 'Line to Translate: ' + subbedT response = openai.ChatCompletion.create( temperature=0, + frequency_penalty=1, model="gpt-3.5-turbo", messages=[ {"role": "system", "content": system}, @@ -1272,10 +1271,10 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Current Text to Translate: ', '') + translatedText = translatedText.replace('Line to Translate: ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Current Text to Translate:', '') + translatedText = translatedText.replace('Line to Translate:', '') # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: diff --git a/modules/textfile.py b/modules/textfile.py index 68c441e..4176c88 100644 --- a/modules/textfile.py +++ b/modules/textfile.py @@ -131,40 +131,47 @@ def translateText(data, pbar): tokens = 0 speaker = '' speakerFlag = False + currentGroup = [] + syncIndex = 0 for i in range(len(data)): - if '◆' in data[i]: - jaString = data[i] + if i != syncIndex: + continue + + match = re.findall(r'm\[[0-9]+\] = \"(.+?)\"', data[i]) + if len(match) > 0: + jaString = match[0] ### Translate # Remove any textwrap jaString = re.sub(r'\\n', ' ', jaString) - # Check if speaker - if '◆A' in jaString: - speakerFlag = True + # Grab Speaker + speakerMatch = re.findall(r's\[[0-9]+\] = \"(.+?)[／\"]', data[i-1]) + if len(match) != 0: + response = translateGPT(speakerMatch[0], 'Reply with only the english translation of the NPC name', True) + tokens += response[1] + speaker = response[0].strip('.') - # Need to remove outside code and put it back later - startString = re.search(r'^◆[a-zA-Z0-9]+◆', jaString) - jaString = re.sub(r'^◆[a-zA-Z0-9]+◆', '', jaString) - endString = re.search(r'\n$', jaString) - jaString = re.sub(r'\n$', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() - if endString is None: endString = '' - else: endString = endString.group() - - # Remove Repeating Chars - jaString = re.sub(r'([\u3000-\uffef])\1{1,}', r'\1', jaString) + # Grab rest of the messages + currentGroup.append(jaString) + start = i + while (re.search(r'm\[[0-9]+\] = \"(.+?)\"', data[i+1]) != None): + i+=1 + match = re.findall(r'm\[[0-9]+\] = \"(.+?)\"', data[i]) + currentGroup.append(match[0]) + finalJAString = ''.join(currentGroup) # Translate if speaker != '': - response = translateGPT(jaString, 'Previous Text for Context: ' + ' '.join(textHistory) \ - + '\n\n\n###\n\n\nCurrent Speaker: ' + speaker, True) + response = translateGPT(f'{speaker}: {finalJAString}', 'Previous Text for Context: ' + ' '.join(textHistory), True) else: - response = translateGPT(jaString, 'Previous Text for Context: ' + ' '.join(textHistory), True) + response = translateGPT(finalJAString, 'Previous Text for Context: ' + ' '.join(textHistory), True) tokens += response[1] translatedText = response[0] + + # Remove added speaker + translatedText = re.sub(r'^.+?:\s', '', translatedText) # TextHistory is what we use to give GPT Context, so thats appended here. # rawTranslatedText = re.sub(r'[\\<>]+[a-zA-Z]+\[[a-zA-Z0-9]+\]', '', translatedText) @@ -176,6 +183,7 @@ def translateText(data, pbar): # Keep textHistory list at length maxHistory if len(textHistory) > maxHistory: textHistory.pop(0) + currentGroup = [] # Textwrap translatedText = textwrap.fill(translatedText, width=WIDTH) @@ -193,45 +201,93 @@ def translateText(data, pbar): speakerFlag = False # Write - data[i] = startString + translatedText + endString + data[i] = translatedText + syncIndex = i + 1 pbar.update() return [data, tokens] +def subVars(jaString): + varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\\\[\]\.<>a-zA-Z0-9]+' + count = 0 + + varList = re.findall(varRegex, jaString) + if len(varList) != 0: + for var in varList: + jaString = jaString.replace(var, '') + count += 1 + + return [jaString, varList] + +def resubVars(translatedText, varList): + count = 0 + + if len(varList) != 0: + for var in varList: + translatedText = translatedText.replace('', var) + count += 1 + + # Remove Color Variables Spaces + if '\\c' in translatedText: + translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r'\1', translatedText) + translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) + return translatedText + @retry(exceptions=Exception, tries=5, delay=5) def translateGPT(t, history, fullPromptFlag): with LOCK: # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, 0) + # Sub Vars + varResponse = subVars(t) + subbedT = varResponse[0] + # If there isn't any Japanese in the text just skip - if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+', t): + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', subbedT): return(t, 0) """Translate text using GPT""" + context = 'Eroge Names Context: カレン == Karen | Female, エリス == Eris | Female, コレット == Colette | Female, テオ == Teo | Male, メイヴィス == Mavis | Female, ' if fullPromptFlag: - system = PROMPT + history + system = PROMPT + user = 'Line to Translate: ' + subbedT else: - system = 'You are going to pretend to be Japanese visual novel translator, \ -editor, and localizer. ' + history + system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' + user = 'Line to Translate: ' + subbedT response = openai.ChatCompletion.create( temperature=0, + frequency_penalty=1, model="gpt-3.5-turbo", messages=[ {"role": "system", "content": system}, - {"role": "user", "content": "Text to Translate: " + t} + {"role": "user", "content": context}, + {"role": "user", "content": history}, + {"role": "user", "content": user} ], request_timeout=30, ) - # Make sure translation didn't wonk out - mlen=len(response.choices[0].message.content) - elnt=10*len(t) - if len(response.choices[0].message.content) > 9 * len(t): + # Save Translated Text + translatedText = response.choices[0].message.content + tokens = response.usage.total_tokens + + # Resub Vars + translatedText = resubVars(translatedText, varResponse[1]) + + # Remove Placeholder Text + translatedText = translatedText.replace('English Translation: ', '') + translatedText = translatedText.replace('Translation: ', '') + translatedText = translatedText.replace('Line to Translate: ', '') + translatedText = translatedText.replace('English Translation:', '') + translatedText = translatedText.replace('Translation:', '') + translatedText = translatedText.replace('Line to Translate:', '') + + # Return Translation + if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: return [t, response.usage.total_tokens] else: - return [response.choices[0].message.content, response.usage.total_tokens] - + return [translatedText, tokens] \ No newline at end of file