From cabaf9f82a26e8bf2722fa6651a351ab9696408b Mon Sep 17 00:00:00 2001 From: Dazed Date: Mon, 18 Sep 2023 04:52:30 -0500 Subject: [PATCH] Update the scripts --- modules/main.py | 9 +- modules/rpgmakerace.py | 804 ++++++++++++++++++++++++++-------------- modules/rpgmakermvmz.py | 9 +- 3 files changed, 532 insertions(+), 290 deletions(-) diff --git a/modules/main.py b/modules/main.py index 3423714..4e52e88 100644 --- a/modules/main.py +++ b/modules/main.py @@ -9,7 +9,7 @@ from modules.rpgmakerace import handleACE from modules.csvtl import handleCSV from modules.textfile import handleTextfile -THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread. +THREADS = 20 # For GPT4 rate limit will be hit if you have more than 1 thread. # Info Message print(Fore.LIGHTYELLOW_EX + "WARNING: Once a translation starts do not close it unless you want to lose your\ @@ -88,9 +88,10 @@ def main(): case _: version = '' - if estimate == False and totalCost != 'Fail': - # This is to encourage people to grab what's in /translated instead - deleteFolderFiles('files') + if totalCost != 'Fail': + if estimate == False: + # This is to encourage people to grab what's in /translated instead + deleteFolderFiles('files') # Prevent immediately closing of CLI print(totalCost) diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index e4abc9a..544f9c4 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -33,6 +33,7 @@ ESTIMATE = '' TOTALCOST = 0 TOKENS = 0 TOTALTOKENS = 0 +NAMESLIST = [] #tqdm Globals BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' @@ -41,7 +42,7 @@ LEAVE=False # Flags CODE401 = True -CODE405 = True +CODE405 = False CODE102 = True CODE122 = False CODE101 = False @@ -53,6 +54,8 @@ CODE320 = False CODE324 = False CODE111 = False CODE408 = False +CODE108 = False +NAMES = False # Output a list of all the character names found def handleACE(filename, estimate): global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST @@ -65,6 +68,8 @@ def handleACE(filename, estimate): # Print Result end = time.time() tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) + if NAMES is True: + tqdm.write(str(NAMESLIST)) with LOCK: TOTALCOST += TOKENS * .001 * APICOST TOTALTOKENS += TOKENS @@ -73,20 +78,24 @@ def handleACE(filename, estimate): return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') else: - with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: - start = time.time() - translatedData = openFiles(filename) + try: + with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: + start = time.time() + translatedData = openFiles(filename) - # Print Result - end = time.time() - yaml=YAML(pure=True) - yaml.width = 4096 - yaml.default_style = "'" - yaml.dump(translatedData[0], outFile) - tqdm.write(getResultString(translatedData, end - start, filename)) - with LOCK: - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] + # Print Result + end = time.time() + yaml=YAML(pure=True) + yaml.width = 4096 + yaml.default_style = "'" + yaml.dump(translatedData[0], outFile) + tqdm.write(getResultString(translatedData, end - start, filename)) + with LOCK: + TOTALCOST += translatedData[1] * .001 * APICOST + TOTALTOKENS += translatedData[1] + except Exception as e: + traceback.print_exc() + return 'Fail' return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') @@ -99,7 +108,7 @@ def openFiles(filename): data = yaml.load(f) # Map Files - if 'Map' in filename and filename != 'MapInfos.yaml': + if 'Map' in filename and filename != 'MapInfos.json': translatedData = parseMap(data, filename) # CommonEvents Files @@ -150,6 +159,10 @@ def openFiles(filename): elif 'System' in filename: translatedData = parseSystem(data, filename) + # Scenario File + elif 'Scenario' in filename: + translatedData = parseScenario(data, filename) + else: raise NameError(filename + ' Not Supported') @@ -208,24 +221,21 @@ def translateNote(event, regex): # Regex that only matches text inside LB. jaString = event['note'] - match = re.search(regex, jaString) + match = re.findall(regex, jaString, re.DOTALL) if match: - jaString = match.group(1) - # Need to remove outside code - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】]+', '', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？]+$', '', jaString) - oldjaString = jaString - + oldJAString = match[0] # Remove any textwrap - jaString = re.sub(r'\n', ' ', jaString) - - response = translateGPT(jaString, '', True) + jaString = re.sub(r'\n', ' ', oldJAString) + + # Translate + response = translateGPT(jaString, 'Reply with the English translation of the NPC name.', True) translatedText = response[0] # Textwrap translatedText = textwrap.fill(translatedText, width=LISTWIDTH) - event['note'] = event['note'].replace(oldjaString, translatedText) + translatedText = translatedText.replace('\"', '') + event['note'] = event['note'].replace(oldJAString, translatedText) return response[1] return 0 @@ -350,29 +360,56 @@ def parseSystem(data, filename): return [data, totalTokens, e] return [data, totalTokens, None] +def parseScenario(data, filename): + totalTokens = 0 + totalLines = 0 + global LOCK + + # Get total for progress bar + for page in data.items(): + totalLines += len(page[1]) + + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc=filename + pbar.total=totalLines + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(searchCodes, page[1], pbar) for page in data.items() if page[1] is not None] + for future in as_completed(futures): + try: + totalTokens += future.result() + except Exception as e: + return [data, totalTokens, e] + return [data, totalTokens, None] + def searchThings(name, pbar): tokens = 0 - # Set the context of what we are translating - responseList = [] - responseList.append(translateGPT(name['name'], 'Reply with only the English translation of the RPG Item name.', False)) - responseList.append(translateGPT(name['description'], 'Reply with only the English translation of the description.', False)) + # Name + nameResponse = translateGPT(name['name'], 'Reply with only the english translation of the RPG item name.', False) if 'name' in name else '' - # if '') + # Description + descriptionResponse = translateGPT(name['description'], 'Reply with only the english translation of the description.', False) if 'description' in name else '' - # Extract all our translations in a list from response - for i in range(len(responseList)): - tokens += responseList[i][1] - responseList[i] = responseList[i][0] + # Note + if '') + + # Count Tokens + tokens += nameResponse[1] if nameResponse != '' else 0 + tokens += descriptionResponse[1] if descriptionResponse != '' else 0 # Set Data - name['name'] = responseList[0].strip('.\"') - responseList[1] = textwrap.fill(responseList[1], LISTWIDTH) - name['description'] = responseList[1].strip('\"') - # name['note'] = responseList[2] - pbar.update(1) + if 'name' in name: + name['name'] = nameResponse[0].strip('\"') + if 'description' in name: + description = descriptionResponse[0] + # Remove Textwrap + description = description.replace('\n', ' ') + description = textwrap.fill(descriptionResponse[0], LISTWIDTH) + name['description'] = description.strip('\"') + + pbar.update(1) return tokens def searchNames(name, pbar, context): @@ -382,7 +419,7 @@ def searchNames(name, pbar, context): if 'Actors' in context: newContext = 'Reply with only the english translation of the NPC name' if 'Armors' in context: - newContext = 'Reply with only the english translation of the RPG armor/clothing name' + newContext = 'Reply with only the english translation of the RPG equipment name' if 'Classes' in context: newContext = 'Reply with only the english translation of the RPG class name' if 'MapInfos' in context: @@ -394,17 +431,22 @@ def searchNames(name, pbar, context): # Extract Data responseList = [] - responseList.append(translateGPT(name['name'], newContext, False)) + responseList.append(translateGPT(name['name'], newContext, True)) if 'Actors' in context: responseList.append(translateGPT(name['description'], '', True)) - responseList.append(translateGPT(name['nickname'], 'Reply with ONLY the english translation of the NPC nickname', False)) + responseList.append(translateGPT(name['nickname'], 'Reply with ONLY the english translation of the NPC nickname', True)) if 'Armors' in context or 'Weapons' in context: - responseList.append(translateGPT(name['description'], '', True)) + if 'description' in name: + responseList.append(translateGPT(name['description'], '', True)) + else: + responseList.append(['', 0]) + if 'hint' in name['note']: + tokens += translateNote(name, r'\n([\s\S]*?)\n') if 'Enemies' in context: - if 'desc1' in name['note']: - tokens += translateNote(name, r']*)>') + if 'variable_update_skill' in name['note']: + tokens += translateNote(name, r'111:(.+?)\n') if 'desc2' in name['note']: tokens += translateNote(name, r']*)>') @@ -424,12 +466,15 @@ def searchNames(name, pbar, context): name['profile'] = translatedText.strip('\"') translatedText = textwrap.fill(responseList[2], LISTWIDTH) name['nickname'] = translatedText.strip('\"') + if '<特徴1:' in name['note']: + tokens += translateNote(name, r'<特徴1:([^>]*)>') if 'Armors' in context or 'Weapons' in context: translatedText = textwrap.fill(responseList[1], LISTWIDTH) - name['description'] = translatedText.strip('\"') - if ']*)>') + if 'description' in name: + name['description'] = translatedText.strip('\"') + if '\n([\s\S]*?)\n') pbar.update(1) return tokens @@ -441,62 +486,101 @@ def searchCodes(page, pbar): maxHistory = MAXHISTORY tokens = 0 speaker = '' + speakerVar = '' + nametag = '' match = [] + syncIndex = 0 global LOCK + global NAMESLIST try: - for i in range(len(page['list'])): - with LOCK: + if 'list' in page: + codeList = page['list'] + else: + codeList = page + for i in range(len(codeList)): + with LOCK: + if syncIndex > i: + i = syncIndex pbar.update(1) ### All the codes are here which translate specific functions in the MAP files. ### IF these crash or fail your game will do the same. Use the flags to skip codes. ## Event Code: 401 Show Text - if page['list'][i]['c'] == 401 and CODE401 == True or page['list'][i]['c'] == 405 and CODE405: + if codeList[i]['c'] == 401 and CODE401 == True or codeList[i]['c'] == 405 and CODE405: # Use this to place text later + code = codeList[i]['c'] j = i # Grab String - jaString = page['list'][i]['p'][0] - - # # Catch Speaker - # if jaString.startswith('　'): - # jaString = jaString + ': ' - - jaString = re.sub(r'([\u3000-\uffef])\1{3,}', r'\1\1\1', jaString) + jaString = codeList[i]['p'][0] + firstJAString = jaString # Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior) currentGroup.append(jaString) - while (page['list'][i+1]['c'] == 401 or page['list'][i+1]['c'] == 405): - page['list'][i]['p'][0] = '' - i += 1 + if len(codeList) > i+1: + while (codeList[i+1]['c'] == 401 or codeList[i+1]['c'] == 405): + codeList[i]['p'][0] = '' + codeList[i]['c'] = 0 + i += 1 - jaString = page['list'][i]['p'][0] - jaString = re.sub(r'([\u3000-\uffef])\1{3,}', r'\1\1\1', jaString) - currentGroup.append(jaString) + jaString = codeList[i]['p'][0] + currentGroup.append(jaString) # Join up 401 groups for better translation. if len(currentGroup) > 0: finalJAString = ''.join(currentGroup) oldjaString = finalJAString - # Check for speaker - if '\\N' in finalJAString or '\\n' in finalJAString: - match = re.findall(r'([\\]+[Nn]<([一-龠ぁ-ゔァ-ヴー]+)>)', finalJAString) - if len(match) != 0: - response = translateGPT(match[0][1], 'Reply with only the english translation of the NPC name', True) - tokens += response[1] - speaker = response[0].strip('.') - speakerVar = match[0][0].replace(match[0][1], response[0]) - finalJAString = finalJAString.replace(match[0][0], '') + # Color Regex: ^([\\]+[cC]\[[0-9]\]+(.+?)[\\]+[cC]\[[0]\]) + matchList = re.findall(r'(.*?([\\]+[nN]<(.+?)>).*)', finalJAString) + if len(matchList) > 0: + response = translateGPT(matchList[0][2], 'Reply with only the english translation of the NPC name', True) + tokens += response[1] + speaker = response[0].strip('.') + nametag = matchList[0][1].replace(matchList[0][2], speaker) + finalJAString = finalJAString.replace(matchList[0][1], '') - # Need to remove outside code and put it back later - startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', finalJAString) - finalJAString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）[]<>「」『』a-zA-Z0-9Ａ-Ｚ０-９\\]+', '', finalJAString) - if startString is None: startString = '' - else: startString = startString.group() + # Set next item as dialogue + if (codeList[j + 1]['c'] == -1 and len(codeList[j + 1]['p']) > 0) or codeList[j + 1]['c'] == -1: + # Set name var to top of list + codeList[j]['p'][0] = nametag + codeList[j]['c'] = code + + j += 1 + codeList[j]['p'][0] = finalJAString + codeList[j]['c'] = code + nametag = '' + else: + # Set nametag in string + codeList[j]['p'][0] = nametag + finalJAString + codeList[j]['c'] = code + + # Put names in list + if speaker not in NAMESLIST: + with LOCK: + NAMESLIST.append(speaker) + elif '\\kw' in finalJAString: + match = re.findall(r'\\+kw\[[0-9]+\]', finalJAString) + if len(match) != 0: + if '1' in match[0]: + speaker = 'Ayako Nagatsuki' + if '2' in match[0]: + speaker = 'Rei' + + # Set name var to top of list + codeList[j]['p'][0] = match[0] + codeList[j]['c'] = code + + # Set next item as dialogue + j += 1 + codeList[j]['p'][0] = match[0] + codeList[j]['c'] = code + + # Remove nametag from final string + finalJAString = finalJAString.replace(match[0], '') # Remove any textwrap finalJAString = re.sub(r'\n', ' ', finalJAString) @@ -508,52 +592,77 @@ def searchCodes(page, pbar): finalJAString = finalJAString.replace('―', '-') finalJAString = finalJAString.replace('…', '...') finalJAString = finalJAString.replace('　', '') + finalJAString = finalJAString.replace('\\#', '') + + # Remove any RPGMaker Code at start + startStringMatch = re.findall(r'^[\\]+[^一-龠ぁ-ゔァ-ヴーcnv]+\]', finalJAString) + if len(startStringMatch) > 0: + startString = startStringMatch[0] + finalJAString = finalJAString.replace(startString, '') + else: + startString = '' + + # Remove \\r codes (Display furigana instead of kanji) + rcodeMatch = re.findall(r'([\\]+r\[(.+?),.+?\])', finalJAString) + if len(rcodeMatch) > 0: + for match in rcodeMatch: + finalJAString = finalJAString.replace(match[0],match[1]) # Translate if speaker == '' and finalJAString != '': - response = translateGPT(finalJAString, 'Previous Dialogue: ' + '|'.join(textHistory), True) + response = translateGPT(finalJAString, 'Past Translated Text: ' + '|\n\n'.join(textHistory), True) tokens += response[1] translatedText = response[0] textHistory.append('\"' + translatedText + '\"') elif finalJAString != '': - response = translateGPT(speaker + ': ' + finalJAString, 'Previous Dialogue: ' + '|'.join(textHistory), True) + response = translateGPT(speaker + ': ' + finalJAString, 'Past Translated Text: ' + '|\n\n'.join(textHistory), True) tokens += response[1] translatedText = response[0] textHistory.append('\"' + translatedText + '\"') # Remove added speaker - translatedText = translatedText.replace(speaker + ': ', speakerVar) + translatedText = re.sub(r'^.+?:\s?', '', translatedText) speaker = '' - speakerVar = '' else: translatedText = finalJAString # Textwrap - translatedText = textwrap.fill(translatedText, width=WIDTH) + textList = re.findall(r'(^.+?)(\s.+)$', translatedText) # Strip out vars + if len(textList) > 0: + translatedText = textList[0][0] + textwrap.fill(textList[0][1], width=WIDTH) + else: + translatedText = textwrap.fill(translatedText, width=WIDTH) - # Resub start and end + # Add Beginning Text translatedText = startString + translatedText + translatedText = nametag + translatedText + nametag = '' # Set Data - translatedText = translatedText.replace('ッ', '') - translatedText = translatedText.replace('っ', '') - translatedText = translatedText.replace('ー', '') translatedText = translatedText.replace('\"', '') translatedText = translatedText.replace('\\CL ', '\\CL') translatedText = translatedText.replace('\\CL', '\\CL ') - page['list'][i]['p'][0] = '' - page['list'][j]['p'][0] = translatedText + codeList[i]['p'][0] = '' + codeList[i]['c'] = 0 + codeList[j]['p'][0] = translatedText + codeList[j]['c'] = code speaker = '' match = [] + syncIndex = i + 1 # Keep textHistory list at length maxHistory if len(textHistory) > maxHistory: textHistory.pop(0) - currentGroup = [] + currentGroup = [] - ## Event Code: 122 [Control Variables] [Optional] - if page['list'][i]['c'] == 122 and CODE122 == True: - jaString = page['list'][i]['p'][4] + ## Event Code: 122 [Set Variables] + if codeList[i]['c'] == 122 and CODE122 == True: + # This is going to be the var being set. (IMPORTANT) + varNum = codeList[i]['p'][0] + if varNum != 328: + continue + + jaString = codeList[i]['p'][4] if type(jaString) != str: continue @@ -561,46 +670,38 @@ def searchCodes(page, pbar): if '■' in jaString or '_' in jaString: continue - # If there isn't any Japanese in the text just skip - if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): - continue + # Need to remove outside code and put it back later + matchList = re.findall(r"[\'\"\`](.*?)[\'\"\`]", jaString) - # Definitely don't want to mess with files - if '\"' not in jaString: - continue + for match in matchList: + # Remove Textwrap + match = match.replace('\\n', '') + match = match.replace('\'', '') + response = translateGPT(match, 'Reply with the English translation.', True) + translatedText = response[0] + tokens += response[1] - # Remove outside text - oldjaString = jaString - startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', jaString) - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】\\]+', '', jaString) - endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー\<\>【】。！？\\]+$', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() - if endString is None: endString = '' - else: endString = endString.group() + # Replace + translatedText = jaString.replace(jaString, translatedText) - # Translate - response = translateGPT(jaString, '', True) - tokens += response[1] - translatedText = response[0] - - # Remove characters that may break scripts - charList = ['.', '\"', "\'"] - for char in charList: - translatedText = translatedText.replace(char, '') - - # Proper Formatting - translatedText = translatedText.replace('"', '\"') + # Remove characters that may break scripts + charList = ['.', '\"', '\\n'] + for char in charList: + translatedText = translatedText.replace(char, '') + + # Textwrap + translatedText = textwrap.fill(translatedText, width=30) + translatedText = translatedText.replace('\n', '\\n') + translatedText = translatedText.replace('\'', '\\\'') + translatedText = '\'' + translatedText + '\'' # Set Data - translatedText = startString + translatedText + endString - page['list'][i]['p'][4] = translatedText + codeList[i]['p'][4] = translatedText ## Event Code: 357 [Picture Text] [Optional] - if page['list'][i]['c'] == 357 and CODE357 == True: - if 'message' in page['list'][i]['p'][3]: - jaString = page['list'][i]['p'][3]['message'] + if codeList[i]['c'] == 357 and CODE357 == True: + if 'message' in codeList[i]['p'][3]: + jaString = codeList[i]['p'][3]['message'] if type(jaString) != str: continue @@ -631,12 +732,12 @@ def searchCodes(page, pbar): translatedText = textwrap.fill(translatedText, width=WIDTH) # Set Data - page['list'][i]['p'][3]['message'] = startString + translatedText + codeList[i]['p'][3]['message'] = startString + translatedText ## Event Code: 657 [Picture Text] [Optional] - if page['list'][i]['c'] == 657 and CODE657 == True: - if 'text' in page['list'][i]['p'][0]: - jaString = page['list'][i]['p'][0] + if codeList[i]['c'] == 657 and CODE657 == True: + if 'text' in codeList[i]['p'][0]: + jaString = codeList[i]['p'][0] if type(jaString) != str: continue @@ -678,11 +779,11 @@ def searchCodes(page, pbar): # Set Data if '\\' in jaString: print('Hi') - page['list'][i]['p'][0] = translatedText + codeList[i]['p'][0] = translatedText ## Event Code: 101 [Name] [Optional] - if page['list'][i]['c'] == 101 and CODE101 == True: - jaString = page['list'][i]['p'][4] + if codeList[i]['c'] == 101 and CODE101 == True: + jaString = codeList[i]['p'][4] if type(jaString) != str: continue @@ -719,22 +820,25 @@ def searchCodes(page, pbar): # Set Data speaker = translatedText - page['list'][i]['p'][4] = translatedText + codeList[i]['p'][4] = translatedText + if speaker not in NAMESLIST: + with LOCK: + NAMESLIST.append(speaker) ## Event Code: 355 or 655 Scripts [Optional] - if (page['list'][i]['c'] == 355) and CODE355655 == True: - jaString = page['list'][i]['p'][0] + if (codeList[i]['c'] == 355) and CODE355655 == True: + jaString = codeList[i]['p'][0] # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): continue # Want to translate this script - if page['list'][i]['c'] == 355 and '$game_actors[1].nickname' not in jaString: + if codeList[i]['c'] == 355 and 'BattleManager._logWindow.addText' not in jaString: continue # Don't want to touch certain scripts - if page['list'][i]['c'] == 655 and '.' in jaString: + if codeList[i]['c'] == 655 and '.' in jaString: continue # Need to remove outside code and put it back later @@ -748,7 +852,7 @@ def searchCodes(page, pbar): else: endString = endString.group() # Translate - response = translateGPT(jaString, '', True) + response = translateGPT(jaString, 'Reply with the English Translation of the text.', True) tokens += response[1] translatedText = response[0] @@ -759,21 +863,25 @@ def searchCodes(page, pbar): # Set Data translatedText = startString + translatedText + endString - page['list'][i]['p'][0] = translatedText + codeList[i]['p'][0] = translatedText ## Event Code: 408 (Script) - if (page['list'][i]['c'] == 408) and CODE408 == True: - jaString = page['list'][i]['p'][0] + if (codeList[i]['c'] == 408) and CODE408 == True: + jaString = codeList[i]['p'][0] # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): continue + # Want to translate this script + # if 'ans:' not in jaString: + # continue + # Need to remove outside code and put it back later startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】]+', jaString) jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】]+', '', jaString) - endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー【】。！？]+$', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー【】。！？]+$', '', jaString) + endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー【】。、…！？]+$', jaString) + jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー【】。、…！？]+$', '', jaString) if startString is None: startString = '' else: startString = startString.group() if endString is None: endString = '' @@ -794,11 +902,43 @@ def searchCodes(page, pbar): translatedText = translatedText.replace('"', '\"') # Set Data - page['list'][i]['p'][0] = translatedText + codeList[i]['p'][0] = translatedText + + ## Event Code: 108 (Script) + if (codeList[i]['c'] == 108) and CODE108 == True: + jaString = codeList[i]['p'][0] + + # If there isn't any Japanese in the text just skip + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): + continue + + # Want to translate this script + if 'text_indicator : ' not in jaString: + continue + + # Need to remove outside code and put it back later + matchList = re.findall(r'text_indicator : (.+)', jaString) + + # Translate + if len(matchList) > 0: + response = translateGPT(matchList[0], 'Reply with the English translation of the Location Title', True) + tokens += response[1] + translatedText = response[0] + + # Remove characters that may break scripts + charList = ['.', '\"'] + for char in charList: + translatedText = translatedText.replace(char, '') + + translatedText = jaString.replace(matchList[0], translatedText) + translatedText = translatedText.replace('"', '\"') + + # Set Data + codeList[i]['p'][0] = translatedText ## Event Code: 356 D_TEXT - if page['list'][i]['c'] == 356 and CODE356 == True: - jaString = page['list'][i]['p'][0] + if codeList[i]['c'] == 356 and CODE356 == True: + jaString = codeList[i]['p'][0] oldjaString = jaString # If there isn't any Japanese in the text just skip @@ -806,42 +946,77 @@ def searchCodes(page, pbar): continue # Want to translate this script - if 'addLog' not in jaString: + # if 'D_TEXT ' in jaString: + # # Remove any textwrap + # jaString = re.sub(r'\n', '_', jaString) + + # # Capture Arguments and text + # arg1 = re.findall(r'^(\S+)', jaString)[0] + # arg2 = re.findall(r'(\S+)$', jaString)[0] + # dtext = re.findall(r'(?<=\s)(.*?)(?=\s)', jaString)[0] + + # # Remove underscores + # dtext = re.sub(r'_', ' ', dtext) + + # # Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior) + # currentGroup.append(dtext) + + # while (codeList[i+1]['c'] == 356): + # # Want to translate this script + # if 'D_TEXT ' not in codeList[i+1]['p'][0]: + # break + + # codeList[i]['p'][0] = '' + # i += 1 + # jaString = codeList[i]['p'][0] + # dtext = re.findall(r'(?<=\s)(.*?)(?=\s)', jaString)[0] + # currentGroup.append(dtext) + + # # Join up 356 groups for better translation. + # if len(currentGroup) > 0: + # finalJAString = ' '.join(currentGroup) + # else: + # finalJAString = dtext + + # # Clear Group + # currentGroup = [] + if 'ShowInfo' in jaString: + # Remove any textwrap + jaString = re.sub(r'\n', '_', jaString) + + # Capture Arguments and text + matchList = re.findall(r'(ShowInfo) (.+)', jaString) + showInfo = matchList[0][0] + text = matchList[0][1] + + # Remove underscores + text = re.sub(r'_', ' ', text) + + # Translate + response = translateGPT(text, '', True) + translatedText = response[0] + tokens += response[1] + + # Remove characters that may break scripts + charList = ['.', '\"'] + for char in charList: + translatedText = translatedText.replace(char, '') + + # Cant have spaces? + translatedText = translatedText.replace(' ', '_') + + # Put Args Back + translatedText = showInfo + ' ' + translatedText + + # Set Data + codeList[i]['p'][0] = translatedText + else: continue - # Need to remove outside code and put it back later - startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】（）「」『』]+', jaString) - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】（）「」『』]+', '', jaString) - endString = re.search(r' [^一-龠ぁ-ゔァ-ヴー\<\>【】（）「」『』 。！？]+$', jaString) - jaString = re.sub(r' [^一-龠ぁ-ゔァ-ヴー\<\>【】（）「」『』 。！？]+$', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() - if endString is None: endString = '' - else: endString = endString.group() - - # Translate - response = translateGPT(jaString, 'Reply with only the English Translation of the text.', True) - tokens += response[1] - translatedText = response[0] - - # Remove characters that may break scripts - charList = ['.', '\"', '\\n'] - for char in charList: - translatedText = translatedText.replace(char, '') - - # Cant have spaces? - translatedText = translatedText.replace(' ', '　') - - # Textwrap - translatedText = textwrap.fill(translatedText, width=1000) - - # Set Data - page['list'][i]['p'][0] = startString + translatedText + endString - ### Event Code: 102 Show Choice - if page['list'][i]['c'] == 102 and CODE102 == True: - for choice in range(len(page['list'][i]['p'][0])): - jaString = page['list'][i]['p'][0][choice] + if codeList[i]['c'] == 102 and CODE102 == True: + for choice in range(len(codeList[i]['p'][0])): + jaString = codeList[i]['p'][0][choice] jaString = jaString.replace(' 。', '.') # Need to remove outside code and put it back later @@ -855,10 +1030,10 @@ def searchCodes(page, pbar): else: endString = endString.group() if len(textHistory) > 0: - response = translateGPT(jaString, 'Previous text for context: ' + textHistory[len(textHistory)-1], True) + response = translateGPT(jaString, 'Keep your translation as brief as possible. Previous text for context: ' + textHistory[len(textHistory)-1], True) translatedText = response[0] else: - response = translateGPT(jaString, '', True) + response = translateGPT(jaString, 'Keep your translation as brief as possible.', True) translatedText = response[0] # Remove characters that may break scripts @@ -868,73 +1043,77 @@ def searchCodes(page, pbar): # Set Data tokens += response[1] - page['list'][i]['p'][0][choice] = startString + translatedText + endString + codeList[i]['p'][0][choice] = startString + translatedText + endString ### Event Code: 111 Script - if page['list'][i]['c'] == 111 and CODE111 == True: - for j in range(len(page['list'][i]['p'])): - jaString = page['list'][i]['p'][j] + if codeList[i]['c'] == 111 and CODE111 == True: + for j in range(len(codeList[i]['p'])): + jaString = codeList[i]['p'][j] # Check if String if type(jaString) != str: continue + # Only TL the Game Variable + if '$gameVariables' not in jaString: + continue + + # This is going to be the var being set. (IMPORTANT) + if '1045' not in jaString: + continue + # Need to remove outside code and put it back later - startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】]+', jaString) - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー\<\>【】]+', '', jaString) - endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー【】 。！？]+$', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー【】 。！？]+$', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() - if endString is None: endString = '' - else: endString = endString.group() + matchList = re.findall(r"'(.*?)'", jaString) + + for match in matchList: + response = translateGPT(match, '', True) + translatedText = response[0] + tokens += response[1] - response = translateGPT(jaString, 'Reply with only the english translation.', True) - translatedText = response[0] + # Remove characters that may break scripts + charList = ['.', '\"', '\'', '\\n'] + for char in charList: + translatedText = translatedText.replace(char, '') - # Remove characters that may break scripts - charList = ['.', '\"', '\\n'] - for char in charList: - translatedText = translatedText.replace(char, '') + jaString = jaString.replace(match, translatedText) # Set Data - tokens += response[1] - page['list'][i]['p'][j] = startString + translatedText + endString + translatedText = jaString + codeList[i]['p'][j] = translatedText ### Event Code: 320 Set Variable - if page['list'][i]['c'] == 320 and CODE320 == True or page['list'][i]['c'] == 324 and CODE324 == True: - jaString = page['list'][i]['p'][1] + if codeList[i]['c'] == 320 and CODE320 == True: + jaString = codeList[i]['p'][1] + if type(jaString) != str: + continue + + # Definitely don't want to mess with files + if '■' in jaString or '_' in jaString: + continue - # Need to remove outside code and put it back later - startString = re.search(r'^[^一-龠ぁ-ゔァ-ヴー【】a-zA-Z\\]+', jaString) - jaString = re.sub(r'^[^一-龠ぁ-ゔァ-ヴー【】a-zA-Z\\]+', '', jaString) - endString = re.search(r'[^一-龠ぁ-ゔァ-ヴー【】。！？]+$', jaString) - jaString = re.sub(r'[^一-龠ぁ-ゔァ-ヴー【】。！？]+$', '', jaString) - if startString is None: startString = '' - else: startString = startString.group() - if endString is None: endString = '' - else: endString = endString.group() - - response = translateGPT(jaString, 'Reply with only the english translation of the npc nickname.', False) + # If there isn't any Japanese in the text just skip + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): + continue + + response = translateGPT(jaString, 'Reply with the English translation of the NPC name.', True) translatedText = response[0] + tokens += response[1] # Remove characters that may break scripts - charList = ['\"'] + charList = ['.', '\"', '\'', '\\n'] for char in charList: translatedText = translatedText.replace(char, '') - translatedText = translatedText.strip('.') - # Set Data - tokens += response[1] - page['list'][i]['p'][1] = startString + translatedText + endString + codeList[i]['p'][1] = translatedText except IndexError as e: - # This is part of the logic so we just pass it. - pass + # This is part of the logic so we just pass it + traceback.print_exc() + # raise Exception(str(e) + '|Line:' + tracebackLineNo) except Exception as e: - tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - raise Exception(str(e) + '|Line:' + tracebackLineNo + '| Failed to translate: ' + oldjaString) + traceback.print_exc() + raise Exception(str(e) + 'Failed to translate: ' + oldjaString) # Append leftover groups in 401 if len(currentGroup) > 0: @@ -942,22 +1121,18 @@ def searchCodes(page, pbar): response = translateGPT(finalJAString, 'Previous Translated Text for Context: ' + '\n\n'.join(textHistory), True) tokens += response[1] translatedText = response[0] - textHistory.append(translatedText) - # if speakerCaught == True: - # translatedText = speakerRaw + ':\n' + translatedText - # speakerCaught = False + # TextHistory is what we use to give GPT Context, so thats appended here. + textHistory.append('\"' + translatedText + '\"') # Textwrap translatedText = textwrap.fill(translatedText, width=WIDTH) - # Resub start and end - translatedText = startString + translatedText - # Set Data translatedText = translatedText.replace('ッ', '') translatedText = translatedText.replace('っ', '') - page['list'][i]['p'][0] = translatedText + translatedText = translatedText.replace('\"', '') + codeList[i]['p'][0] = translatedText speaker = '' match = [] @@ -971,41 +1146,71 @@ def searchCodes(page, pbar): def searchSS(state, pbar): '''Searches skills and states json files''' tokens = 0 - responseList = [0] * 7 - responseList[0] = (translateGPT(state['message1'], 'reply with only the english translation of the text.', False)) - responseList[1] = (translateGPT(state['message2'], 'reply with only the english translation of the text.', False)) - responseList[2] = (translateGPT(state.get('message3', ''), 'reply with only the english translation of the text.', False)) - responseList[3] = (translateGPT(state.get('message4', ''), 'reply with only the english translation of the text.', False)) - responseList[4] = (translateGPT(state['name'], 'Reply with only the english translation of the RPG item name.', False)) - if 'description' in state: - responseList[6] = (translateGPT(state['description'], 'reply with only the english translation of the description.', False)) + # Name + nameResponse = translateGPT(state['name'], 'Reply with only the english translation of the RPG Skill name.', True) if 'name' in state else '' + + # Description + descriptionResponse = translateGPT(state['description'], 'Reply with only the english translation of the description.', True) if 'description' in state else '' + + # Messages + message1Response = '' + message4Response = '' + message2Response = '' + message3Response = '' + + if 'message1' in state: + if len(state['message1']) > 0 and state['message1'][0] in ['は', 'を', 'の']: + message1Response = translateGPT('Taro' + state['message1'], 'reply with only the gender neutral english translation of the action. Always start the sentence with Taro.', True) + else: + message1Response = translateGPT(state['message1'], 'reply with only the gender neutral english translation', True) + + if 'message2' in state: + if len(state['message2']) > 0 and state['message2'][0] in ['は', 'を', 'の']: + message2Response = translateGPT('Taro' + state['message2'], 'reply with only the gender neutral english translation of the action. Always start the sentence with Taro.', True) + else: + message2Response = translateGPT(state['message2'], 'reply with only the gender neutral english translation', True) + + if 'message3' in state: + if len(state['message3']) > 0 and state['message3'][0] in ['は', 'を', 'の']: + message3Response = translateGPT('Taro' + state['message3'], 'reply with only the gender neutral english translation of the action. Always start the sentence with Taro.', True) + else: + message3Response = translateGPT(state['message3'], 'reply with only the gender neutral english translation', True) + + if 'message4' in state: + if len(state['message4']) > 0 and state['message4'][0] in ['は', 'を', 'の']: + message4Response = translateGPT('Taro' + state['message4'], 'reply with only the gender neutral english translation of the action. Always start the sentence with Taro.', True) + else: + message4Response = translateGPT(state['message4'], 'reply with only the gender neutral english translation', True) # if 'note' in state: - # if 'raceDesc' in state['note']: - # tokens += translateNote(state, r']*)>') - - # Put all our translations in a list - for i in range(len(responseList)): - if responseList[i] != 0: - tokens += responseList[i][1] - responseList[i] = responseList[i][0].strip('.\"') + if 'help' in state['note']: + tokens += translateNote(state, r']*)>') - # Set Data - if responseList[0] != '': - if responseList[0][0] != ' ': - state['message1'] = ' ' + responseList[0][0].lower() + responseList[0][1:] - state['message2'] = responseList[1] - if responseList[2] != '': - state['message3'] = responseList[2] - if responseList[3] != '': - state['message4'] = responseList[3] - state['name'] = responseList[4].strip('.') - # state['note'] = responseList[5] - if responseList[6] != 0: - responseList[6] = textwrap.fill(responseList[6], LISTWIDTH) - state['description'] = responseList[6].strip('\"') + # Count Tokens + tokens += nameResponse[1] if nameResponse != '' else 0 + tokens += descriptionResponse[1] if descriptionResponse != '' else 0 + tokens += message1Response[1] if message1Response != '' else 0 + tokens += message2Response[1] if message2Response != '' else 0 + tokens += message3Response[1] if message3Response != '' else 0 + tokens += message4Response[1] if message4Response != '' else 0 + # Set Data + if 'name' in state: + state['name'] = nameResponse[0].strip('\"') + if 'description' in state: + # Textwrap + translatedText = descriptionResponse[0] + translatedText = textwrap.fill(translatedText, width=LISTWIDTH) + state['description'] = translatedText.strip('\"') + if 'message1' in state: + state['message1'] = message1Response[0].strip('\"').replace('Taro', '') + if 'message2' in state: + state['message2'] = message2Response[0].strip('\"').replace('Taro', '') + if 'message3' in state: + state['message3'] = message3Response[0].strip('\"').replace('Taro', '') + if 'message4' in state: + state['message4'] = message4Response[0].strip('\"').replace('Taro', '') pbar.update(1) return tokens @@ -1038,45 +1243,76 @@ def searchSystem(data, pbar): data['armor_types'][i] = response[0].strip('.\"') pbar.update(1) - # Weapon Types - for i in range(len(data['weapon_types'])): - response = translateGPT(data['weapon_types'][i], 'Reply with only the english translation of the weapon type', False) - tokens += response[1] - data['weapon_types'][i] = response[0].strip('.\"') - pbar.update(1) - # Skill Types for i in range(len(data['skill_types'])): - response = translateGPT(data['skill_types'][i], 'Reply with only the english translation of the skill type', False) + response = translateGPT(data['skill_types'][i], 'Reply with only the english translation', False) tokens += response[1] data['skill_types'][i] = response[0].strip('.\"') pbar.update(1) + + # Equip Types + for i in range(len(data['equip_types'])): + response = translateGPT(data['equip_types'][i], 'Reply with only the english translation of the equipment type. No disclaimers.', False) + tokens += response[1] + data['equip_types'][i] = response[0].strip('.\"') + pbar.update(1) + + # Variables (Optional ususally) + for i in range(len(data['variables'])): + response = translateGPT(data['variables'][i], 'Reply with only the english translation of the title', False) + tokens += response[1] + data['variables'][i] = response[0].strip('.\"') + pbar.update(1) + + # Messages + messages = (data['terms']['messages']) + for key, value in messages.items(): + response = translateGPT(value, 'Reply with only the english translation of the battle text.', False) + translatedText = response[0] + + # Remove characters that may break scripts + charList = ['.', '\"', '\\n'] + for char in charList: + translatedText = translatedText.replace(char, '') + + tokens += response[1] + messages[key] = translatedText + pbar.update(1) return tokens def subVars(jaString): - varRegex = r'\\+[a-zA-Z]+\[[0-9a-zA-Z\\\[\]]+\]+|[\\]+[#|]+|\\+[\.<>a-zA-Z]+' + jaString = jaString.replace('\u3000', ' ') + varRegex = r'[\\]+[\w.\\\s]+?\[.+?\]]?|[\\]+[\w.\\]+?\<.+?\>\>?|[\\]+[#{}<>.]' count = 0 varList = re.findall(varRegex, jaString) + varList = set(varList) if len(varList) != 0: for var in varList: - jaString = jaString.replace(var, '[v' + str(count) + ']') + jaString = jaString.replace(var, '@' + str(count) + '') count += 1 return [jaString, varList] def resubVars(translatedText, varList): count = 0 + + # Fix Spacing and ChatGPT Nonsense + matchList = re.findall(r'@\s?[0-9]+?', translatedText) + if len(matchList) > 0: + for match in matchList: + text = match.replace(' ', '') + translatedText = translatedText.replace(match, text) if len(varList) != 0: for var in varList: - translatedText = translatedText.replace('[v' + str(count) + ']', var) + translatedText = translatedText.replace('@' + str(count) + '', var) count += 1 # Remove Color Variables Spaces if '\\c' in translatedText: - translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r'\1', translatedText) + translatedText = re.sub(r'\s*(\\+c\[[1-9]+\])\s*', r' \1', translatedText) translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) return translatedText @@ -1086,7 +1322,7 @@ def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") + enc = tiktoken.encoding_for_model("gpt-3.5-turbo") TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, 0) @@ -1099,15 +1335,17 @@ def translateGPT(t, history, fullPromptFlag): return(t, 0) """Translate text using GPT""" - context = 'Character Context: アリス == Alice | Female, ヒルダ == Hilda | Female, J = J | Male' + context = 'Eroge Names Context: ボク == Boku | Male' if fullPromptFlag: system = PROMPT - user = 'Text to Translate: ' + subbedT + user = 'Line to Translate: ' + subbedT else: system = 'You are an expert translator who translates everything to English. Reply with only the English Translation of the text.' - user = 'Text to Translate: ' + subbedT + user = 'Line to Translate: ' + subbedT response = openai.ChatCompletion.create( temperature=0, + frequency_penalty=0.2, + presence_penalty=0.2, model="gpt-3.5-turbo", messages=[ {"role": "system", "content": system}, @@ -1128,10 +1366,12 @@ def translateGPT(t, history, fullPromptFlag): # Remove Placeholder Text translatedText = translatedText.replace('English Translation: ', '') translatedText = translatedText.replace('Translation: ', '') - translatedText = translatedText.replace('Text to Translate: ', '') + translatedText = translatedText.replace('Line to Translate: ', '') translatedText = translatedText.replace('English Translation:', '') translatedText = translatedText.replace('Translation:', '') - translatedText = translatedText.replace('Text to Translate:', '') + translatedText = translatedText.replace('Line to Translate:', '') + translatedText = re.sub(r'\n\nPast Translated Text:.*', '', translatedText, 0, re.DOTALL) + translatedText = re.sub(r'Note:.*', '', translatedText) # Return Translation if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 776bacd..e263694 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -26,7 +26,7 @@ PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = 10 # For GPT4 rate limit will be hit if you have more than 1 thread. LOCK = threading.Lock() WIDTH = 40 -LISTWIDTH = 70 +LISTWIDTH = 60 MAXHISTORY = 10 ESTIMATE = '' TOTALCOST = 0 @@ -67,7 +67,8 @@ def handleMVMZ(filename, estimate): # Print Result end = time.time() tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) - tqdm.write(str(NAMESLIST)) + if NAMES == True: + tqdm.write(str(NAMESLIST)) with LOCK: TOTALCOST += TOKENS * .001 * APICOST TOTALTOKENS += TOKENS @@ -559,7 +560,7 @@ def searchCodes(page, pbar): codeList[j]['code'] = code # Put names in list - if NAMES == True and speaker not in NAMESLIST: + if speaker not in NAMESLIST: with LOCK: NAMESLIST.append(speaker) elif '\\kw' in finalJAString: @@ -821,7 +822,7 @@ def searchCodes(page, pbar): # Set Data speaker = translatedText codeList[i]['parameters'][4] = translatedText - if NAMES == True and speaker not in NAMESLIST: + if speaker not in NAMESLIST: with LOCK: NAMESLIST.append(speaker)