From 3d816d9fd37b1e00838d002e95896a4c09bc64e9 Mon Sep 17 00:00:00 2001 From: Dazed Date: Wed, 14 Jun 2023 18:39:05 -0500 Subject: [PATCH] feat: Rework ACE support from ground up --- .gitignore | 2 + src/main.py | 2 +- src/rpgmakerace.py | 532 +++++++++++++++++++++++++++++++------------- src/rpgmakermvmz.py | 27 ++- 4 files changed, 392 insertions(+), 171 deletions(-) diff --git a/.gitignore b/.gitignore index fea31c2..b93dd79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .env *.json +*.yaml +*.yml *.txt !requirements.txt *.csv diff --git a/src/main.py b/src/main.py index b50b3f3..0515d35 100644 --- a/src/main.py +++ b/src/main.py @@ -48,7 +48,7 @@ def main(): # Open File (Threads) with ThreadPoolExecutor(max_workers=THREADS) as executor: futures = [executor.submit(handleACE, filename, estimate) \ - for filename in os.listdir("files") if filename.endswith('json')] + for filename in os.listdir("files") if filename.endswith('yaml')] for future in as_completed(futures): try: diff --git a/src/rpgmakerace.py b/src/rpgmakerace.py index b694e00..e232bc6 100644 --- a/src/rpgmakerace.py +++ b/src/rpgmakerace.py @@ -1,5 +1,4 @@ from concurrent.futures import ThreadPoolExecutor, as_completed -import json import os from pathlib import Path import re @@ -15,6 +14,11 @@ from dotenv import load_dotenv import openai from retry import retry from tqdm import tqdm +from ruamel.yaml import YAML + +#Yaml +yaml = YAML() +yaml.preserve_quotes = True #Globals load_dotenv() @@ -25,7 +29,8 @@ APICOST = .002 # Depends on the model https://openai.com/pricing PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = 20 LOCK = threading.Lock() -WIDTH = 60 +WIDTH = 70 +LISTWIDTH = 75 MAXHISTORY = 10 ESTIMATE = '' TOTALCOST = 0 @@ -44,9 +49,12 @@ CODE122 = False CODE101 = False CODE355655 = False CODE357 = False +CODE356 = False +CODE320 = False +CODE111 = False def handleACE(filename, estimate): - global ESTIMATE, TOTALTOKENS, TOTALCOST, LOCK + global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST ESTIMATE = estimate if estimate: @@ -55,11 +63,13 @@ def handleACE(filename, estimate): # Print Result end = time.time() - tqdm.write(getResultString(translatedData, end - start, filename)) - + tqdm.write(getResultString(['', TOKENS, None], end - start, filename)) with LOCK: - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] + TOTALCOST += TOKENS * .001 * APICOST + TOTALTOKENS += TOKENS + TOKENS = 0 + + return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') else: with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: @@ -68,21 +78,20 @@ def handleACE(filename, estimate): # Print Result end = time.time() - json.dump(translatedData[0], outFile, ensure_ascii=False) + yaml.dump(translatedData[0], outFile) tqdm.write(getResultString(translatedData, end - start, filename)) - - with LOCK: - TOTALCOST += translatedData[1] * .001 * APICOST - TOTALTOKENS += translatedData[1] + with LOCK: + TOTALCOST += translatedData[1] * .001 * APICOST + TOTALTOKENS += translatedData[1] return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') def openFiles(filename): with open('files/' + filename, 'r', encoding='UTF-8') as f: - data = json.load(f) + data = yaml.load(f) # Map Files - if 'Map' in filename and filename != 'MapInfos.json': + if 'Map' in filename and filename != 'MapInfos.yaml': translatedData = parseMap(data, filename) # CommonEvents Files @@ -93,26 +102,38 @@ def openFiles(filename): elif 'Actors' in filename: translatedData = parseNames(data, filename, 'Actors') - # Armors File + # Armor File elif 'Armors' in filename: - translatedData = parseThings(data, filename, 'Armor') + translatedData = parseNames(data, filename, 'Armors') + + # Weapons File + elif 'Weapons' in filename: + translatedData = parseNames(data, filename, 'Weapons') # Classes File elif 'Classes' in filename: translatedData = parseNames(data, filename, 'Classes') + # Enemies File + elif 'Enemies' in filename: + translatedData = parseNames(data, filename, 'Enemies') + # Items File elif 'Items' in filename: - translatedData = parseThings(data, filename, 'Items') + translatedData = parseThings(data, filename) # MapInfo File elif 'MapInfos' in filename: - translatedData = parseMapInfos(data, filename) + translatedData = parseNames(data, filename, 'MapInfos') # Skills File elif 'Skills' in filename: translatedData = parseSS(data, filename) + # Troops File + elif 'Troops' in filename: + translatedData = parseTroops(data, filename) + # States File elif 'States' in filename: translatedData = parseSS(data, filename) @@ -148,48 +169,33 @@ def getResultString(translatedData, translationTime, filename): def parseMap(data, filename): totalTokens = 0 totalLines = 0 - events = data['@events'] + events = data['events'] global LOCK + # Translate displayName for Map files + if 'Map' in filename: + response = translateGPT(data['displayName'], 'Reply with only the english translated name', False) + totalTokens += response[1] + data['displayName'] = response[0].strip('.\"') + # Get total for progress bar - for event in events.items(): + for event in events: if event is not None: - for item in event: - if type(item) is dict: - for page in item['@pages']: - totalLines += len(page['@list']) + for page in event['pages']: + totalLines += len(page['list']) with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: pbar.desc=filename pbar.total=totalLines with ThreadPoolExecutor(max_workers=THREADS) as executor: - for event in events.items(): + for event in events: if event is not None: - for item in event: - if type(item) is dict: - futures = [executor.submit(searchCodes, page, pbar) for page in item['@pages'] if page is not None] - for future in as_completed(futures): - try: - totalTokens += future.result() - except Exception as e: - return [data, totalTokens, e] - return [data, totalTokens, None] - -def parseMapInfos(data, filename): - totalTokens = 0 - totalLines = 0 - totalLines += len(data) - - with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: - pbar.desc=filename - pbar.total=totalLines - for name in data.items(): - if name is not None: - try: - result = searchMapInfos(name, pbar) - totalTokens += result - except Exception as e: - return [data, totalTokens, e] + futures = [executor.submit(searchCodes, page, pbar) for page in event['pages'] if page is not None] + for future in as_completed(futures): + try: + totalTokens += future.result() + except Exception as e: + return [data, totalTokens, e] return [data, totalTokens, None] def parseCommonEvents(data, filename): @@ -200,7 +206,7 @@ def parseCommonEvents(data, filename): # Get total for progress bar for page in data: if page is not None: - totalLines += len(page['@list']) + totalLines += len(page['list']) with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: pbar.desc=filename @@ -213,6 +219,31 @@ def parseCommonEvents(data, filename): except Exception as e: return [data, totalTokens, e] return [data, totalTokens, None] + +def parseTroops(data, filename): + totalTokens = 0 + totalLines = 0 + global LOCK + + # Get total for progress bar + for troop in data: + if troop is not None: + for page in troop['pages']: + totalLines += len(page['list']) + 1 # The +1 is because each page has a name. + + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc=filename + pbar.total=totalLines + for troop in data: + if troop is not None: + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(searchCodes, page, pbar) for page in troop['pages'] if page is not None] + for future in as_completed(futures): + try: + totalTokens += future.result() + except Exception as e: + return [data, totalTokens, e] + return [data, totalTokens, None] def parseNames(data, filename, context): totalTokens = 0 @@ -231,7 +262,7 @@ def parseNames(data, filename, context): return [data, totalTokens, e] return [data, totalTokens, None] -def parseThings(data, filename, context): +def parseThings(data, filename): totalTokens = 0 totalLines = 0 totalLines += len(data) @@ -270,11 +301,13 @@ def parseSystem(data, filename): totalLines = 0 # Calculate Total Lines - for term in data['@terms']: - termList = data['@terms'][term] + for term in data['terms']: + termList = data['terms'][term] totalLines += len(termList) - totalLines += len(data['@game_title']) - totalLines += len(data['@terms']['@params']) + totalLines += len(data['game_title']) + totalLines += len(data['armor_types']) + totalLines += len(data['skill_types']) + totalLines += len(data['weapon_types']) with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: pbar.desc=filename @@ -291,9 +324,9 @@ def searchThings(name, pbar): # Set the context of what we are translating responseList = [] - responseList.append(translateGPT(name['@name'], 'Reply with only the menu item name.', False)) - responseList.append(translateGPT(name['@description'], 'Reply with only the description.', False)) - responseList.append(translateGPT(name['@note'], 'Reply with only the note.', False)) + responseList.append(translateGPT(name['name'], 'Reply with only the english translated menu item name.', False)) + responseList.append(translateGPT(name['description'], 'Reply with only the english translated description.', True)) + # responseList.append(translateGPT(name['note'], 'Reply with only the english translated note.', False)) # Extract all our translations in a list from response for i in range(len(responseList)): @@ -301,18 +334,10 @@ def searchThings(name, pbar): responseList[i] = responseList[i][0] # Set Data - name['@name'] = responseList[0].strip('.') - name['@description'] = responseList[1] - name['@note'] = responseList[2] - pbar.update(1) - - return tokens - -def searchMapInfos(name, pbar): - response = translateGPT(name[1]['@name'], 'Reply with only the map name.', False) - tokens = response[1] - translatedText = response[0] - name[1]['@name'] = translatedText + name['name'] = responseList[0].strip('.\"') + responseList[1] = textwrap.fill(responseList[1], LISTWIDTH) + name['description'] = responseList[1].strip('\"') + # name['note'] = responseList[2] pbar.update(1) return tokens @@ -322,16 +347,26 @@ def searchNames(name, pbar, context): # Set the context of what we are translating if 'Actors' in context: - newContext = 'Reply with only the actor name' + newContext = 'Reply with only the english translation. The original text is a menu item.' + if 'Armors' in context: + newContext = 'Reply with only the english translation.' if 'Classes' in context: - newContext = 'Reply with only the class name' + newContext = 'Reply with only the english translated class name' + if 'MapInfos' in context: + newContext = 'Reply with only the english translated map name' + if 'Enemies' in context: + newContext = 'Reply with only the english translated enemy' + if 'Weapons' in context: + newContext = 'Reply with only the english translated weapon name' + # Extract Data responseList = [] - responseList.append(translateGPT(name['@name'], newContext, False)) + responseList.append(translateGPT(name['name'], newContext, True)) + if 'Actors' in context: + responseList.append(translateGPT(name['profile'], '', True)) - if 'MapInfos' not in context: - responseList.append(translateGPT(name['@description'], newContext, False)) - responseList.append(translateGPT(name['@note'], newContext, False)) + if 'Armors' in context or 'Weapons' in context: + responseList.append(translateGPT(name['description'], '', True)) # Extract all our translations in a list from response for i in range(len(responseList)): @@ -339,10 +374,14 @@ def searchNames(name, pbar, context): responseList[i] = responseList[i][0] # Set Data - name['@name'] = responseList[0].strip('.') - if 'MapInfos' not in context: - name['@description'] = responseList[1] - name['@note'] = responseList[2] + name['name'] = responseList[0].strip('.\"') + if 'Actors' in context: + translatedText = textwrap.fill(responseList[1], LISTWIDTH) + name['profile'] = translatedText.strip('\"') + + if 'Armors' in context or 'Weapons' in context: + translatedText = textwrap.fill(responseList[1], LISTWIDTH) + name['description'] = translatedText.strip('\"') pbar.update(1) return tokens @@ -354,6 +393,8 @@ def searchCodes(page, pbar): maxHistory = MAXHISTORY tokens = 0 speaker = '' + match = [] + speakerCaught = False global LOCK # Regex @@ -361,7 +402,7 @@ def searchCodes(page, pbar): reSubVarRegex = r'\<([\\a-zA-Z]+)([a-zA-Z0-9一-龠ぁ-ゔァ-ヴー\s]+)\>' try: - for i in range(len(page['@list'])): + for i in range(len(page['list'])): with LOCK: pbar.update(1) @@ -369,8 +410,8 @@ def searchCodes(page, pbar): ### IF these crash or fail your game will do the same. Use the flags to skip codes. ## Event Code: 401 Show Text - if page['@list'][i]['@code'] == 401 and CODE401 == True: - jaString = page['@list'][i]['@parameters'][0] + if page['list'][i]['code'] == 401 and CODE401 == True: + jaString = page['list'][i]['parameters'][0] oldjaString = jaString jaString = jaString.replace('ﾞ', '') jaString = jaString.replace('。', '.') @@ -379,9 +420,9 @@ def searchCodes(page, pbar): # Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior) currentGroup.append(jaString) - while (page['@list'][i+1]['@code'] == 401): - del page['@list'][i] - jaString = page['@list'][i]['@parameters'][0] + while (page['list'][i+1]['code'] == 401): + del page['list'][i] + jaString = page['list'][i]['parameters'][0] jaString = jaString.replace('ﾞ', '') jaString = jaString.replace('。', '.') jaString = re.sub(r'([\u3000-\uffef])\1{1,}', r'\1', jaString) @@ -432,6 +473,12 @@ def searchCodes(page, pbar): else: textHistory.append('\"' + translatedText + '\"') + # Name Handling + if len(match) != 0: + name = '\\nw[' + speaker + ']' + if name not in translatedText: + translatedText = translatedText + '\\nw[' + speaker + ']' + # if speakerCaught == True: # translatedText = speakerRaw + ':\n' + translatedText # speakerCaught = False @@ -443,18 +490,18 @@ def searchCodes(page, pbar): translatedText = startString + translatedText # Set Data - page['@list'][i]['@parameters'][0] = translatedText.replace('\"', '') + page['list'][i]['parameters'][0] = translatedText.replace('\"', '') speaker = '' match = [] # Keep textHistory list at length maxHistory if len(textHistory) > maxHistory: textHistory.pop(0) - currentGroup = [] + currentGroup = [] ## Event Code: 122 [Control Variables] [Optional] - if page['@list'][i]['@code'] == 122 and CODE122 == True: - jaString = page['@list'][i]['@parameters'][4] + if page['list'][i]['code'] == 122 and CODE122 == True: + jaString = page['list'][i]['parameters'][4] if type(jaString) != str: continue @@ -462,6 +509,10 @@ def searchCodes(page, pbar): if '_' in jaString: continue + # If there isn't any Japanese in the text just skip + if re.search(r'[a-zA-Z0-9]+', jaString): + continue + # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): continue @@ -469,11 +520,21 @@ def searchCodes(page, pbar): # Remove repeating characters because it confuses ChatGPT jaString = re.sub(r'([\u3000-\uffef])\1{2,}', r'\1\1', jaString) + # Need to remove outside code and put it back later + startString = re.search(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', jaString) + jaString = re.sub(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', '', jaString) + endString = re.search(r'[^ぁ-んァ-ン一-龯\<\>【】 。！？]+$', jaString) + jaString = re.sub(r'[^ぁ-んァ-ン一-龯\<\>【】 。！？]+$', '', jaString) + if startString is None: startString = '' + else: startString = startString.group() + if endString is None: endString = '' + else: endString = endString.group() + # Sub Vars - jaString = re.sub(r'\\+([a-zA-Z]+)\[([0-9]+)\]', r'[\1\2]', jaString) + jaString = re.sub(subVarRegex, r'<\1\2>', jaString) # Translate - response = translateGPT(jaString, '', False) + response = translateGPT(jaString, 'Reply with only the english translation', False) tokens += response[1] translatedText = response[0] @@ -483,15 +544,15 @@ def searchCodes(page, pbar): translatedText = translatedText.replace(char, '') # ReSub Vars - translatedText = re.sub(r'\[([a-zA-Z]+)([0-9]+)]', r'\\\\\1[\2]', translatedText) + translatedText = re.sub(reSubVarRegex, r'\1[\2]', translatedText) # Set Data - page['@list'][i]['@parameters'][4] = '\"' + translatedText + '\"' + page['list'][i]['parameters'][4] = startString + translatedText + endString ## Event Code: 357 [Picture Text] [Optional] - if page['@list'][i]['@code'] == 357 and CODE357 == True: - if '@text' in page['@list'][i]['@parameters'][3]: - jaString = page['@list'][i]['@parameters'][3]['@text'] + if page['list'][i]['code'] == 357 and CODE357 == True: + if 'text' in page['list'][i]['parameters'][3]: + jaString = page['list'][i]['parameters'][3]['text'] if type(jaString) != str: continue @@ -513,7 +574,7 @@ def searchCodes(page, pbar): jaString = re.sub(r'\\+([a-zA-Z]+)\[([0-9]+)\]', r'[\1\2]', jaString) # Translate - response = translateGPT(jaString, '', False) + response = translateGPT(jaString, '', True) tokens += response[1] translatedText = response[0] @@ -529,11 +590,11 @@ def searchCodes(page, pbar): translatedText = re.sub(r'\[([a-zA-Z]+)([0-9]+)]', r'\\\\\1[\2]', translatedText) # Set Data - page['@list'][i]['@parameters'][3]['@text'] = startString + translatedText + page['list'][i]['parameters'][3]['text'] = startString + translatedText ## Event Code: 101 [Name] [Optional] - if page['@list'][i]['@code'] == 101 and CODE101 == True: - jaString = page['@list'][i]['@parameters'][4] + if page['list'][i]['code'] == 101 and CODE101 == True: + jaString = page['list'][i]['parameters'][4] if type(jaString) != str: continue @@ -547,7 +608,7 @@ def searchCodes(page, pbar): continue # Translate - response = translateGPT(jaString, 'Reply with only the english translated name', False) + response = translateGPT(jaString, 'Reply with only the english translation. NEVER reply in anything other than English. I repeat, only reply with the english translation of the original text.', False) tokens += response[1] translatedText = response[0] @@ -558,36 +619,36 @@ def searchCodes(page, pbar): # Set Data speaker = translatedText - page['@list'][i]['@parameters'][4] = translatedText + page['list'][i]['parameters'][4] = translatedText ## Event Code: 355 or 655 Scripts [Optional] - if (page['@list'][i]['@code'] == 355 or page['@list'][i]['@code'] == 655) and CODE355655 == True: - jaString = page['@list'][i]['@parameters'][0] + if (page['list'][i]['code'] == 355 or page['list'][i]['code'] == 655) and CODE355655 == True: + jaString = page['list'][i]['parameters'][0] # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): continue # Want to translate this script - if page['@list'][i]['@code'] == 355 and 'this.BLogAdd' not in jaString: + if page['list'][i]['code'] == 355 and '.setName' not in jaString: continue # Don't want to touch certain scripts - if page['@list'][i]['@code'] == 655 and 'this.' in jaString: + if page['list'][i]['code'] == 655 and 'this.' in jaString: continue # Need to remove outside code and put it back later startString = re.search(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', jaString) jaString = re.sub(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', '', jaString) - endString = re.search(r'[^ぁ-んァ-ン一-龯\<\>【】]+$', jaString) - jaString = re.sub(r'[^ぁ-んァ-ン一-龯\<\>【】]+$', '', jaString) + endString = re.search(r'[^ぁ-んァ-ン一-龯\<\>【】 。！？]+$', jaString) + jaString = re.sub(r'[^ぁ-んァ-ン一-龯\<\>【】 。！？]+$', '', jaString) if startString is None: startString = '' else: startString = startString.group() if endString is None: endString = '' else: endString = endString.group() # Translate - response = translateGPT(jaString, '', False) + response = translateGPT(jaString, 'Reply with only the english translation.', True) tokens += response[1] translatedText = response[0] @@ -597,24 +658,63 @@ def searchCodes(page, pbar): translatedText = translatedText.replace(char, '') # Set Data - page['@list'][i]['@parameters'][0] = startString + translatedText + endString + page['list'][i]['parameters'][0] = startString + translatedText + endString + + ## Event Code: 356 D_TEXT + if page['list'][i]['code'] == 356 and CODE356 == True: + jaString = page['list'][i]['parameters'][0] + + # If there isn't any Japanese in the text just skip + if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): + continue + + # Want to translate this script + if 'PSM_SHOW_POPUP' not in jaString: + continue + + # Need to remove outside code and put it back later + startString = re.search(r'^[^ぁ-んァ-ン一-龯【】（）「」]+-1 ', jaString) + jaString = re.sub(r'^[^ぁ-んァ-ン一-龯【】（）「」]+-1 ', '', jaString) + endString = re.search(r' [^ぁ-んァ-ン一-龯\<\>【】 。！？]+$', jaString) + jaString = re.sub(r' [^ぁ-んァ-ン一-龯\<\>【】 。！？]+$', '', jaString) + if startString is None: startString = '' + else: startString = startString.group() + if endString is None: endString = '' + else: endString = endString.group() + + # Translate + response = translateGPT(jaString, 'Reply with only the English Translation.', True) + tokens += response[1] + translatedText = response[0] + + # Remove characters that may break scripts + charList = ['.', '\"', '\\n'] + for char in charList: + translatedText = translatedText.replace(char, '') + + # Cant have spaces? + translatedText = translatedText.replace(' ', '　') + + # Set Data + page['list'][i]['parameters'][0] = startString + translatedText + endString ### Event Code: 102 Show Choice - if page['@list'][i]['@code'] == 102 and CODE102 == True: - for choice in range(len(page['@list'][i]['@parameters'][0])): - choiceText = page['@list'][i]['@parameters'][0][choice] + if page['list'][i]['code'] == 102 and CODE102 == True: + for choice in range(len(page['list'][i]['parameters'][0])): + jaString = page['list'][i]['parameters'][0][choice] translatedText = translatedText.replace(' 。', '.') - # Need to remove outside non-japanese text and put it back later - startString = re.search(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', choiceText) - choiceText = re.sub(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', '', choiceText) + # Need to remove outside code and put it back later + startString = re.search(r'^[^ぁ-んァ-ン一-龯\<\>【】（）Ａ-Ｚ０-９]+', jaString) + jaString = re.sub(r'^[^ぁ-んァ-ン一-龯\<\>【】（）Ａ-Ｚ０-９]+', '', jaString) + endString = re.search(r'[^ぁ-んァ-ン一-龯【】 。！？（）Ａ-Ｚ０-９]+$', jaString) + jaString = re.sub(r'[^ぁ-んァ-ン一-龯【】 。！？（）Ａ-Ｚ０-９]+$', '', jaString) if startString is None: startString = '' else: startString = startString.group() + if endString is None: endString = '' + else: endString = endString.group() - if len(textHistory) > 0: - response = translateGPT(choiceText, 'Reply with the english translation for the answer. QUESTION: ' + textHistory[-1], True) - else: - response = translateGPT(choiceText, 'Reply with the english translation for the answer. QUESTION: ' + '', True) + response = translateGPT(jaString, 'Keep your reply prompt.', True) translatedText = response[0] # Remove characters that may break scripts @@ -624,92 +724,203 @@ def searchCodes(page, pbar): # Set Data tokens += response[1] - page['@list'][i]['@parameters'][0][choice] = startString + translatedText + page['list'][i]['parameters'][0][choice] = startString + translatedText + endString + + ### Event Code: 111 Script + if page['list'][i]['code'] == 111 and CODE111 == True: + for j in range(len(page['list'][i]['parameters'])): + jaString = page['list'][i]['parameters'][j] + + # Check if String + if type(jaString) != str: + continue + + # Need to remove outside code and put it back later + startString = re.search(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', jaString) + jaString = re.sub(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', '', jaString) + endString = re.search(r'[^ぁ-んァ-ン一-龯【】 。！？]+$', jaString) + jaString = re.sub(r'[^ぁ-んァ-ン一-龯【】 。！？]+$', '', jaString) + if startString is None: startString = '' + else: startString = startString.group() + if endString is None: endString = '' + else: endString = endString.group() + + response = translateGPT(jaString, 'Reply with only the english translation.', True) + translatedText = response[0] + + # Remove characters that may break scripts + charList = ['.', '\"', '\\n'] + for char in charList: + translatedText = translatedText.replace(char, '') + + # Set Data + tokens += response[1] + page['list'][i]['parameters'][j] = startString + translatedText + endString + + ### Event Code: 320 Set Variable + if page['list'][i]['code'] == 320 and CODE320 == True: + jaString = page['list'][i]['parameters'][1] + translatedText = translatedText.replace(' 。', '.') + + # Need to remove outside code and put it back later + startString = re.search(r'^[^ぁ-んァ-ン一-龯【】a-zA-Z\\]+', jaString) + jaString = re.sub(r'^[^ぁ-んァ-ン一-龯【】a-zA-Z\\]+', '', jaString) + endString = re.search(r'[^ぁ-んァ-ン一-龯【】 。！？]+$', jaString) + jaString = re.sub(r'[^ぁ-んァ-ン一-龯【】 。！？]+$', '', jaString) + if startString is None: startString = '' + else: startString = startString.group() + if endString is None: endString = '' + else: endString = endString.group() + + response = translateGPT(jaString, 'Reply with only the english translation.', True) + translatedText = response[0] + + # Remove characters that may break scripts + charList = ['.', '\"', '\\n'] + for char in charList: + translatedText = translatedText.replace(char, '') + + # Set Data + tokens += response[1] + page['list'][i]['parameters'][1] = startString + translatedText + endString except IndexError: # This is part of the logic so we just pass it. pass except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - raise Exception(str(e) + '|Line:' + tracebackLineNo + '| Failed to translate: ' + jaString) + raise Exception(str(e) + '|Line:' + tracebackLineNo + '| Failed to translate: ' + oldjaString) # Append leftover groups in 401 if len(currentGroup) > 0: - response = translateGPT(''.join(currentGroup), ' '.join(textHistory), True) + # Translate + if speaker != '': + response = translateGPT(finalJAString, 'Previous text for context: ' + ' '.join(textHistory) \ + + '\n\n\n###\n\n\nCurrent Speaker: ' + speaker, True) + else: + response = translateGPT(finalJAString, 'Previous text for context: ' + ' '.join(textHistory), True) tokens += response[1] translatedText = response[0] - #Cleanup + # ReSub Vars + translatedText = re.sub(reSubVarRegex, r'\1[\2]', translatedText) + # TextHistory is what we use to give GPT Context, so thats appended here. - textHistory.append(translatedText) + rawTranslatedText = re.sub(r'[\\<>]+[a-zA-Z]+\[[a-zA-Z0-9]+\]', '', translatedText) + if speaker != '': + textHistory.append(speaker + ': ' + rawTranslatedText) + else: + textHistory.append('\"' + rawTranslatedText + '\"') + + # Name Handling + if len(match) != 0: + name = '\\nw[' + speaker + ']' + if name not in translatedText: + translatedText = translatedText + '\\nw[' + speaker + ']' + + # if speakerCaught == True: + # translatedText = speakerRaw + ':\n' + translatedText + # speakerCaught = False # Textwrap - if page['@list'][i]['@code'] == 401: - translatedText = textwrap.fill(translatedText, width=WIDTH) + translatedText = textwrap.fill(translatedText, width=WIDTH) + + # Resub start and end + translatedText = startString + translatedText # Set Data - page['@list'][i]['@parameters'][0] = translatedText + page['list'][i]['parameters'][0] = translatedText.replace('\"', '') + speaker = '' + match = [] # Keep textHistory list at length maxHistory if len(textHistory) > maxHistory: textHistory.pop(0) - currentGroup = [] - page['@list'][i]['@parameters'][0] = translatedText - currentGroup = [] + currentGroup = [] return tokens def searchSS(state, pbar): - '''Searches skills and states json files''' + '''Searches skills and states yaml files''' tokens = 0 - responseList = [0] * 6 + responseList = [0] * 7 - responseList[0] = (translateGPT(state['@message1'], 'Reply with only the message.', False)) - responseList[1] = (translateGPT(state['@message2'], 'Reply with only the message.', False)) - responseList[2] = (translateGPT(state.get('@message3', ''), 'Reply with only the message.', False)) - responseList[3] = (translateGPT(state.get('@message4', ''), 'Reply with only the message.', False)) - responseList[4] = (translateGPT(state['@name'], 'Reply with only the state name.', False)) - responseList[5] = (translateGPT(state['@note'], 'Reply with only the note.', False)) + responseList[0] = (translateGPT(state['message1'], 'Reply with the english translated Action being performed and no subject.', False)) + responseList[1] = (translateGPT(state['message2'], 'Reply with the english translated Action being performed and no subject.', False)) + responseList[2] = (translateGPT(state.get('message3', ''), 'Reply with the english translated Action being performed and no subject..', False)) + responseList[3] = (translateGPT(state.get('message4', ''), 'Reply with the english translated Action being performed and no subject..', False)) + responseList[4] = (translateGPT(state['name'], 'Reply with only the english translation', True)) + # responseList[5] = (translateGPT(state['note'], 'Reply with only the translated english note.', False)) + if 'description' in state: + responseList[6] = (translateGPT(state['description'], 'Reply with the english translated description.', True)) # Put all our translations in a list for i in range(len(responseList)): - tokens += responseList[i][1] - responseList[i] = responseList[i][0] + if responseList[i] != 0: + tokens += responseList[i][1] + responseList[i] = responseList[i][0].strip('.\"') # Set Data - state['@message1'] = responseList[0] - state['@message2'] = responseList[1] + if responseList[0] != '': + if responseList[0][0] != ' ': + state['message1'] = ' ' + responseList[0][0].lower() + responseList[0][1:] + state['message2'] = responseList[1] if responseList[2] != '': - state['@message3'] = responseList[2] + state['message3'] = responseList[2] if responseList[3] != '': - state['@message4'] = responseList[3] - state['@name'] = responseList[4].strip('.') - state['@note'] = responseList[5] + state['message4'] = responseList[3] + state['name'] = responseList[4].strip('.') + # state['note'] = responseList[5] + if responseList[6] != 0: + responseList[6] = textwrap.fill(responseList[6], LISTWIDTH) + state['description'] = responseList[6].strip('\"') + pbar.update(1) return tokens def searchSystem(data, pbar): tokens = 0 - context = 'Reply with only the menu item.' + context = 'Reply with only the english translated menu item.' # Title - response = translateGPT(data['@game_title'], context, False) + response = translateGPT(data['game_title'], context, True) tokens += response[1] - data['@game_title'] = response[0].strip('.') + data['game_title'] = response[0].strip('.') pbar.update(1) # Terms - for term in data['@terms']: - if term != '@messages': - termList = data['@terms'][term] + for term in data['terms']: + if term != 'messages': + termList = data['terms'][term] for i in range(len(termList)): # Last item is a messages object - if termList[i] is not None and 'json' not in term: - response = translateGPT(termList[i], context, False) + if termList[i] is not None: + response = translateGPT(termList[i], context, True) tokens += response[1] termList[i] = response[0].strip('.\"') pbar.update(1) - + + # Armor Types + for i in range(len(data['armor_types'])): + response = translateGPT(data['armor_types'][i], 'Reply with only the english translated armor type', False) + tokens += response[1] + data['armor_types'][i] = response[0].strip('.\"') + pbar.update(1) + + # Skill Types + for i in range(len(data['skill_types'])): + response = translateGPT(data['skill_types'][i], 'Reply with only the english translation', False) + tokens += response[1] + data['skill_types'][i] = response[0].strip('.\"') + pbar.update(1) + + # Weapon Types + for i in range(len(data['weapon_types'])): + response = translateGPT(data['weapon_types'][i], 'Reply with only the english translated equipment type. No disclaimers.', False) + tokens += response[1] + data['weapon_types'][i] = response[0].strip('.\"') + pbar.update(1) + return tokens @retry(exceptions=Exception, tries=5, delay=5) @@ -718,7 +929,7 @@ def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, 0) @@ -748,4 +959,5 @@ editor, and localizer. ' + history if len(response.choices[0].message.content) > 9 * len(t): return [t, response.usage.total_tokens] else: - return [response.choices[0].message.content, response.usage.total_tokens] \ No newline at end of file + return [response.choices[0].message.content, response.usage.total_tokens] + \ No newline at end of file diff --git a/src/rpgmakermvmz.py b/src/rpgmakermvmz.py index 1f0f3fc..fb2ca78 100644 --- a/src/rpgmakermvmz.py +++ b/src/rpgmakermvmz.py @@ -25,7 +25,7 @@ APICOST = .002 # Depends on the model https://openai.com/pricing PROMPT = Path('prompt.txt').read_text(encoding='utf-8') THREADS = 20 LOCK = threading.Lock() -WIDTH = 50 +WIDTH = 70 LISTWIDTH = 75 MAXHISTORY = 10 ESTIMATE = '' @@ -355,9 +355,12 @@ def searchNames(name, pbar, context): # Extract Data responseList = [] - responseList.append(translateGPT(name['name'], newContext, False)) + responseList.append(translateGPT(name['name'], newContext, True)) + if 'Actors' in context: + responseList.append(translateGPT(name['profile'], '', True)) + if 'Armors' in context or 'Weapons' in context: - responseList.append(translateGPT(name['description'], newContext, True)) + responseList.append(translateGPT(name['description'], '', True)) # Extract all our translations in a list from response for i in range(len(responseList)): @@ -366,6 +369,10 @@ def searchNames(name, pbar, context): # Set Data name['name'] = responseList[0].strip('.\"') + if 'Actors' in context: + translatedText = textwrap.fill(responseList[1], LISTWIDTH) + name['profile'] = translatedText.strip('\"') + if 'Armors' in context or 'Weapons' in context: translatedText = textwrap.fill(responseList[1], LISTWIDTH) name['description'] = translatedText.strip('\"') @@ -692,16 +699,16 @@ def searchCodes(page, pbar): translatedText = translatedText.replace(' 。', '.') # Need to remove outside code and put it back later - startString = re.search(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', jaString) - jaString = re.sub(r'^[^ぁ-んァ-ン一-龯\<\>【】]+', '', jaString) - endString = re.search(r'[^ぁ-んァ-ン一-龯【】 。！？]+$', jaString) - jaString = re.sub(r'[^ぁ-んァ-ン一-龯【】 。！？]+$', '', jaString) + startString = re.search(r'^[^ぁ-んァ-ン一-龯\<\>【】（）Ａ-Ｚ０-９]+', jaString) + jaString = re.sub(r'^[^ぁ-んァ-ン一-龯\<\>【】（）Ａ-Ｚ０-９]+', '', jaString) + endString = re.search(r'[^ぁ-んァ-ン一-龯【】 。！？（）Ａ-Ｚ０-９]+$', jaString) + jaString = re.sub(r'[^ぁ-んァ-ン一-龯【】 。！？（）Ａ-Ｚ０-９]+$', '', jaString) if startString is None: startString = '' else: startString = startString.group() if endString is None: endString = '' else: endString = endString.group() - response = translateGPT(jaString, 'Reply with only the english translation. NEVER reply in anything other than English. I repeat, only reply with the english translation of the original text. The original text is a dialogue choice.', True) + response = translateGPT(jaString, 'Keep your reply prompt.', True) translatedText = response[0] # Remove characters that may break scripts @@ -896,7 +903,7 @@ def searchSystem(data, pbar): # Skill Types for i in range(len(data['skillTypes'])): - response = translateGPT(data['skillTypes'][i], 'Reply with only the english translated skill type', False) + response = translateGPT(data['skillTypes'][i], 'Reply with only the english translation', False) tokens += response[1] data['skillTypes'][i] = response[0].strip('.\"') pbar.update(1) @@ -931,7 +938,7 @@ def translateGPT(t, history, fullPromptFlag): # If ESTIMATE is True just count this as an execution and return. if ESTIMATE: global TOKENS - enc = tiktoken.encoding_for_model("gpt-3.5-turbo") + enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0613") TOKENS += len(enc.encode(t)) * 2 + len(enc.encode(history)) + len(enc.encode(PROMPT)) return (t, 0)