From 446dd1d83f374b123eb4515dfd57a51cf0477863 Mon Sep 17 00:00:00 2001 From: Dazed Date: Thu, 17 Aug 2023 08:35:46 -0500 Subject: [PATCH] Adjust searchCodes to accept scenario.json --- modules/csvtl.py | 26 ++++---- modules/rpgmakermvmz.py | 143 ++++++++++++++++++++++++---------------- 2 files changed, 101 insertions(+), 68 deletions(-) diff --git a/modules/csvtl.py b/modules/csvtl.py index 07c09c7..c11b411 100644 --- a/modules/csvtl.py +++ b/modules/csvtl.py @@ -41,7 +41,7 @@ def handleCSV(filename, estimate): global ESTIMATE, TOKENS, TOTALTOKENS, TOTALCOST ESTIMATE = estimate - with open('translated/' + filename, 'w+t', newline='', encoding='UTF-8') as writeFile: + with open('translated/' + filename, 'w+t', newline='', encoding='utf-16-le') as writeFile: start = time.time() translatedData = openFiles(filename, writeFile) @@ -64,7 +64,7 @@ def handleCSV(filename, estimate): return getResultString(['', TOTALTOKENS, None], end - start, 'TOTAL') def openFiles(filename, writeFile): - with open('files/' + filename, 'r', encoding='shift-jis') as readFile, writeFile: + with open('files/' + filename, 'r', encoding='utf-16-le') as readFile, writeFile: translatedData = parseCSV(readFile, writeFile, filename) return translatedData @@ -171,7 +171,7 @@ def translateCSV(row, pbar, writer, textHistory, format): continue jaString = row[i] - matchList = re.findall(r':name\[(.+?),.+?\](.+?[」）]+?)', jaString) + matchList = re.findall(r':name\[(.+?),.+?\](.+?[」）\"。]+)', jaString) for match in matchList: speaker = match[0] @@ -183,7 +183,8 @@ def translateCSV(row, pbar, writer, textHistory, format): tokens += response[1] # Translate Line - response = translateGPT(translatedSpeaker + ': ' + text, 'Previous Translated Text: ' + '|'.join(textHistory), True) + jaText = re.sub(r'([\u3000-\uffef])\1{3,}', r'\1\1\1', text) + response = translateGPT(translatedSpeaker + ': ' + jaText, 'Previous Translated Text: ' + '|'.join(textHistory), True) translatedText = response[0] tokens += response[1] @@ -194,29 +195,32 @@ def translateCSV(row, pbar, writer, textHistory, format): translatedText = re.sub(r'.+?: ', '', translatedText) # Set Data + translatedSpeaker = translatedSpeaker.replace('\"', '') translatedText = translatedText.replace('\"', '') translatedText = translatedText.replace('「', '') translatedText = translatedText.replace('」', '') - translatedText = translatedText.replace('\n', '') + row[i] = row[i].replace('\n', ' ') # Textwrap translatedText = textwrap.fill(translatedText, width=WIDTH) - translatedText = '「' + translatedText + '」\n' - row[i] = re.sub(rf'\b{re.escape(speaker)}\b', translatedSpeaker, row[i]) + translatedText = '「' + translatedText + '」' + row[i] = re.sub(rf':name\[({re.escape(speaker)}),', f':name[{translatedSpeaker},', row[i]) row[i] = row[i].replace(text, translatedText) - # Keep textHistory list at length maxHistory + # Keep History at fixed length. + with LOCK: + if len(textHistory) > maxHistory: + textHistory.pop(0) + with LOCK: - if len(textHistory) > maxHistory: - textHistory.pop(0) if not ESTIMATE: writer.writerow(row) pbar.update(1) except Exception as e: tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) - raise Exception(str(e) + '|Line:' + tracebackLineNo + '| Failed to translate: ' + jaString) + raise Exception(str(e) + '|Line:' + tracebackLineNo + '| Failed to translate: ' + text) return tokens diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 9b869a4..b702908 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -49,7 +49,7 @@ CODE355655 = False CODE357 = False CODE657 = False CODE356 = False -CODE320 = True +CODE320 = False CODE324 = False CODE111 = False CODE408 = False @@ -146,6 +146,10 @@ def openFiles(filename): elif 'System' in filename: translatedData = parseSystem(data, filename) + # Scenario File + elif 'Scenario' in filename: + translatedData = parseScenario(data, filename) + else: raise NameError(filename + ' Not Supported') @@ -354,6 +358,27 @@ def parseSystem(data, filename): return [data, totalTokens, e] return [data, totalTokens, None] +def parseScenario(data, filename): + totalTokens = 0 + totalLines = 0 + global LOCK + + # Get total for progress bar + for page in data.items(): + totalLines += len(page[1]) + + with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: + pbar.desc=filename + pbar.total=totalLines + with ThreadPoolExecutor(max_workers=THREADS) as executor: + futures = [executor.submit(searchCodes, page[1], pbar) for page in data.items() if page[1] is not None] + for future in as_completed(futures): + try: + totalTokens += future.result() + except Exception as e: + return [data, totalTokens, e] + return [data, totalTokens, None] + def searchThings(name, pbar): tokens = 0 @@ -458,7 +483,11 @@ def searchCodes(page, pbar): global NAMESLIST try: - for i in range(len(page['list'])): + if 'list' in page: + codeList = page['list'] + else: + codeList = page + for i in range(len(codeList)): with LOCK: pbar.update(1) @@ -466,12 +495,12 @@ def searchCodes(page, pbar): ### IF these crash or fail your game will do the same. Use the flags to skip codes. ## Event Code: 401 Show Text - if page['list'][i]['code'] == 401 and CODE401 == True or page['list'][i]['code'] == 405 and CODE405: + if codeList[i]['code'] == 401 and CODE401 == True or codeList[i]['code'] == 405 and CODE405: # Use this to place text later j = i # Grab String - jaString = page['list'][i]['parameters'][0] + jaString = codeList[i]['parameters'][0] # # Catch Speaker # if jaString.startswith('　'): @@ -482,12 +511,12 @@ def searchCodes(page, pbar): # Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior) currentGroup.append(jaString) - while (page['list'][i+1]['code'] == 401 or page['list'][i+1]['code'] == 405): - page['list'][i]['parameters'][0] = '' - page['list'][i]['code'] = 0 + while (codeList[i+1]['code'] == 401 or codeList[i+1]['code'] == 405): + codeList[i]['parameters'][0] = '' + codeList[i]['code'] = 0 i += 1 - jaString = page['list'][i]['parameters'][0] + jaString = codeList[i]['parameters'][0] jaString = re.sub(r'([\u3000-\uffef])\1{3,}', r'\1\1\1', jaString) currentGroup.append(jaString) @@ -561,10 +590,10 @@ def searchCodes(page, pbar): translatedText = translatedText.replace('\"', '') translatedText = translatedText.replace('\\CL ', '\\CL') translatedText = translatedText.replace('\\CL', '\\CL ') - page['list'][i]['parameters'][0] = '' - page['list'][i]['code'] = 0 - page['list'][j]['parameters'][0] = translatedText - page['list'][j]['code'] = 401 + codeList[i]['parameters'][0] = '' + codeList[i]['code'] = 0 + codeList[j]['parameters'][0] = translatedText + codeList[j]['code'] = 401 speaker = '' match = [] @@ -574,13 +603,13 @@ def searchCodes(page, pbar): currentGroup = [] ## Event Code: 122 [Set Variables] - if page['list'][i]['code'] == 122 and CODE122 == True: + if codeList[i]['code'] == 122 and CODE122 == True: # This is going to be the var being set. (IMPORTANT) - varNum = page['list'][i]['parameters'][0] + varNum = codeList[i]['parameters'][0] if varNum != 1238: continue - jaString = page['list'][i]['parameters'][4] + jaString = codeList[i]['parameters'][4] if type(jaString) != str: continue @@ -609,12 +638,12 @@ def searchCodes(page, pbar): # Set Data translatedText = jaString - page['list'][i]['parameters'][4] = translatedText + codeList[i]['parameters'][4] = translatedText ## Event Code: 357 [Picture Text] [Optional] - if page['list'][i]['code'] == 357 and CODE357 == True: - if 'message' in page['list'][i]['parameters'][3]: - jaString = page['list'][i]['parameters'][3]['message'] + if codeList[i]['code'] == 357 and CODE357 == True: + if 'message' in codeList[i]['parameters'][3]: + jaString = codeList[i]['parameters'][3]['message'] if type(jaString) != str: continue @@ -645,12 +674,12 @@ def searchCodes(page, pbar): translatedText = textwrap.fill(translatedText, width=WIDTH) # Set Data - page['list'][i]['parameters'][3]['message'] = startString + translatedText + codeList[i]['parameters'][3]['message'] = startString + translatedText ## Event Code: 657 [Picture Text] [Optional] - if page['list'][i]['code'] == 657 and CODE657 == True: - if 'text' in page['list'][i]['parameters'][0]: - jaString = page['list'][i]['parameters'][0] + if codeList[i]['code'] == 657 and CODE657 == True: + if 'text' in codeList[i]['parameters'][0]: + jaString = codeList[i]['parameters'][0] if type(jaString) != str: continue @@ -692,11 +721,11 @@ def searchCodes(page, pbar): # Set Data if '\\' in jaString: print('Hi') - page['list'][i]['parameters'][0] = translatedText + codeList[i]['parameters'][0] = translatedText ## Event Code: 101 [Name] [Optional] - if page['list'][i]['code'] == 101 and CODE101 == True: - jaString = page['list'][i]['parameters'][4] + if codeList[i]['code'] == 101 and CODE101 == True: + jaString = codeList[i]['parameters'][4] if type(jaString) != str: continue @@ -733,22 +762,22 @@ def searchCodes(page, pbar): # Set Data speaker = translatedText - page['list'][i]['parameters'][4] = translatedText + codeList[i]['parameters'][4] = translatedText ## Event Code: 355 or 655 Scripts [Optional] - if (page['list'][i]['code'] == 355) and CODE355655 == True: - jaString = page['list'][i]['parameters'][0] + if (codeList[i]['code'] == 355) and CODE355655 == True: + jaString = codeList[i]['parameters'][0] # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): continue # Want to translate this script - if page['list'][i]['code'] == 355 and '_logWindow.push' not in jaString: + if codeList[i]['code'] == 355 and '_logWindow.push' not in jaString: continue # Don't want to touch certain scripts - if page['list'][i]['code'] == 655 and '.' in jaString: + if codeList[i]['code'] == 655 and '.' in jaString: continue # Need to remove outside code and put it back later @@ -773,11 +802,11 @@ def searchCodes(page, pbar): # Set Data translatedText = startString + translatedText + endString - page['list'][i]['parameters'][0] = translatedText + codeList[i]['parameters'][0] = translatedText ## Event Code: 408 (Script) - if (page['list'][i]['code'] == 408) and CODE408 == True: - jaString = page['list'][i]['parameters'][0] + if (codeList[i]['code'] == 408) and CODE408 == True: + jaString = codeList[i]['parameters'][0] # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): @@ -812,11 +841,11 @@ def searchCodes(page, pbar): translatedText = translatedText.replace('"', '\"') # Set Data - page['list'][i]['parameters'][0] = translatedText + codeList[i]['parameters'][0] = translatedText ## Event Code: 108 (Script) - if (page['list'][i]['code'] == 108) and CODE108 == True: - jaString = page['list'][i]['parameters'][0] + if (codeList[i]['code'] == 108) and CODE108 == True: + jaString = codeList[i]['parameters'][0] # If there isn't any Japanese in the text just skip if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString): @@ -851,11 +880,11 @@ def searchCodes(page, pbar): translatedText = translatedText.replace('"', '\"') # Set Data - page['list'][i]['parameters'][0] = translatedText + codeList[i]['parameters'][0] = translatedText ## Event Code: 356 D_TEXT - if page['list'][i]['code'] == 356 and CODE356 == True: - jaString = page['list'][i]['parameters'][0] + if codeList[i]['code'] == 356 and CODE356 == True: + jaString = codeList[i]['parameters'][0] oldjaString = jaString # If there isn't any Japanese in the text just skip @@ -880,14 +909,14 @@ def searchCodes(page, pbar): # Using this to keep track of 401's in a row. Throws IndexError at EndOfList (Expected Behavior) currentGroup.append(dtext) - while (page['list'][i+1]['code'] == 356): + while (codeList[i+1]['code'] == 356): # Want to translate this script - if 'D_TEXT ' not in page['list'][i+1]['parameters'][0]: + if 'D_TEXT ' not in codeList[i+1]['parameters'][0]: break - page['list'][i]['parameters'][0] = '' + codeList[i]['parameters'][0] = '' i += 1 - jaString = page['list'][i]['parameters'][0] + jaString = codeList[i]['parameters'][0] dtext = re.findall(r'(?<=\s)(.*?)(?=\s)', jaString)[0] currentGroup.append(dtext) @@ -917,15 +946,15 @@ def searchCodes(page, pbar): translatedText = arg1 + ' ' + translatedText + ' ' + arg2 # Set Data - page['list'][i]['parameters'][0] = translatedText + codeList[i]['parameters'][0] = translatedText # Clear Group currentGroup = [] ### Event Code: 102 Show Choice - if page['list'][i]['code'] == 102 and CODE102 == True: - for choice in range(len(page['list'][i]['parameters'][0])): - jaString = page['list'][i]['parameters'][0][choice] + if codeList[i]['code'] == 102 and CODE102 == True: + for choice in range(len(codeList[i]['parameters'][0])): + jaString = codeList[i]['parameters'][0][choice] jaString = jaString.replace(' 。', '.') # Need to remove outside code and put it back later @@ -952,12 +981,12 @@ def searchCodes(page, pbar): # Set Data tokens += response[1] - page['list'][i]['parameters'][0][choice] = startString + translatedText + endString + codeList[i]['parameters'][0][choice] = startString + translatedText + endString ### Event Code: 111 Script - if page['list'][i]['code'] == 111 and CODE111 == True: - for j in range(len(page['list'][i]['parameters'])): - jaString = page['list'][i]['parameters'][j] + if codeList[i]['code'] == 111 and CODE111 == True: + for j in range(len(codeList[i]['parameters'])): + jaString = codeList[i]['parameters'][j] # Check if String if type(jaString) != str: @@ -988,11 +1017,11 @@ def searchCodes(page, pbar): # Set Data translatedText = jaString - page['list'][i]['parameters'][j] = translatedText + codeList[i]['parameters'][j] = translatedText ### Event Code: 320 Set Variable - if page['list'][i]['code'] == 320 and CODE320 == True: - jaString = page['list'][i]['parameters'][1] + if codeList[i]['code'] == 320 and CODE320 == True: + jaString = codeList[i]['parameters'][1] if type(jaString) != str: continue @@ -1014,7 +1043,7 @@ def searchCodes(page, pbar): translatedText = translatedText.replace(char, '') # Set Data - page['list'][i]['parameters'][1] = translatedText + codeList[i]['parameters'][1] = translatedText except IndexError as e: # This is part of the logic so we just pass it @@ -1044,7 +1073,7 @@ def searchCodes(page, pbar): translatedText = translatedText.replace('ッ', '') translatedText = translatedText.replace('っ', '') translatedText = translatedText.replace('\"', '') - page['list'][i]['parameters'][0] = translatedText + codeList[i]['parameters'][0] = translatedText speaker = '' match = []